Initial push
This commit is contained in:
@@ -0,0 +1,440 @@
|
||||
package migrate
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/arescom/docker-migrate/internal/dkr"
|
||||
"github.com/arescom/docker-migrate/internal/job"
|
||||
"github.com/arescom/docker-migrate/internal/spec"
|
||||
)
|
||||
|
||||
// PackageFormat selects how the finished package is laid out on disk.
|
||||
type PackageFormat string
|
||||
|
||||
const (
|
||||
// FormatDir leaves an unpacked directory, easiest to inspect and to copy
|
||||
// onto a USB stick that is already mounted.
|
||||
FormatDir PackageFormat = "dir"
|
||||
// FormatTar produces a single .tar file, easiest to move around.
|
||||
FormatTar PackageFormat = "tar"
|
||||
)
|
||||
|
||||
// Packager writes a self-contained migration package: the container specs, the
|
||||
// data archives, optionally the images, and a shell installer that replays it
|
||||
// all on a target that has nothing but docker.
|
||||
type Packager struct {
|
||||
Src *dkr.Client
|
||||
Containers []spec.Container
|
||||
Volumes []spec.Volume
|
||||
Networks []spec.Network
|
||||
Plan spec.Plan
|
||||
|
||||
// OutputDir is the directory packages are created under.
|
||||
OutputDir string
|
||||
// Format selects a directory or a single tar file.
|
||||
Format PackageFormat
|
||||
// SourceHost is recorded in the manifest.
|
||||
SourceHost string
|
||||
}
|
||||
|
||||
// Result describes the produced package.
|
||||
type Result struct {
|
||||
Path string `json:"path"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Run builds the package, reporting progress into j.
|
||||
func (p *Packager) Run(ctx context.Context, j *job.Job) (*Result, error) {
|
||||
opts := p.Plan.Options
|
||||
name := p.Plan.PackageName
|
||||
if name == "" {
|
||||
name = "docker-migration-" + time.Now().Format("20060102-150405")
|
||||
}
|
||||
name = sanitize(name)
|
||||
|
||||
root := filepath.Join(p.OutputDir, name)
|
||||
if _, err := os.Stat(root); err == nil {
|
||||
return nil, fmt.Errorf("package %s already exists in %s", name, p.OutputDir)
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create package directory: %w", err)
|
||||
}
|
||||
cleanup := true
|
||||
defer func() {
|
||||
if cleanup {
|
||||
os.RemoveAll(root)
|
||||
}
|
||||
}()
|
||||
|
||||
byID := map[string]*spec.Container{}
|
||||
for i := range p.Containers {
|
||||
byID[p.Containers[i].ID] = &p.Containers[i]
|
||||
}
|
||||
|
||||
var prepared []*Prepared
|
||||
for _, sel := range p.Plan.Items {
|
||||
if !sel.Include {
|
||||
continue
|
||||
}
|
||||
pr, err := Prepare(byID[sel.ContainerID], sel, p.Volumes, p.Networks)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("container %s: %w", sel.ContainerID, err)
|
||||
}
|
||||
prepared = append(prepared, pr)
|
||||
}
|
||||
if len(prepared) == 0 {
|
||||
return nil, errors.New("nothing selected to migrate")
|
||||
}
|
||||
|
||||
man := spec.Manifest{
|
||||
FormatVersion: 1,
|
||||
CreatedAt: time.Now(),
|
||||
CreatedBy: "docker-migrate",
|
||||
SourceHost: p.SourceHost,
|
||||
Options: opts,
|
||||
Items: p.Plan.Items,
|
||||
}
|
||||
if v, err := p.Src.Ping(ctx); err == nil {
|
||||
man.DockerVersion = v
|
||||
}
|
||||
|
||||
savedImages := map[string]string{} // image ref -> payload path
|
||||
|
||||
for _, pr := range prepared {
|
||||
item := j.AddItem(pr.Source.ID, pr.Source.Name)
|
||||
for _, n := range pr.Source.Warnings {
|
||||
j.AddItemWarning(item, "%s: %s", pr.Source.Name, n)
|
||||
}
|
||||
for _, n := range pr.Notes {
|
||||
j.AddItemWarning(item, "%s: %s", pr.Source.Name, n)
|
||||
}
|
||||
|
||||
err := p.packOne(ctx, j, item, root, pr, opts, &man, savedImages)
|
||||
if err != nil {
|
||||
j.SetItemState(item, job.StateFailed, err)
|
||||
return nil, fmt.Errorf("%s: %w", pr.Source.Name, err)
|
||||
}
|
||||
j.SetItemState(item, job.StateSucceeded, nil)
|
||||
|
||||
man.Containers = append(man.Containers, *pr.Target)
|
||||
man.Volumes = append(man.Volumes, pr.Volumes...)
|
||||
for _, n := range pr.Networks {
|
||||
if !hasNetwork(man.Networks, n.Name) {
|
||||
man.Networks = append(man.Networks, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := writeJSON(filepath.Join(root, "manifest.json"), man); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
installer := renderInstaller(prepared, &man, savedImages, name)
|
||||
if err := os.WriteFile(filepath.Join(root, "install.sh"), []byte(installer), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("write installer: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "README.txt"), []byte(renderReadme(name, prepared, opts)), 0o644); err != nil {
|
||||
return nil, fmt.Errorf("write readme: %w", err)
|
||||
}
|
||||
j.Logf(job.LevelInfo, "", "wrote installer, manifest and readme")
|
||||
|
||||
if p.Format == FormatTar {
|
||||
tarPath := root + ".tar"
|
||||
j.Logf(job.LevelInfo, "", "packing %s into a single archive", name)
|
||||
size, err := tarDirectory(ctx, root, tarPath, name)
|
||||
if err != nil {
|
||||
os.Remove(tarPath)
|
||||
return nil, fmt.Errorf("create package archive: %w", err)
|
||||
}
|
||||
os.RemoveAll(root)
|
||||
cleanup = false
|
||||
j.SetArtifact(tarPath, size)
|
||||
return &Result{Path: tarPath, Bytes: size, Name: name + ".tar"}, nil
|
||||
}
|
||||
|
||||
size, _ := dirSize(root)
|
||||
cleanup = false
|
||||
j.SetArtifact(root, size)
|
||||
return &Result{Path: root, Bytes: size, Name: name}, nil
|
||||
}
|
||||
|
||||
func (p *Packager) packOne(
|
||||
ctx context.Context, j *job.Job, item *job.Item, root string,
|
||||
pr *Prepared, opts spec.Options, man *spec.Manifest, savedImages map[string]string,
|
||||
) error {
|
||||
compress := opts.Compress
|
||||
level := opts.CompressLevel
|
||||
|
||||
// Image.
|
||||
if pr.Selection.MigrateImage && pr.Selection.ImageMode != spec.ImageSkip && pr.Selection.ImageMode != spec.ImagePull {
|
||||
ref := pr.Target.Image
|
||||
if _, done := savedImages[ref]; !done {
|
||||
size := p.Src.ImageSizeBytes(ctx, ref)
|
||||
st := j.AddStep(item, "image", "save image "+ref, size)
|
||||
j.StartStep(st)
|
||||
if opts.DryRun {
|
||||
j.SkipStep(st, "dry run: image not written")
|
||||
} else {
|
||||
rel := filepath.ToSlash(filepath.Join("images", sanitize(ref)+tarExt(compress)))
|
||||
payload, err := p.streamToFile(ctx, j, st, filepath.Join(root, filepath.FromSlash(rel)), rel, compress, level,
|
||||
func() (io.ReadCloser, error) { return p.Src.SaveImage(ctx, ref) })
|
||||
j.FinishStep(st, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("save image %s: %w", ref, err)
|
||||
}
|
||||
payload.Kind, payload.Image = "image", ref
|
||||
man.Payloads = append(man.Payloads, *payload)
|
||||
savedImages[ref] = rel
|
||||
}
|
||||
} else {
|
||||
st := j.AddStep(item, "image", "image "+ref+" already in package", 0)
|
||||
j.StartStep(st)
|
||||
j.SkipStep(st, "shared with another container")
|
||||
}
|
||||
} else if pr.Selection.ImageMode == spec.ImagePull {
|
||||
j.Logf(job.LevelInfo, item.ID, "%s: image %s will be pulled by the installer", pr.Source.Name, pr.Target.Image)
|
||||
}
|
||||
|
||||
// Data. The source container is stopped for the duration when asked.
|
||||
if len(pr.Transfers) == 0 {
|
||||
return nil
|
||||
}
|
||||
restore, err := p.quiesce(ctx, j, item, pr, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if restore != nil {
|
||||
restore()
|
||||
}
|
||||
}()
|
||||
|
||||
for i, t := range pr.Transfers {
|
||||
st := j.AddStep(item, fmt.Sprintf("data-%d", i), t.Label, t.SizeBytes)
|
||||
j.StartStep(st)
|
||||
if opts.DryRun {
|
||||
j.SkipStep(st, "dry run: data not written")
|
||||
continue
|
||||
}
|
||||
rel := filepath.ToSlash(filepath.Join("data", sanitize(pr.ContainerName()),
|
||||
fmt.Sprintf("%02d-%s%s", i, sanitize(strings.Trim(t.Destination, "/")), tarExt(compress))))
|
||||
payload, err := p.streamToFile(ctx, j, st, filepath.Join(root, filepath.FromSlash(rel)), rel, compress, level,
|
||||
func() (io.ReadCloser, error) { return p.Src.CopyOut(ctx, pr.Source.ID, t.SourcePath) })
|
||||
j.FinishStep(st, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("archive %s: %w", t.Label, err)
|
||||
}
|
||||
payload.Kind = "mount"
|
||||
payload.Container = pr.ContainerName()
|
||||
payload.Destination = t.Destination
|
||||
man.Payloads = append(man.Payloads, *payload)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// streamToFile copies a stream to a file inside the package, optionally
|
||||
// gzipping it, while counting bytes and computing a checksum.
|
||||
func (p *Packager) streamToFile(
|
||||
ctx context.Context, j *job.Job, st *job.Step,
|
||||
absPath, relPath string, compress bool, level int,
|
||||
open func() (io.ReadCloser, error),
|
||||
) (*spec.Payload, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(absPath), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
src, err := open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
f, err := os.Create(absPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
// The checksum covers the bytes as stored, so the installer can verify the
|
||||
// file it is about to feed to docker.
|
||||
out := io.MultiWriter(f, hash)
|
||||
|
||||
counted := job.NewCountingReader(src, j, st)
|
||||
var copyErr error
|
||||
if compress {
|
||||
gz, gerr := gzip.NewWriterLevel(out, gzipLevel(nil, level))
|
||||
if gerr != nil {
|
||||
return nil, gerr
|
||||
}
|
||||
_, copyErr = io.Copy(gz, counted)
|
||||
if cerr := gz.Close(); copyErr == nil {
|
||||
copyErr = cerr
|
||||
}
|
||||
} else {
|
||||
_, copyErr = io.Copy(out, counted)
|
||||
}
|
||||
counted.Flush()
|
||||
if copyErr != nil {
|
||||
return nil, copyErr
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return &spec.Payload{
|
||||
Path: relPath,
|
||||
Bytes: info.Size(),
|
||||
SHA256: hex.EncodeToString(hash.Sum(nil)),
|
||||
Compressed: compress,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Packager) quiesce(ctx context.Context, j *job.Job, item *job.Item, pr *Prepared, opts spec.Options) (func(), error) {
|
||||
if opts.DryRun {
|
||||
return nil, nil
|
||||
}
|
||||
wasRunning := pr.Source.State == "running"
|
||||
if !pr.Selection.StopSourceDuringCopy {
|
||||
if wasRunning {
|
||||
j.AddItemWarning(item,
|
||||
"archiving %s while it is running; data written during the copy may be inconsistent", pr.Source.Name)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if !wasRunning {
|
||||
return nil, nil
|
||||
}
|
||||
st := j.AddStep(item, "quiesce", "stop source "+pr.Source.Name, 0)
|
||||
j.StartStep(st)
|
||||
err := p.Src.Stop(ctx, pr.Source.ID, 30*time.Second)
|
||||
j.FinishStep(st, err)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stop source container: %w", err)
|
||||
}
|
||||
return func() {
|
||||
// Building a package does not move the workload anywhere, so the source
|
||||
// is always put back the way it was found.
|
||||
if err := p.Src.Start(context.WithoutCancel(ctx), pr.Source.ID); err != nil {
|
||||
j.AddItemWarning(item, "could not restart source container: %v", err)
|
||||
} else {
|
||||
j.Logf(job.LevelInfo, item.ID, "source container %s restarted", pr.Source.Name)
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func tarExt(compress bool) string {
|
||||
if compress {
|
||||
return ".tar.gz"
|
||||
}
|
||||
return ".tar"
|
||||
}
|
||||
|
||||
func writeJSON(path string, v any) error {
|
||||
b, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, b, 0o644)
|
||||
}
|
||||
|
||||
func hasNetwork(ns []spec.Network, name string) bool {
|
||||
for _, n := range ns {
|
||||
if n.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// tarDirectory packs a package directory into a single tar file, keeping the
|
||||
// directory name as the archive's top-level entry.
|
||||
func tarDirectory(ctx context.Context, dir, dest, prefix string) (int64, error) {
|
||||
f, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
tw := tar.NewWriter(f)
|
||||
|
||||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
rel, err := filepath.Rel(dir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := prefix
|
||||
if rel != "." {
|
||||
name = prefix + "/" + filepath.ToSlash(rel)
|
||||
}
|
||||
hdr, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hdr.Name = name
|
||||
if info.IsDir() {
|
||||
hdr.Name += "/"
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
src, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
_, err = io.Copy(tw, src)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
tw.Close()
|
||||
return 0, err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func dirSize(dir string) (int64, error) {
|
||||
var total int64
|
||||
err := filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return total, err
|
||||
}
|
||||
Reference in New Issue
Block a user