Initial push

This commit is contained in:
2026-08-11 09:00:01 +02:00
commit fe8b354adc
54 changed files with 12640 additions and 0 deletions
+309
View File
@@ -0,0 +1,309 @@
//go:build e2e
// Package e2e exercises the full pipeline against a real Docker daemon:
// inventory -> plan -> package build -> generated installer -> restored
// container, and then checks that the data actually arrived.
//
// It needs a Linux host with a local Docker daemon and bash. Run it with:
//
// go test -tags e2e ./test/... -v
package e2e
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/arescom/docker-migrate/internal/dkr"
"github.com/arescom/docker-migrate/internal/job"
"github.com/arescom/docker-migrate/internal/migrate"
"github.com/arescom/docker-migrate/internal/spec"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/volume"
)
const (
testImage = "alpine:3.20"
srcName = "dmtest-src"
volName = "dmtest-vol"
restoreSufix = "-restored"
)
func TestPackageRoundTrip(t *testing.T) {
if _, err := exec.LookPath("bash"); err != nil {
t.Skip("bash is required to run the generated installer")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("the docker CLI is required by the generated installer")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
c, err := dkr.New("")
if err != nil {
t.Skipf("no docker daemon available: %v", err)
}
defer c.Close()
if _, err := c.Ping(ctx); err != nil {
t.Skipf("no docker daemon available: %v", err)
}
bindDir := t.TempDir()
if err := os.WriteFile(filepath.Join(bindDir, "app.conf"), []byte("mode=production\n"), 0o644); err != nil {
t.Fatal(err)
}
cleanup(ctx, t, c, srcName, srcName+restoreSufix)
t.Cleanup(func() {
cctx, ccancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer ccancel()
cleanup(cctx, t, c, srcName, srcName+restoreSufix)
})
pullImage(ctx, t, c, testImage)
createSource(ctx, t, c, bindDir)
// --- inventory -------------------------------------------------------
inv, err := c.Inventory(ctx)
if err != nil {
t.Fatal(err)
}
var src *spec.Container
for i := range inv.Containers {
if inv.Containers[i].Name == srcName {
src = &inv.Containers[i]
}
}
if src == nil {
t.Fatalf("source container %s not found in the inventory", srcName)
}
t.Logf("source: %s state=%s image=%s mounts=%d", src.Name, src.State, src.Image, len(src.Mounts))
wantMounts := map[string]spec.MountKind{
"/data": spec.MountVolume,
"/conf": spec.MountBind,
"/anon": spec.MountAnonymous,
}
for dest, kind := range wantMounts {
found := false
for _, m := range src.Mounts {
if m.Destination == dest {
found = true
if m.Kind != kind {
t.Errorf("mount %s: kind %s, want %s", dest, m.Kind, kind)
}
}
}
if !found {
t.Errorf("mount %s missing from the inventory", dest)
}
}
// --- build the package ----------------------------------------------
sel := spec.DefaultSelection(src)
sel.Include = true
// The image is already on this daemon, so there is no point carrying it
// through a round trip that restores onto the same host.
sel.MigrateImage = true
sel.ImageMode = spec.ImageSkip
opts := spec.DefaultOptions()
opts.Conflict = spec.ConflictRename
opts.RenameSuffix = restoreSufix
outDir := t.TempDir()
packager := &migrate.Packager{
Src: c, Containers: inv.Containers, Volumes: inv.Volumes, Networks: inv.Networks,
Plan: spec.Plan{Items: []spec.ItemSelection{sel}, Options: opts, PackageName: "dmtest"},
OutputDir: outDir, Format: migrate.FormatDir, SourceHost: inv.Host,
}
jobs := job.NewManager()
var res *migrate.Result
var runErr error
j := jobs.Run(ctx, job.KindPackage, "e2e", false, func(ctx context.Context, j *job.Job) error {
res, runErr = packager.Run(ctx, j)
return runErr
})
<-j.Done()
snap := j.Snapshot()
for _, l := range snap.Log {
t.Logf("[%s] %s", l.Level, l.Message)
}
if snap.State != job.StateSucceeded {
t.Fatalf("package job %s: %s", snap.State, snap.Error)
}
t.Logf("package built at %s (%d bytes)", res.Path, res.Bytes)
for _, want := range []string{"install.sh", "manifest.json", "README.txt"} {
if _, err := os.Stat(filepath.Join(res.Path, want)); err != nil {
t.Fatalf("package is missing %s: %v", want, err)
}
}
// --- dry run first ---------------------------------------------------
if out, err := runInstaller(res.Path, "--dry-run", "--yes"); err != nil {
t.Fatalf("installer dry run failed: %v\n%s", err, out)
} else {
t.Logf("dry run ok:\n%s", indent(out))
}
if _, err := containerExists(ctx, c, srcName+restoreSufix); err == nil {
t.Fatal("dry run created a container; it must change nothing")
}
// --- real restore ----------------------------------------------------
out, err := runInstaller(res.Path, "--yes", "--conflict", "rename", "--rename-suffix", restoreSufix)
if err != nil {
t.Fatalf("installer failed: %v\n%s", err, out)
}
t.Logf("restore output:\n%s", indent(out))
restored := srcName + restoreSufix
if _, err := containerExists(ctx, c, restored); err != nil {
t.Fatalf("restored container %s does not exist: %v", restored, err)
}
// --- verify the data actually travelled ------------------------------
checks := []struct {
path, want string
}{
{"/data/hello.txt", "hello from the volume"},
{"/data/nested/deep.txt", "nested payload"},
{"/conf/app.conf", "mode=production"},
{"/anon/anon.txt", "anonymous volume payload"},
}
for _, chk := range checks {
got, err := readFileFromContainer(ctx, c, restored, chk.path)
if err != nil {
t.Errorf("reading %s from %s: %v", chk.path, restored, err)
continue
}
if !strings.Contains(got, chk.want) {
t.Errorf("%s = %q, want it to contain %q", chk.path, got, chk.want)
} else {
t.Logf("verified %s", chk.path)
}
}
// The read-only mount must still be read-only on the restored container.
insp, err := c.API().ContainerInspect(ctx, restored)
if err != nil {
t.Fatal(err)
}
for _, m := range insp.Mounts {
if m.Destination == "/conf" && m.RW {
t.Error("/conf was read-only on the source but is writable on the target")
}
}
}
func createSource(ctx context.Context, t *testing.T, c *dkr.Client, bindDir string) {
t.Helper()
api := c.API()
if _, err := api.VolumeCreate(ctx, volume.CreateOptions{Name: volName}); err != nil {
t.Fatal(err)
}
script := strings.Join([]string{
"echo 'hello from the volume' > /data/hello.txt",
"mkdir -p /data/nested",
"echo 'nested payload' > /data/nested/deep.txt",
"ln -sf hello.txt /data/link.txt",
"echo 'anonymous volume payload' > /anon/anon.txt",
"sleep 3600",
}, " && ")
resp, err := api.ContainerCreate(ctx,
&container.Config{
Image: testImage,
Cmd: []string{"sh", "-c", script},
Env: []string{"DM_TEST=1", "DM_QUOTED=a'b\"c"},
Labels: map[string]string{"dm.test": "yes"},
},
&container.HostConfig{
Mounts: []mount.Mount{
{Type: mount.TypeVolume, Source: volName, Target: "/data"},
{Type: mount.TypeBind, Source: bindDir, Target: "/conf", ReadOnly: true},
{Type: mount.TypeVolume, Target: "/anon"},
},
RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped},
},
nil, nil, srcName)
if err != nil {
t.Fatal(err)
}
if err := api.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {
t.Fatal(err)
}
// Give the entrypoint script time to write the files.
time.Sleep(2 * time.Second)
}
func pullImage(ctx context.Context, t *testing.T, c *dkr.Client, ref string) {
t.Helper()
if c.ImageExists(ctx, ref) {
return
}
rc, err := c.API().ImagePull(ctx, ref, image.PullOptions{})
if err != nil {
t.Fatalf("pull %s: %v", ref, err)
}
defer rc.Close()
if _, err := io.Copy(io.Discard, rc); err != nil {
t.Fatalf("pull %s: %v", ref, err)
}
}
func cleanup(ctx context.Context, t *testing.T, c *dkr.Client, names ...string) {
t.Helper()
api := c.API()
for _, n := range names {
_ = api.ContainerRemove(ctx, n, container.RemoveOptions{Force: true, RemoveVolumes: false})
}
_ = api.VolumeRemove(ctx, volName, true)
}
func containerExists(ctx context.Context, c *dkr.Client, name string) (string, error) {
j, err := c.API().ContainerInspect(ctx, name)
if err != nil {
return "", err
}
return j.ID, nil
}
// readFileFromContainer reads one file through the same archive API the
// migration itself uses, which avoids needing the container to be running.
func readFileFromContainer(ctx context.Context, c *dkr.Client, name, path string) (string, error) {
rc, err := c.CopyOut(ctx, name, path)
if err != nil {
return "", err
}
defer rc.Close()
return firstFileInTar(rc)
}
func runInstaller(dir string, args ...string) (string, error) {
cmd := exec.Command("bash", append([]string{"./install.sh"}, args...)...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
return string(out), err
}
func indent(s string) string {
var b strings.Builder
for _, l := range strings.Split(strings.TrimRight(s, "\n"), "\n") {
fmt.Fprintf(&b, " %s\n", l)
}
return b.String()
}
+14
View File
@@ -0,0 +1,14 @@
//go:build e2e
package e2e
import (
"errors"
"github.com/arescom/docker-migrate/internal/sshx"
)
// asHostKeyError unwraps err into a *sshx.HostKeyError if it is one.
func asHostKeyError(err error, target **sshx.HostKeyError) bool {
return errors.As(err, target)
}
+261
View File
@@ -0,0 +1,261 @@
//go:build e2e
package e2e
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/arescom/docker-migrate/internal/dkr"
"github.com/arescom/docker-migrate/internal/job"
"github.com/arescom/docker-migrate/internal/migrate"
"github.com/arescom/docker-migrate/internal/spec"
"github.com/arescom/docker-migrate/internal/sshx"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/volume"
)
// TestSSHMigration drives the host-to-host engine over a real SSH connection.
//
// Source and target are the same daemon, reached over SSH, so the whole
// transport is exercised — ssh, gzip streaming, the target's docker CLI, the
// staging container for read-only mounts and the verify step — without needing
// a second machine. The container and its volume are renamed on the way in so
// the copy is genuinely verified rather than finding the data already there.
//
// Configure it with:
//
// DM_SSH_HOST=10.0.0.5 DM_SSH_USER=root DM_SSH_KEY=/root/.ssh/id_ed25519 \
// go test -tags e2e ./test/... -run TestSSHMigration -v
const (
sshSrcName = "dmssh-src"
sshSrcVolume = "dmssh-vol"
sshDstName = "dmssh-src-viassh"
sshDstVolume = "dmssh-vol-viassh"
)
func TestSSHMigration(t *testing.T) {
host := os.Getenv("DM_SSH_HOST")
user := os.Getenv("DM_SSH_USER")
key := os.Getenv("DM_SSH_KEY")
if host == "" || user == "" || key == "" {
t.Skip("set DM_SSH_HOST, DM_SSH_USER and DM_SSH_KEY to run the SSH migration test")
}
keyPEM, err := os.ReadFile(key)
if err != nil {
t.Fatalf("read %s: %v", key, err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
c, err := dkr.New("")
if err != nil {
t.Skipf("no docker daemon available: %v", err)
}
defer c.Close()
bindDir := t.TempDir()
if err := os.WriteFile(filepath.Join(bindDir, "site.conf"), []byte("listen=8443\n"), 0o644); err != nil {
t.Fatal(err)
}
sshCleanup(ctx, c)
t.Cleanup(func() {
cctx, ccancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer ccancel()
sshCleanup(cctx, c)
})
pullImage(ctx, t, c, testImage)
createSSHSource(ctx, t, c, bindDir)
// --- connect ---------------------------------------------------------
hkPath := filepath.Join(t.TempDir(), "known_hosts")
hk, err := sshx.NewKnownHosts(hkPath)
if err != nil {
t.Fatal(err)
}
cfg := sshx.Config{
Name: "loopback", Host: host, Port: 22, User: user,
Auth: sshx.AuthKey, PrivateKey: string(keyPEM),
}
// An unknown host key must be refused before it is trusted; that is the
// whole point of the trust store.
if _, err := sshx.Dial(ctx, cfg, hk); err == nil {
t.Fatal("dialling an untrusted host key must fail")
} else {
var hke *sshx.HostKeyError
if !asHostKeyError(err, &hke) {
t.Fatalf("expected a host key error, got %v", err)
}
t.Logf("host key presented: %s %s", hke.KeyType, hke.Fingerprint)
}
if err := sshx.TrustFromProbe(ctx, cfg, hk, ""); err != nil {
t.Fatalf("trust host key: %v", err)
}
client, err := sshx.Dial(ctx, cfg, hk)
if err != nil {
t.Fatalf("dial after trusting: %v", err)
}
defer client.Close()
rd := sshx.NewRemoteDocker(client)
pre, err := rd.Preflight(ctx)
if err != nil {
t.Fatal(err)
}
t.Logf("target preflight: docker %s %s/%s gzip=%v free=%d problems=%v",
pre.ServerVersion, pre.OS, pre.Arch, pre.HasGzip, pre.DiskFreeBytes, pre.Problems)
if pre.ServerVersion == "" {
t.Fatalf("target docker unusable: %v", pre.Problems)
}
// --- plan ------------------------------------------------------------
inv, err := c.Inventory(ctx)
if err != nil {
t.Fatal(err)
}
var src *spec.Container
for i := range inv.Containers {
if inv.Containers[i].Name == sshSrcName {
src = &inv.Containers[i]
}
}
if src == nil {
t.Fatalf("source container %s not found", sshSrcName)
}
sel := spec.DefaultSelection(src)
sel.Include = true
sel.NameOverride = sshDstName
sel.ImageMode = spec.ImageSkip // same daemon; the image is already there
sel.StopSourceAfter = false // exercise the restart path
sel.Mounts["/data"] = spec.MountSelection{Action: spec.MountActionCopy, TargetName: sshDstVolume}
sel.MigratePorts = false // the source publishes nothing, but be explicit
opts := spec.DefaultOptions()
opts.Compress = true
opts.VerifyAfter = true
runner := &migrate.SSHRunner{
Src: c, Dst: rd,
Containers: inv.Containers, Volumes: inv.Volumes, Networks: inv.Networks,
Plan: spec.Plan{Items: []spec.ItemSelection{sel}, Options: opts},
}
jobs := job.NewManager()
j := jobs.Run(ctx, job.KindSSH, "ssh e2e", false, runner.Run)
<-j.Done()
snap := j.Snapshot()
for _, l := range snap.Log {
t.Logf("[%-5s] %s", l.Level, l.Message)
}
for _, it := range snap.Items {
for _, st := range it.Steps {
t.Logf(" step %-12s %-9s %8d bytes %s", st.ID, st.State, st.BytesDone, st.Label)
}
}
if snap.State != job.StateSucceeded {
t.Fatalf("migration %s: %s", snap.State, snap.Error)
}
if snap.BytesDone == 0 {
t.Error("no bytes were transferred; the copy did nothing")
}
// --- verify ----------------------------------------------------------
checks := []struct{ path, want string }{
{"/data/payload.txt", "streamed over ssh"},
{"/data/sub/inner.txt", "inner payload"},
{"/conf/site.conf", "listen=8443"},
{"/anon/scratch.txt", "anonymous over ssh"},
}
for _, chk := range checks {
got, err := readFileFromContainer(ctx, c, sshDstName, chk.path)
if err != nil {
t.Errorf("reading %s: %v", chk.path, err)
continue
}
if !strings.Contains(got, chk.want) {
t.Errorf("%s = %q, want it to contain %q", chk.path, got, chk.want)
} else {
t.Logf("verified %s", chk.path)
}
}
// The renamed volume must be a genuinely new one, holding the copied data.
if _, err := c.API().VolumeInspect(ctx, sshDstVolume); err != nil {
t.Errorf("renamed volume %s was not created: %v", sshDstVolume, err)
}
// The source was asked to keep running.
if st, err := c.State(ctx, sshSrcName); err != nil {
t.Errorf("source state: %v", err)
} else if st != "running" {
t.Errorf("source container should have been restarted, state is %q", st)
}
// The staging container used for the read-only mount must be gone.
out, err := rd.Run(ctx, "ps", "-a", "--format", "{{.Names}}")
if err != nil {
t.Fatal(err)
}
if strings.Contains(out, "dm-stage-") {
t.Errorf("a staging container was left behind:\n%s", out)
}
}
func createSSHSource(ctx context.Context, t *testing.T, c *dkr.Client, bindDir string) {
t.Helper()
api := c.API()
if _, err := api.VolumeCreate(ctx, volume.CreateOptions{Name: sshSrcVolume}); err != nil {
t.Fatal(err)
}
script := strings.Join([]string{
"echo 'streamed over ssh' > /data/payload.txt",
"mkdir -p /data/sub",
"echo 'inner payload' > /data/sub/inner.txt",
"echo 'anonymous over ssh' > /anon/scratch.txt",
"sleep 3600",
}, " && ")
resp, err := api.ContainerCreate(ctx,
&container.Config{
Image: testImage,
Cmd: []string{"sh", "-c", script},
Env: []string{"DM_MODE=ssh"},
},
&container.HostConfig{
Mounts: []mount.Mount{
{Type: mount.TypeVolume, Source: sshSrcVolume, Target: "/data"},
{Type: mount.TypeBind, Source: bindDir, Target: "/conf", ReadOnly: true},
{Type: mount.TypeVolume, Target: "/anon"},
},
},
nil, nil, sshSrcName)
if err != nil {
t.Fatal(err)
}
if err := api.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {
t.Fatal(err)
}
time.Sleep(2 * time.Second)
}
func sshCleanup(ctx context.Context, c *dkr.Client) {
api := c.API()
for _, n := range []string{sshSrcName, sshDstName} {
_ = api.ContainerRemove(ctx, n, container.RemoveOptions{Force: true})
}
for _, v := range []string{sshSrcVolume, sshDstVolume} {
_ = api.VolumeRemove(ctx, v, true)
}
}
+33
View File
@@ -0,0 +1,33 @@
//go:build e2e
package e2e
import (
"archive/tar"
"errors"
"io"
"strings"
)
// firstFileInTar returns the contents of the first regular file in an archive
// produced by the Docker archive API.
func firstFileInTar(r io.Reader) (string, error) {
tr := tar.NewReader(r)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
return "", errors.New("archive contains no regular file")
}
if err != nil {
return "", err
}
if hdr.Typeflag != tar.TypeReg {
continue
}
var b strings.Builder
if _, err := io.Copy(&b, tr); err != nil {
return "", err
}
return b.String(), nil
}
}