From db3259ee12190ceddaa9e9523561399bafae369f Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Wed, 21 Oct 2026 09:20:56 +0200 Subject: [PATCH 1/2] feat(compose): sync layers and resume an interrupted install An overlayfs snapshot commit is not durable: containerd syncs its metadata but not the snapshot's files, so a power cut during the unpack left committed snapshots without their directory. The install could not tell an intact snapshot from a torn one, so it deleted every leftover lease and extracted the whole app again. The unpack now applies every layer with the differ's sync_fs option: the snapshot's filesystem is synced before the snapshot is committed, so a committed snapshot is on disk. The install lease carries the label composectl/install.synced when the containerd reports 1.7.14 or later, the first version whose differ honours the option. The next install of the same app keeps such a lease, removes the snapshot the cut extraction left active, so that its space is free again, and goes on under the lease: the scan skips the layers whose snapshots exist. Every other install lease, including an unlabelled one of the same app, is deleted as before. The records are still created only after the final syncfs. The unpack watcher's poll interval is a variable so that the rig test can cut the extraction of a small layer. Assisted-by: Claude Code:claude-fable-5-1 Signed-off-by: Mike Sul --- pkg/compose/ctrd_install.go | 133 +++++++++++++++----- pkg/compose/ctrd_install_rig_test.go | 167 +++++++++++++++++++++++--- pkg/compose/ctrd_install_test.go | 49 ++++++-- pkg/compose/docker_client_test.go | 6 + pkg/compose/export_test.go | 4 + test/fixtures/composectl_cmds.go | 14 ++- test/integration/ctrd_install_test.go | 25 +++- 7 files changed, 339 insertions(+), 59 deletions(-) diff --git a/pkg/compose/ctrd_install.go b/pkg/compose/ctrd_install.go index fd93ae6..99c82ce 100644 --- a/pkg/compose/ctrd_install.go +++ b/pkg/compose/ctrd_install.go @@ -16,10 +16,12 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" dockerclient "github.com/docker/docker/client" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" @@ -36,11 +38,17 @@ const ( // whose compressed layers are not in containerd; `ps` and `check --install` read that list. MinDockerVersionForCtrdInstall = "29.5.2" - installLeasePrefix = "composectl/install/" - installLockFileName = ".install.lock" - contentPluginType = "io.containerd.content.v1" - gcRefConfigLabel = "containerd.io/gc.ref.content.config" - gcRefManifestLabel = "containerd.io/gc.ref.content.m.0" + installLeasePrefix = "composectl/install/" + // installLeaseSyncedLabel marks an install lease whose snapshots were synced to disk before + // they were committed; an interrupted install under such a lease is resumed, not restarted. + installLeaseSyncedLabel = "composectl/install.synced" + // minCtrdVersionForSyncFs is the oldest containerd whose differ honours the sync_fs apply + // option; an older one commits unsynced snapshots, so its install leases get no synced label. + minCtrdVersionForSyncFs = "1.7.14" + installLockFileName = ".install.lock" + contentPluginType = "io.containerd.content.v1" + gcRefConfigLabel = "containerd.io/gc.ref.content.config" + gcRefManifestLabel = "containerd.io/gc.ref.content.m.0" // recordCleanupTimeout bounds the removal of partial image records after a failed write. recordCleanupTimeout = 30 * time.Second // ingestAbortAttempts and ingestAbortDelay bound the retries of a failed ingest abort. @@ -55,6 +63,7 @@ type ( snapshotter string contentRoot string snapshotterRoot string + syncFs bool // the containerd syncs a snapshot before it commits it when asked } // ctrdImage is one app image as the containerd install needs it. @@ -96,7 +105,7 @@ func PruneInstallLeases(ctx context.Context, cfg *Config) error { return err } defer c.Close() - if err := deleteInstallLeases(ctx, c.LeasesService()); err != nil { + if _, err := deleteInstallLeases(ctx, c.LeasesService(), ""); err != nil { return fmt.Errorf("failed to prune install leases: %w", err) } return nil @@ -154,10 +163,17 @@ func newCtrdClient(ctx context.Context, info *DockerInfo) (*ctrdClient, error) { if cc.contentRoot, err = pluginRoot(ctx, c, contentPluginType, "content"); err == nil { cc.snapshotterRoot, err = pluginRoot(ctx, c, snapshotterPluginType, cc.snapshotter) } + var version containerd.Version + if err == nil { + if version, err = c.Version(ctx); err != nil { + err = fmt.Errorf("failed to query the containerd version: %w", err) + } + } if err != nil { _ = c.Close() return nil, err } + cc.syncFs = versionAtLeast(version.Version, minCtrdVersionForSyncFs) return cc, nil } @@ -173,9 +189,12 @@ func pluginRoot(ctx context.Context, c *containerd.Client, pluginType, id string } // installImagesCtrd installs the images of one app into dockerd's containerd. Under a lease it links -// the layers that have no snapshot yet, unpacks every image and syncs the snapshotter's filesystem, -// and only then creates the image records: a record never points at a snapshot that a power cut -// can still lose. Deleting the lease lets containerd's GC drop the compressed layers. +// the layers that have no snapshot yet, unpacks every image with each layer synced before its +// snapshot is committed, syncs the snapshotter's filesystem, and only then creates the image +// records: a record never points at a snapshot that a power cut can still lose. An interrupted +// install of the same app left its lease behind; its committed snapshots are intact, so the +// install drops the unfinished one and goes on under that lease, skipping the committed ones. +// Deleting the lease at the end lets containerd's GC drop the compressed layers. func installImagesCtrd(ctx context.Context, c *ctrdClient, app App, cfg *Config, report ProgressCallbackFn) error { blobsRoot := cfg.GetBlobsRoot() imgs, err := ctrdImagesOf(ctx, app, NewStoreBlobProvider(blobsRoot)) @@ -183,11 +202,16 @@ func installImagesCtrd(ctx context.Context, c *ctrdClient, app App, cfg *Config, return fmt.Errorf("failed to read the app images from the store: %w", err) } ls := c.LeasesService() - if err := deleteInstallLeases(ctx, ls); err != nil { + lease := leases.Lease{ID: installLeaseID(app.Name(), app.Ref().Digest)} + resumed, err := deleteInstallLeases(ctx, ls, lease.ID) + if err != nil { return fmt.Errorf("failed to remove leases of interrupted installs: %w", err) } - lease, err := ls.Create(ctx, leases.WithID(installLeaseID(app.Name(), app.Ref().Digest))) - if err != nil { + if resumed { + if err := c.removeActiveSnapshots(ctx, lease); err != nil { + return fmt.Errorf("failed to drop the unfinished snapshots of the interrupted install: %w", err) + } + } else if _, err := ls.Create(ctx, installLeaseOpts(lease.ID, c.syncFs)...); err != nil { return fmt.Errorf("failed to create the install lease: %w", err) } lctx := leases.WithLease(ctx, lease.ID) @@ -216,29 +240,76 @@ func installLeaseID(appName string, appDigest digest.Digest) string { return installLeasePrefix + appName + "@" + appDigest.String() } -func installLeases(all []leases.Lease) []leases.Lease { - var found []leases.Lease +// installLeaseOpts creates the install lease with the synced label when the containerd syncs +// every snapshot before it commits it; a lease without the label is never resumed. +func installLeaseOpts(id string, syncFs bool) []leases.Opt { + opts := []leases.Opt{leases.WithID(id)} + if syncFs { + opts = append(opts, leases.WithLabels(map[string]string{installLeaseSyncedLabel: "true"})) + } + return opts +} + +// removeActiveSnapshots removes the snapshots the lease holds that were never committed: the +// extraction the interrupted install was in the middle of, whose space the resumed extraction +// needs. The record goes at once and the GC that the removal triggers frees the directory; a +// directory that a power cut never wrote is simply not there to remove. Committed snapshots stay. +func (c *ctrdClient) removeActiveSnapshots(ctx context.Context, lease leases.Lease) error { + rs, err := c.LeasesService().ListResources(ctx, lease) + if err != nil { + return fmt.Errorf("failed to list the resources of lease %s: %w", lease.ID, err) + } + sn := c.SnapshotService(c.snapshotter) + for _, r := range rs { + if r.Type != "snapshots/"+c.snapshotter { + continue + } + info, err := sn.Stat(ctx, r.ID) + if errdefs.IsNotFound(err) || err == nil && info.Kind != snapshots.KindActive { + continue + } + if err != nil { + return fmt.Errorf("failed to stat snapshot %s: %w", r.ID, err) + } + if err := sn.Remove(ctx, r.ID); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to remove snapshot %s: %w", r.ID, err) + } + } + return nil +} + +// staleInstallLeases selects, among all leases, the install leases that an install under keepID +// deletes first, and reports whether the lease keepID itself is kept: it is when it exists and +// its snapshots were synced before they were committed. Every other install lease is stale. +func staleInstallLeases(all []leases.Lease, keepID string) (stale []leases.Lease, kept bool) { for _, l := range all { - if strings.HasPrefix(l.ID, installLeasePrefix) { - found = append(found, l) + if !strings.HasPrefix(l.ID, installLeasePrefix) { + continue } + if l.ID == keepID && l.Labels[installLeaseSyncedLabel] == "true" { + kept = true + continue + } + stale = append(stale, l) } - return found + return stale, kept } -// deleteInstallLeases removes the leases that interrupted installs left behind. Under the install -// lock no other install is running, so every install lease is such a leftover. -func deleteInstallLeases(ctx context.Context, ls leases.Manager) error { +// deleteInstallLeases removes the install leases other than keepID, which stays when its snapshots +// are intact (see staleInstallLeases); keepID is empty to remove them all. Under the install lock +// no other install is running, so every install lease was left by an interrupted install. +func deleteInstallLeases(ctx context.Context, ls leases.Manager, keepID string) (kept bool, err error) { all, err := ls.List(ctx) if err != nil { - return fmt.Errorf("failed to list containerd leases: %w", err) + return false, fmt.Errorf("failed to list containerd leases: %w", err) } - for _, l := range installLeases(all) { + stale, kept := staleInstallLeases(all, keepID) + for _, l := range stale { if err := ls.Delete(ctx, l, leases.SynchronousDelete); err != nil && !errdefs.IsNotFound(err) { - return fmt.Errorf("failed to delete the install lease %s: %w", l.ID, err) + return false, fmt.Errorf("failed to delete the install lease %s: %w", l.ID, err) } } - return nil + return kept, nil } // installImage announces every layer of the image, as existing when its snapshot is there and as @@ -322,9 +393,7 @@ type unpackWatch struct { func (w *unpackWatch) run(ctx context.Context) { defer close(w.done) w.extracting = -1 - // A poll is a stat of the layer being extracted and the free space of the snapshotter root, - // cheap enough to sample a layer that is extracted within a second a few times. - ticker := time.NewTicker(100 * time.Millisecond) + ticker := time.NewTicker(unpackPollInterval) defer ticker.Stop() for { w.scan(ctx, false) @@ -585,9 +654,12 @@ func (c *ctrdClient) writeBlob(ctx context.Context, blobsRoot string, desc ocisp // unpack creates the snapshots of the image's layers. The target is the platform manifest the app // tree selected, not the index: on arm64 the default platform matcher would also accept arm/v7. +// The differ syncs the snapshot's filesystem after applying each layer and before the snapshot +// is committed, so a committed snapshot survives a power cut; a containerd older than +// minCtrdVersionForSyncFs ignores the request and commits unsynced snapshots. func (c *ctrdClient) unpack(ctx context.Context, img *ctrdImage, platform platforms.MatchComparer) error { ci := containerd.NewImageWithPlatform(c.Client, images.Image{Name: img.ref, Target: img.manifest}, platform) - return ci.Unpack(ctx, c.snapshotter) + return ci.Unpack(ctx, c.snapshotter, containerd.WithUnpackApplyOpts(diff.WithSyncFs(true))) } // putImageRecords creates the pinned digest reference and the short tag reference, or points the @@ -761,6 +833,11 @@ func uncompressedSize(blobsRoot string, desc ocispec.Descriptor) int64 { return int64(binary.LittleEndian.Uint32(trailer[:])) } +// unpackPollInterval is how often the unpack watcher polls the snapshots: a stat of the layer +// being extracted and the free space of the snapshotter root, cheap enough to sample a layer +// that is extracted within a second a few times; a variable so that tests can poll faster. +var unpackPollInterval = 100 * time.Millisecond + // freeBytes returns the free space of the filesystem that holds path; a variable so that tests // can drive the measure. var freeBytes = func(path string) (int64, error) { diff --git a/pkg/compose/ctrd_install_rig_test.go b/pkg/compose/ctrd_install_rig_test.go index ec82ab4..2504032 100644 --- a/pkg/compose/ctrd_install_rig_test.go +++ b/pkg/compose/ctrd_install_rig_test.go @@ -21,6 +21,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" "github.com/foundriesio/composeapp/pkg/compose" f "github.com/foundriesio/composeapp/test/fixtures" "github.com/opencontainers/go-digest" @@ -242,9 +243,24 @@ func TestCtrdInstallLinksLayers(t *testing.T) { } } +// installLeasesOf returns the install leases dockerd's containerd holds. +func installLeasesOf(t *testing.T, c *compose.CtrdClient) []leases.Lease { + t.Helper() + all, err := c.LeasesService().List(context.Background()) + f.Check(t, err) + var found []leases.Lease + for _, l := range all { + if strings.HasPrefix(l.ID, "composectl/install/") { + found = append(found, l) + } + } + return found +} + // TestCtrdInstallFailsAcrossFilesystems: with the store on another filesystem the link fails and // so does the install, naming both directories, without a copy of any layer in the content root; -// the next install from a store on the content root's filesystem removes the lease it left. +// the next install from a store on the content root's filesystem goes on under the lease it left +// and removes it at the end. func TestCtrdInstallFailsAcrossFilesystems(t *testing.T) { c := ctrdRig(t) ctx := context.Background() @@ -270,24 +286,147 @@ func TestCtrdInstallFailsAcrossFilesystems(t *testing.T) { t.Errorf("layer %s is in the content root after the failed install: %v", l.Digest, err) } } - installLeases := func() (n int) { - all, err := c.LeasesService().List(ctx) - f.Check(t, err) - for _, l := range all { - if strings.HasPrefix(l.ID, "composectl/install/") { - n++ - } + if left := installLeasesOf(t, c); len(left) != 1 { + t.Fatalf("expected the failed install to leave its lease, found %+v", left) + } + f.Check(t, compose.InstallImagesCtrd(ctx, c, tree, cfg, nil)) + defer deleteImage(t, c, tree) + if left := installLeasesOf(t, c); len(left) != 0 { + t.Fatalf("expected the next install to remove the lease, found %+v", left) + } +} + +// TestCtrdInstallResumesAfterInterruption: an install cancelled in the middle of a layer's +// extraction leaves its lease, marked as synced, holding the committed snapshots, the unfinished +// one and no image record. The next install of the same app goes on under that lease: it drops +// the unfinished snapshot before anything else, loads only the layers without a snapshot, removes +// the lease at the end, and leaves the image as a clean install does. A lease left by another +// app's install is removed first. +func TestCtrdInstallResumesAfterInterruption(t *testing.T) { + c := ctrdRig(t) + ctx := context.Background() + cfg := f.NewTestConfig(t) + store := compose.NewStoreBlobProvider(cfg.GetBlobsRoot()) + // A small layer under one of many files, whose extraction outlasts several polls of the + // install's watcher: a poll finds the small layer's snapshot while the last layer is still + // being extracted. A layer of a few MB is written within one poll on a fast disk. + pollInterval := *compose.UnpackPollInterval + *compose.UnpackPollInterval = 50 * time.Millisecond + t.Cleanup(func() { *compose.UnpackPollInterval = pollInterval }) + image := f.PublishDerivedImageOfFiles(t, f.PublishDerivedImage(t, "ghcr.io/foundriesio/busybox:1.36", 1), 5000) + app := f.NewApp(t, fmt.Sprintf(derivedAppDef, image)) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + tree, err := cfg.AppLoader.LoadAppTree(ctx, store, platforms.OnlyStrict(cfg.Platform), app.PublishedUri) + f.Check(t, err) + layers, err := compose.LayerDescriptorsOf(ctx, tree, store) + f.Check(t, err) + toRegister := layersToRegister(t, c, cfg, tree, layers[0]) + if len(toRegister) == 0 { + t.Fatal("every layer of the fresh image already has a snapshot") + } + chains, err := compose.ChainIDsOf(ctx, tree, store) + f.Check(t, err) + chain := chains[0] + missing := chain[len(chain)-len(toRegister):] // the chain IDs of toRegister + lastLayer, lastChainID := layers[0][len(layers[0])-1], chain[len(chain)-1] + ref := tree.GetComposeRoot().Children[0].Ref() + sn := c.SnapshotService(compose.CtrdSnapshotter(c)) + + cutCtx, cancel := context.WithCancel(ctx) + defer cancel() + // A layer is reported syncing only while the unpack runs and its snapshot is the first + // one missing; with bytes taken from the filesystem, its extraction is in progress. + midExtraction := func(p *compose.LoadImageProgress) { + if p.State == compose.ImageLoadStateLayerSyncing && p.ID == lastLayer.Digest.Encoded()[:12] && p.Current > 0 { + cancel() } - return n } - if installLeases() != 1 { - t.Fatalf("expected the failed install to leave its lease, found %d", installLeases()) + if err := compose.InstallImagesCtrd(cutCtx, c, tree, cfg, midExtraction); err == nil || cutCtx.Err() == nil { + t.Fatalf("expected the install to fail once cancelled in the extraction of the last layer, got %v", err) } - f.Check(t, compose.InstallImagesCtrd(ctx, c, tree, cfg, nil)) + left := installLeasesOf(t, c) + if len(left) != 1 || !strings.HasPrefix(left[0].ID, "composectl/install/"+tree.Name()+"@") || + left[0].Labels[compose.InstallLeaseSyncedLabel] != "true" { + t.Fatalf("expected the lease of the interrupted install, marked as synced, found %+v", left) + } + if _, err := c.ImageService().Get(ctx, ref); !errdefs.IsNotFound(err) { + t.Fatalf("expected no record of the image after the interrupted install, got %v", err) + } + committed := 0 + for _, chainID := range missing { + if info, err := sn.Stat(ctx, chainID); err == nil && info.Kind == snapshots.KindCommitted { + committed++ + } + } + if committed == 0 || committed == len(missing) { + t.Fatalf("the interrupted install committed %d of %d snapshots; expected some but not all", committed, len(missing)) + } + t.Logf("the interrupted install committed %d of %d snapshots", committed, len(missing)) + unfinished := activeExtractSnapshot(t, sn, lastChainID) + if unfinished == "" { + t.Fatalf("expected the unfinished extraction of %s to leave an active snapshot", lastChainID) + } + other, err := c.LeasesService().Create(ctx, leases.WithID("composectl/install/other@"+digest.FromString("other").String()), + leases.WithLabels(map[string]string{compose.InstallLeaseSyncedLabel: "true"})) + f.Check(t, err) + defer func() { _ = c.LeasesService().Delete(ctx, other) }() + + var events []compose.LoadImageProgress + dropped := false + atEvent := func(p *compose.LoadImageProgress) { + events = append(events, *p) + if len(events) > 1 || p.State != compose.ImageLoadStateImageLoading { + return + } + // The first event comes before any layer is linked or unpacked. + if _, err := sn.Stat(ctx, unfinished); !errdefs.IsNotFound(err) { + t.Errorf("the unfinished snapshot %s is still there when the resumed install starts: %v", unfinished, err) + } + dropped = true + } + f.Check(t, compose.InstallImagesCtrd(ctx, c, tree, cfg, atEvent)) defer deleteImage(t, c, tree) - if installLeases() != 0 { - t.Fatalf("expected the next install to remove the lease, found %d", installLeases()) + if !dropped { + t.Error("the resumed install did not report the image loading first") + } + loaded := 0 + for _, e := range events { + if e.State == compose.ImageLoadStateLayerLoaded { + loaded++ + } + } + if loaded != len(missing)-committed { + t.Errorf("expected %d layers loaded by the resumed install, %d skipped, got %d loaded", len(missing)-committed, committed, loaded) + } + if left := installLeasesOf(t, c); len(left) != 0 { + t.Errorf("expected the resumed install to remove every install lease, found %+v", left) } + for _, chainID := range chain { + if info, err := sn.Stat(ctx, chainID); err != nil || info.Kind != snapshots.KindCommitted { + t.Errorf("snapshot %s after the resumed install: %v, %v", chainID, info.Kind, err) + } + } + if leftover := activeExtractSnapshot(t, sn, lastChainID); leftover != "" { + t.Errorf("leftover snapshot %s after the resumed install", leftover) + } + out, err := exec.Command("docker", "run", "--rm", "--pull", "never", "--network", "none", ref, "echo", "ok").CombinedOutput() + f.Checkf(t, err, "docker run %s: %s", ref, out) +} + +// activeExtractSnapshot returns the key of the active "extract- " snapshot of an +// unpack that did not commit the layer with chainID, or "" when there is none. +func activeExtractSnapshot(t *testing.T, sn snapshots.Snapshotter, chainID string) string { + t.Helper() + var key string + f.Check(t, sn.Walk(context.Background(), func(_ context.Context, info snapshots.Info) error { + if info.Kind == snapshots.KindActive && strings.HasSuffix(info.Name, " "+chainID) { + key = info.Name + } + return nil + })) + return key } // TestCtrdLinkedBlobAndGC: containerd's blob-directory cleanup runs after a GC that removed content. diff --git a/pkg/compose/ctrd_install_test.go b/pkg/compose/ctrd_install_test.go index f6a1a02..7c28a91 100644 --- a/pkg/compose/ctrd_install_test.go +++ b/pkg/compose/ctrd_install_test.go @@ -28,19 +28,48 @@ func TestInstallLeaseID(t *testing.T) { } } -func TestInstallLeasesFilter(t *testing.T) { +func TestInstallLeaseOpts(t *testing.T) { + for _, syncFs := range []bool{true, false} { + var l leases.Lease + for _, opt := range installLeaseOpts("composectl/install/app@sha256:aaaa", syncFs) { + if err := opt(&l); err != nil { + t.Fatal(err) + } + } + if l.ID != "composectl/install/app@sha256:aaaa" || (l.Labels[installLeaseSyncedLabel] == "true") != syncFs { + t.Fatalf("sync_fs %v: unexpected lease %+v", syncFs, l) + } + } +} + +func TestStaleInstallLeases(t *testing.T) { + synced := map[string]string{installLeaseSyncedLabel: "true"} + mine := "composectl/install/app1@sha256:aaaa" all := []leases.Lease{ - {ID: "composectl/install/app1@sha256:aaaa"}, + {ID: mine, Labels: synced}, {ID: "moby-pull-123"}, - {ID: "composectl/install/app2@sha256:bbbb"}, + {ID: "composectl/install/app2@sha256:bbbb", Labels: synced}, {ID: "composectl/other"}, - } - got := installLeases(all) - if len(got) != 2 || got[0].ID != all[0].ID || got[1].ID != all[2].ID { - t.Fatalf("unexpected install leases: %+v", got) - } - if got := installLeases(nil); len(got) != 0 { - t.Fatalf("expected no leases, got %+v", got) + {ID: "composectl/install/app3@sha256:cccc"}, + } + stale, kept := staleInstallLeases(all, mine) + if !kept || len(stale) != 2 || stale[0].ID != all[2].ID || stale[1].ID != all[4].ID { + t.Fatalf("expected the synced lease of the app kept and the other install leases stale, got kept %v, stale %+v", kept, stale) + } + // The same app's lease without the label: its snapshots may be torn, so it is stale too. + all[0].Labels = nil + stale, kept = staleInstallLeases(all, mine) + if kept || len(stale) != 3 || stale[0].ID != mine { + t.Fatalf("expected the unlabelled lease of the app stale, got kept %v, stale %+v", kept, stale) + } + // No lease to keep: every install lease is stale. + all[0].Labels = synced + stale, kept = staleInstallLeases(all, "") + if kept || len(stale) != 3 { + t.Fatalf("expected every install lease stale, got kept %v, stale %+v", kept, stale) + } + if stale, kept := staleInstallLeases(nil, mine); kept || len(stale) != 0 { + t.Fatalf("expected nothing from no leases, got kept %v, stale %+v", kept, stale) } } diff --git a/pkg/compose/docker_client_test.go b/pkg/compose/docker_client_test.go index fe945f4..99f5a8f 100644 --- a/pkg/compose/docker_client_test.go +++ b/pkg/compose/docker_client_test.go @@ -95,6 +95,12 @@ func TestVersionAtLeast(t *testing.T) { {"29.5", "29.5.2", false}, {"29.6", "29.5.2", true}, {"", "29.5.2", false}, + // containerd versions against the sync_fs floor + {"v2.4.1", "1.7.14", true}, + {"2.2.3+unknown", "1.7.14", true}, + {"v1.7.14", "1.7.14", true}, + {"v1.7.13", "1.7.14", false}, + {"dev", "1.7.14", false}, {"garbage", "29.5.2", false}, } { if got := versionAtLeast(tc.version, tc.min); got != tc.want { diff --git a/pkg/compose/export_test.go b/pkg/compose/export_test.go index 107cb17..81e00ba 100644 --- a/pkg/compose/export_test.go +++ b/pkg/compose/export_test.go @@ -10,6 +10,10 @@ import ( type CtrdClient = ctrdClient +const InstallLeaseSyncedLabel = installLeaseSyncedLabel + +var UnpackPollInterval = &unpackPollInterval + var ( ConnectCtrdIfInstallTarget = connectCtrdIfInstallTarget InstallImagesCtrd = installImagesCtrd diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 011b69f..cda6f45 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -496,6 +496,18 @@ func randomString(length int) string { // pushes it to the test registry through the publisher dockerd, which the publish step reads the // image layers from. It returns the image reference. func PublishDerivedImage(t *testing.T, base string, sizeMB int) string { + t.Helper() + return publishDerivedImage(t, base, fmt.Sprintf("head -c %d /dev/urandom > /derived", sizeMB<<20)) +} + +// PublishDerivedImageOfFiles is PublishDerivedImage with a layer of the given number of small +// files: its extraction takes a syscall per file, not a disk write, so it stays slow on a fast disk. +func PublishDerivedImageOfFiles(t *testing.T, base string, files int) string { + t.Helper() + return publishDerivedImage(t, base, fmt.Sprintf("mkdir /derived-files && for i in $(seq %d); do echo $i > /derived-files/$i; done", files)) +} + +func publishDerivedImage(t *testing.T, base, command string) string { t.Helper() name := "derived-" + randomString(8) ref := "registry:5000/factory/" + name + ":v1" @@ -504,7 +516,7 @@ func PublishDerivedImage(t *testing.T, base string, sizeMB int) string { Checkf(t, err, "docker %v: %s", args, out) } pub("pull", base) - pub("run", "--name", name, base, "sh", "-c", fmt.Sprintf("head -c %d /dev/urandom > /derived", sizeMB<<20)) + pub("run", "--name", name, base, "sh", "-c", command) pub("commit", name, ref) pub("rm", name) pub("push", ref) diff --git a/test/integration/ctrd_install_test.go b/test/integration/ctrd_install_test.go index 0b82249..c95d36f 100644 --- a/test/integration/ctrd_install_test.go +++ b/test/integration/ctrd_install_test.go @@ -330,8 +330,8 @@ func TestCtrdInstallAfterPrune(t *testing.T) { } // TestCtrdInstallRecoversFromKilledInstall: an install is killed after it created snapshots; its -// lease stays, the next install removes the lease and with it the snapshots it alone held, and -// recreates them, so that nothing a power cut may have broken is reused. +// lease stays, marked as synced, and the next install goes on under it: the snapshots the killed +// install committed are kept, the rest are created, and the lease is gone at the end. func TestCtrdInstallRecoversFromKilledInstall(t *testing.T) { r := newCtrdRig(t) const base = "ghcr.io/foundriesio/busybox:1.36" @@ -342,18 +342,31 @@ func TestCtrdInstallRecoversFromKilledInstall(t *testing.T) { defer app.Remove(t) r.pruneImages(r.topChainIDs(app)) killedAt := r.killInstall(app) - if leftovers := r.installLeases(); len(leftovers) != 1 { - t.Fatalf("expected the lease of the killed install to stay, got %+v", leftovers) + if leftovers := r.installLeases(); len(leftovers) != 1 || leftovers[0].Labels["composectl/install.synced"] != "true" { + t.Fatalf("expected the lease of the killed install to stay, marked as synced, got %+v", leftovers) + } + committed := map[digest.Digest]time.Time{} + for _, img := range r.images(app.PublishedUri) { + for _, chainID := range img.chainIDs { + if info, ok := r.snapshot(chainID); ok { + committed[chainID] = info.Created + } + } } app.Install(t) defer app.Uninstall(t) r.checkInstalled(app) + if leftovers := r.installLeases(); len(leftovers) != 0 { + t.Errorf("expected no install lease after the resumed install, got %+v", leftovers) + } for _, img := range r.images(app.PublishedUri) { for _, chainID := range img.chainIDs { info, ok := r.snapshot(chainID) - if !ok || !info.Created.After(killedAt) { - t.Errorf("snapshot %s was not recreated after the killed install: created %v, killed at %v", chainID, info.Created, killedAt) + if createdAt, kept := committed[chainID]; kept && (!ok || !info.Created.Equal(createdAt)) { + t.Errorf("snapshot %s of the killed install was not kept: created %v, was %v", chainID, info.Created, createdAt) + } else if !kept && (!ok || !info.Created.After(killedAt)) { + t.Errorf("snapshot %s was not created after the killed install: created %v, killed at %v", chainID, info.Created, killedAt) } } } From d9900c9b20ce44955e551970f0e478feb86d98b6 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Wed, 21 Oct 2026 09:20:57 +0200 Subject: [PATCH 2/2] docs(design): record the durable unpack and install resume The note said that every leftover install lease is deleted and no snapshot of an interrupted install is reused, which is necessary only while overlayfs snapshot commits are not durable. Record the sync_fs apply option with its containerd floor and the resume of the same app's labelled lease. Assisted-by: Claude Code:claude-fable-5-1 Signed-off-by: Mike Sul --- docs/containerd-install-plan-2026-10-08.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/containerd-install-plan-2026-10-08.md b/docs/containerd-install-plan-2026-10-08.md index a25e875..ab920a4 100644 --- a/docs/containerd-install-plan-2026-10-08.md +++ b/docs/containerd-install-plan-2026-10-08.md @@ -20,14 +20,17 @@ containerd cannot be reached on such a daemon, install fails instead of falling Under an exclusive flock on `/.install.lock`, which uninstall, `rm` and `prune` take too, for each app: -1. Delete every leftover `composectl/install/` lease, then create the lease - `composectl/install/@` with no expiry (a board without a real-time - clock can jump forward in time). +1. Delete every leftover `composectl/install/` lease except this app's own one + labelled `composectl/install.synced`, which is resumed: its active snapshot is + removed. Otherwise create the lease `composectl/install/@` with no + expiry (a board without a real-time clock can jump forward in time), labelled + when containerd is 1.7.14 or newer. 2. For each image, write under the lease the layers from the first one without a snapshot on, the config, the manifest and the index. The manifest has no layer GC labels, so only the lease holds the layers. -3. Unpack the image. If a base snapshot was removed in the meantime, write every - layer and unpack once more. +3. Unpack the image with the `sync_fs` apply option, so that each snapshot is + synced before it is committed (containerd 1.7.14 or newer). If a base snapshot + was removed in the meantime, write every layer and unpack once more. 4. `syncfs` the snapshotter root. 5. Create or update the image records: the digest ref and the tag ref. 6. Delete the lease; containerd's GC drops the compressed layers. @@ -36,8 +39,10 @@ Under an exclusive flock on `/.install.lock`, which uninstall, `rm` a The image records are created only after the `syncfs`, so a recorded image is on disk. Without it, a crash right after install left 3 of 3 images broken on the board. -The snapshots of an interrupted install are held only by its lease, which the next -install, or `composectl prune`, deletes before anything is reused. +The snapshots of an interrupted install are held only by its lease. A labelled lease +holds synced snapshots only, so the next install of the same app keeps them and goes +on from the first missing layer. Any other lease is deleted by the next install, or +by `composectl prune`, before anything is reused. ## Limits