diff --git a/docs/containerd-install-plan-2026-10-08.md b/docs/containerd-install-plan-2026-10-08.md index a25e875..ab920a4 100644 --- a/docs/containerd-install-plan-2026-10-08.md +++ b/docs/containerd-install-plan-2026-10-08.md @@ -20,14 +20,17 @@ containerd cannot be reached on such a daemon, install fails instead of falling Under an exclusive flock on `/.install.lock`, which uninstall, `rm` and `prune` take too, for each app: -1. Delete every leftover `composectl/install/` lease, then create the lease - `composectl/install/@` with no expiry (a board without a real-time - clock can jump forward in time). +1. Delete every leftover `composectl/install/` lease except this app's own one + labelled `composectl/install.synced`, which is resumed: its active snapshot is + removed. Otherwise create the lease `composectl/install/@` with no + expiry (a board without a real-time clock can jump forward in time), labelled + when containerd is 1.7.14 or newer. 2. For each image, write under the lease the layers from the first one without a snapshot on, the config, the manifest and the index. The manifest has no layer GC labels, so only the lease holds the layers. -3. Unpack the image. If a base snapshot was removed in the meantime, write every - layer and unpack once more. +3. Unpack the image with the `sync_fs` apply option, so that each snapshot is + synced before it is committed (containerd 1.7.14 or newer). If a base snapshot + was removed in the meantime, write every layer and unpack once more. 4. `syncfs` the snapshotter root. 5. Create or update the image records: the digest ref and the tag ref. 6. Delete the lease; containerd's GC drops the compressed layers. @@ -36,8 +39,10 @@ Under an exclusive flock on `/.install.lock`, which uninstall, `rm` a The image records are created only after the `syncfs`, so a recorded image is on disk. Without it, a crash right after install left 3 of 3 images broken on the board. -The snapshots of an interrupted install are held only by its lease, which the next -install, or `composectl prune`, deletes before anything is reused. +The snapshots of an interrupted install are held only by its lease. A labelled lease +holds synced snapshots only, so the next install of the same app keeps them and goes +on from the first missing layer. Any other lease is deleted by the next install, or +by `composectl prune`, before anything is reused. ## Limits diff --git a/pkg/compose/ctrd_install.go b/pkg/compose/ctrd_install.go index fd93ae6..99c82ce 100644 --- a/pkg/compose/ctrd_install.go +++ b/pkg/compose/ctrd_install.go @@ -16,10 +16,12 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" dockerclient "github.com/docker/docker/client" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" @@ -36,11 +38,17 @@ const ( // whose compressed layers are not in containerd; `ps` and `check --install` read that list. MinDockerVersionForCtrdInstall = "29.5.2" - installLeasePrefix = "composectl/install/" - installLockFileName = ".install.lock" - contentPluginType = "io.containerd.content.v1" - gcRefConfigLabel = "containerd.io/gc.ref.content.config" - gcRefManifestLabel = "containerd.io/gc.ref.content.m.0" + installLeasePrefix = "composectl/install/" + // installLeaseSyncedLabel marks an install lease whose snapshots were synced to disk before + // they were committed; an interrupted install under such a lease is resumed, not restarted. + installLeaseSyncedLabel = "composectl/install.synced" + // minCtrdVersionForSyncFs is the oldest containerd whose differ honours the sync_fs apply + // option; an older one commits unsynced snapshots, so its install leases get no synced label. + minCtrdVersionForSyncFs = "1.7.14" + installLockFileName = ".install.lock" + contentPluginType = "io.containerd.content.v1" + gcRefConfigLabel = "containerd.io/gc.ref.content.config" + gcRefManifestLabel = "containerd.io/gc.ref.content.m.0" // recordCleanupTimeout bounds the removal of partial image records after a failed write. recordCleanupTimeout = 30 * time.Second // ingestAbortAttempts and ingestAbortDelay bound the retries of a failed ingest abort. @@ -55,6 +63,7 @@ type ( snapshotter string contentRoot string snapshotterRoot string + syncFs bool // the containerd syncs a snapshot before it commits it when asked } // ctrdImage is one app image as the containerd install needs it. @@ -96,7 +105,7 @@ func PruneInstallLeases(ctx context.Context, cfg *Config) error { return err } defer c.Close() - if err := deleteInstallLeases(ctx, c.LeasesService()); err != nil { + if _, err := deleteInstallLeases(ctx, c.LeasesService(), ""); err != nil { return fmt.Errorf("failed to prune install leases: %w", err) } return nil @@ -154,10 +163,17 @@ func newCtrdClient(ctx context.Context, info *DockerInfo) (*ctrdClient, error) { if cc.contentRoot, err = pluginRoot(ctx, c, contentPluginType, "content"); err == nil { cc.snapshotterRoot, err = pluginRoot(ctx, c, snapshotterPluginType, cc.snapshotter) } + var version containerd.Version + if err == nil { + if version, err = c.Version(ctx); err != nil { + err = fmt.Errorf("failed to query the containerd version: %w", err) + } + } if err != nil { _ = c.Close() return nil, err } + cc.syncFs = versionAtLeast(version.Version, minCtrdVersionForSyncFs) return cc, nil } @@ -173,9 +189,12 @@ func pluginRoot(ctx context.Context, c *containerd.Client, pluginType, id string } // installImagesCtrd installs the images of one app into dockerd's containerd. Under a lease it links -// the layers that have no snapshot yet, unpacks every image and syncs the snapshotter's filesystem, -// and only then creates the image records: a record never points at a snapshot that a power cut -// can still lose. Deleting the lease lets containerd's GC drop the compressed layers. +// the layers that have no snapshot yet, unpacks every image with each layer synced before its +// snapshot is committed, syncs the snapshotter's filesystem, and only then creates the image +// records: a record never points at a snapshot that a power cut can still lose. An interrupted +// install of the same app left its lease behind; its committed snapshots are intact, so the +// install drops the unfinished one and goes on under that lease, skipping the committed ones. +// Deleting the lease at the end lets containerd's GC drop the compressed layers. func installImagesCtrd(ctx context.Context, c *ctrdClient, app App, cfg *Config, report ProgressCallbackFn) error { blobsRoot := cfg.GetBlobsRoot() imgs, err := ctrdImagesOf(ctx, app, NewStoreBlobProvider(blobsRoot)) @@ -183,11 +202,16 @@ func installImagesCtrd(ctx context.Context, c *ctrdClient, app App, cfg *Config, return fmt.Errorf("failed to read the app images from the store: %w", err) } ls := c.LeasesService() - if err := deleteInstallLeases(ctx, ls); err != nil { + lease := leases.Lease{ID: installLeaseID(app.Name(), app.Ref().Digest)} + resumed, err := deleteInstallLeases(ctx, ls, lease.ID) + if err != nil { return fmt.Errorf("failed to remove leases of interrupted installs: %w", err) } - lease, err := ls.Create(ctx, leases.WithID(installLeaseID(app.Name(), app.Ref().Digest))) - if err != nil { + if resumed { + if err := c.removeActiveSnapshots(ctx, lease); err != nil { + return fmt.Errorf("failed to drop the unfinished snapshots of the interrupted install: %w", err) + } + } else if _, err := ls.Create(ctx, installLeaseOpts(lease.ID, c.syncFs)...); err != nil { return fmt.Errorf("failed to create the install lease: %w", err) } lctx := leases.WithLease(ctx, lease.ID) @@ -216,29 +240,76 @@ func installLeaseID(appName string, appDigest digest.Digest) string { return installLeasePrefix + appName + "@" + appDigest.String() } -func installLeases(all []leases.Lease) []leases.Lease { - var found []leases.Lease +// installLeaseOpts creates the install lease with the synced label when the containerd syncs +// every snapshot before it commits it; a lease without the label is never resumed. +func installLeaseOpts(id string, syncFs bool) []leases.Opt { + opts := []leases.Opt{leases.WithID(id)} + if syncFs { + opts = append(opts, leases.WithLabels(map[string]string{installLeaseSyncedLabel: "true"})) + } + return opts +} + +// removeActiveSnapshots removes the snapshots the lease holds that were never committed: the +// extraction the interrupted install was in the middle of, whose space the resumed extraction +// needs. The record goes at once and the GC that the removal triggers frees the directory; a +// directory that a power cut never wrote is simply not there to remove. Committed snapshots stay. +func (c *ctrdClient) removeActiveSnapshots(ctx context.Context, lease leases.Lease) error { + rs, err := c.LeasesService().ListResources(ctx, lease) + if err != nil { + return fmt.Errorf("failed to list the resources of lease %s: %w", lease.ID, err) + } + sn := c.SnapshotService(c.snapshotter) + for _, r := range rs { + if r.Type != "snapshots/"+c.snapshotter { + continue + } + info, err := sn.Stat(ctx, r.ID) + if errdefs.IsNotFound(err) || err == nil && info.Kind != snapshots.KindActive { + continue + } + if err != nil { + return fmt.Errorf("failed to stat snapshot %s: %w", r.ID, err) + } + if err := sn.Remove(ctx, r.ID); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("failed to remove snapshot %s: %w", r.ID, err) + } + } + return nil +} + +// staleInstallLeases selects, among all leases, the install leases that an install under keepID +// deletes first, and reports whether the lease keepID itself is kept: it is when it exists and +// its snapshots were synced before they were committed. Every other install lease is stale. +func staleInstallLeases(all []leases.Lease, keepID string) (stale []leases.Lease, kept bool) { for _, l := range all { - if strings.HasPrefix(l.ID, installLeasePrefix) { - found = append(found, l) + if !strings.HasPrefix(l.ID, installLeasePrefix) { + continue } + if l.ID == keepID && l.Labels[installLeaseSyncedLabel] == "true" { + kept = true + continue + } + stale = append(stale, l) } - return found + return stale, kept } -// deleteInstallLeases removes the leases that interrupted installs left behind. Under the install -// lock no other install is running, so every install lease is such a leftover. -func deleteInstallLeases(ctx context.Context, ls leases.Manager) error { +// deleteInstallLeases removes the install leases other than keepID, which stays when its snapshots +// are intact (see staleInstallLeases); keepID is empty to remove them all. Under the install lock +// no other install is running, so every install lease was left by an interrupted install. +func deleteInstallLeases(ctx context.Context, ls leases.Manager, keepID string) (kept bool, err error) { all, err := ls.List(ctx) if err != nil { - return fmt.Errorf("failed to list containerd leases: %w", err) + return false, fmt.Errorf("failed to list containerd leases: %w", err) } - for _, l := range installLeases(all) { + stale, kept := staleInstallLeases(all, keepID) + for _, l := range stale { if err := ls.Delete(ctx, l, leases.SynchronousDelete); err != nil && !errdefs.IsNotFound(err) { - return fmt.Errorf("failed to delete the install lease %s: %w", l.ID, err) + return false, fmt.Errorf("failed to delete the install lease %s: %w", l.ID, err) } } - return nil + return kept, nil } // installImage announces every layer of the image, as existing when its snapshot is there and as @@ -322,9 +393,7 @@ type unpackWatch struct { func (w *unpackWatch) run(ctx context.Context) { defer close(w.done) w.extracting = -1 - // A poll is a stat of the layer being extracted and the free space of the snapshotter root, - // cheap enough to sample a layer that is extracted within a second a few times. - ticker := time.NewTicker(100 * time.Millisecond) + ticker := time.NewTicker(unpackPollInterval) defer ticker.Stop() for { w.scan(ctx, false) @@ -585,9 +654,12 @@ func (c *ctrdClient) writeBlob(ctx context.Context, blobsRoot string, desc ocisp // unpack creates the snapshots of the image's layers. The target is the platform manifest the app // tree selected, not the index: on arm64 the default platform matcher would also accept arm/v7. +// The differ syncs the snapshot's filesystem after applying each layer and before the snapshot +// is committed, so a committed snapshot survives a power cut; a containerd older than +// minCtrdVersionForSyncFs ignores the request and commits unsynced snapshots. func (c *ctrdClient) unpack(ctx context.Context, img *ctrdImage, platform platforms.MatchComparer) error { ci := containerd.NewImageWithPlatform(c.Client, images.Image{Name: img.ref, Target: img.manifest}, platform) - return ci.Unpack(ctx, c.snapshotter) + return ci.Unpack(ctx, c.snapshotter, containerd.WithUnpackApplyOpts(diff.WithSyncFs(true))) } // putImageRecords creates the pinned digest reference and the short tag reference, or points the @@ -761,6 +833,11 @@ func uncompressedSize(blobsRoot string, desc ocispec.Descriptor) int64 { return int64(binary.LittleEndian.Uint32(trailer[:])) } +// unpackPollInterval is how often the unpack watcher polls the snapshots: a stat of the layer +// being extracted and the free space of the snapshotter root, cheap enough to sample a layer +// that is extracted within a second a few times; a variable so that tests can poll faster. +var unpackPollInterval = 100 * time.Millisecond + // freeBytes returns the free space of the filesystem that holds path; a variable so that tests // can drive the measure. var freeBytes = func(path string) (int64, error) { diff --git a/pkg/compose/ctrd_install_rig_test.go b/pkg/compose/ctrd_install_rig_test.go index ec82ab4..2504032 100644 --- a/pkg/compose/ctrd_install_rig_test.go +++ b/pkg/compose/ctrd_install_rig_test.go @@ -21,6 +21,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/snapshots" "github.com/foundriesio/composeapp/pkg/compose" f "github.com/foundriesio/composeapp/test/fixtures" "github.com/opencontainers/go-digest" @@ -242,9 +243,24 @@ func TestCtrdInstallLinksLayers(t *testing.T) { } } +// installLeasesOf returns the install leases dockerd's containerd holds. +func installLeasesOf(t *testing.T, c *compose.CtrdClient) []leases.Lease { + t.Helper() + all, err := c.LeasesService().List(context.Background()) + f.Check(t, err) + var found []leases.Lease + for _, l := range all { + if strings.HasPrefix(l.ID, "composectl/install/") { + found = append(found, l) + } + } + return found +} + // TestCtrdInstallFailsAcrossFilesystems: with the store on another filesystem the link fails and // so does the install, naming both directories, without a copy of any layer in the content root; -// the next install from a store on the content root's filesystem removes the lease it left. +// the next install from a store on the content root's filesystem goes on under the lease it left +// and removes it at the end. func TestCtrdInstallFailsAcrossFilesystems(t *testing.T) { c := ctrdRig(t) ctx := context.Background() @@ -270,24 +286,147 @@ func TestCtrdInstallFailsAcrossFilesystems(t *testing.T) { t.Errorf("layer %s is in the content root after the failed install: %v", l.Digest, err) } } - installLeases := func() (n int) { - all, err := c.LeasesService().List(ctx) - f.Check(t, err) - for _, l := range all { - if strings.HasPrefix(l.ID, "composectl/install/") { - n++ - } + if left := installLeasesOf(t, c); len(left) != 1 { + t.Fatalf("expected the failed install to leave its lease, found %+v", left) + } + f.Check(t, compose.InstallImagesCtrd(ctx, c, tree, cfg, nil)) + defer deleteImage(t, c, tree) + if left := installLeasesOf(t, c); len(left) != 0 { + t.Fatalf("expected the next install to remove the lease, found %+v", left) + } +} + +// TestCtrdInstallResumesAfterInterruption: an install cancelled in the middle of a layer's +// extraction leaves its lease, marked as synced, holding the committed snapshots, the unfinished +// one and no image record. The next install of the same app goes on under that lease: it drops +// the unfinished snapshot before anything else, loads only the layers without a snapshot, removes +// the lease at the end, and leaves the image as a clean install does. A lease left by another +// app's install is removed first. +func TestCtrdInstallResumesAfterInterruption(t *testing.T) { + c := ctrdRig(t) + ctx := context.Background() + cfg := f.NewTestConfig(t) + store := compose.NewStoreBlobProvider(cfg.GetBlobsRoot()) + // A small layer under one of many files, whose extraction outlasts several polls of the + // install's watcher: a poll finds the small layer's snapshot while the last layer is still + // being extracted. A layer of a few MB is written within one poll on a fast disk. + pollInterval := *compose.UnpackPollInterval + *compose.UnpackPollInterval = 50 * time.Millisecond + t.Cleanup(func() { *compose.UnpackPollInterval = pollInterval }) + image := f.PublishDerivedImageOfFiles(t, f.PublishDerivedImage(t, "ghcr.io/foundriesio/busybox:1.36", 1), 5000) + app := f.NewApp(t, fmt.Sprintf(derivedAppDef, image)) + app.Publish(t) + app.Pull(t) + defer app.Remove(t) + tree, err := cfg.AppLoader.LoadAppTree(ctx, store, platforms.OnlyStrict(cfg.Platform), app.PublishedUri) + f.Check(t, err) + layers, err := compose.LayerDescriptorsOf(ctx, tree, store) + f.Check(t, err) + toRegister := layersToRegister(t, c, cfg, tree, layers[0]) + if len(toRegister) == 0 { + t.Fatal("every layer of the fresh image already has a snapshot") + } + chains, err := compose.ChainIDsOf(ctx, tree, store) + f.Check(t, err) + chain := chains[0] + missing := chain[len(chain)-len(toRegister):] // the chain IDs of toRegister + lastLayer, lastChainID := layers[0][len(layers[0])-1], chain[len(chain)-1] + ref := tree.GetComposeRoot().Children[0].Ref() + sn := c.SnapshotService(compose.CtrdSnapshotter(c)) + + cutCtx, cancel := context.WithCancel(ctx) + defer cancel() + // A layer is reported syncing only while the unpack runs and its snapshot is the first + // one missing; with bytes taken from the filesystem, its extraction is in progress. + midExtraction := func(p *compose.LoadImageProgress) { + if p.State == compose.ImageLoadStateLayerSyncing && p.ID == lastLayer.Digest.Encoded()[:12] && p.Current > 0 { + cancel() } - return n } - if installLeases() != 1 { - t.Fatalf("expected the failed install to leave its lease, found %d", installLeases()) + if err := compose.InstallImagesCtrd(cutCtx, c, tree, cfg, midExtraction); err == nil || cutCtx.Err() == nil { + t.Fatalf("expected the install to fail once cancelled in the extraction of the last layer, got %v", err) } - f.Check(t, compose.InstallImagesCtrd(ctx, c, tree, cfg, nil)) + left := installLeasesOf(t, c) + if len(left) != 1 || !strings.HasPrefix(left[0].ID, "composectl/install/"+tree.Name()+"@") || + left[0].Labels[compose.InstallLeaseSyncedLabel] != "true" { + t.Fatalf("expected the lease of the interrupted install, marked as synced, found %+v", left) + } + if _, err := c.ImageService().Get(ctx, ref); !errdefs.IsNotFound(err) { + t.Fatalf("expected no record of the image after the interrupted install, got %v", err) + } + committed := 0 + for _, chainID := range missing { + if info, err := sn.Stat(ctx, chainID); err == nil && info.Kind == snapshots.KindCommitted { + committed++ + } + } + if committed == 0 || committed == len(missing) { + t.Fatalf("the interrupted install committed %d of %d snapshots; expected some but not all", committed, len(missing)) + } + t.Logf("the interrupted install committed %d of %d snapshots", committed, len(missing)) + unfinished := activeExtractSnapshot(t, sn, lastChainID) + if unfinished == "" { + t.Fatalf("expected the unfinished extraction of %s to leave an active snapshot", lastChainID) + } + other, err := c.LeasesService().Create(ctx, leases.WithID("composectl/install/other@"+digest.FromString("other").String()), + leases.WithLabels(map[string]string{compose.InstallLeaseSyncedLabel: "true"})) + f.Check(t, err) + defer func() { _ = c.LeasesService().Delete(ctx, other) }() + + var events []compose.LoadImageProgress + dropped := false + atEvent := func(p *compose.LoadImageProgress) { + events = append(events, *p) + if len(events) > 1 || p.State != compose.ImageLoadStateImageLoading { + return + } + // The first event comes before any layer is linked or unpacked. + if _, err := sn.Stat(ctx, unfinished); !errdefs.IsNotFound(err) { + t.Errorf("the unfinished snapshot %s is still there when the resumed install starts: %v", unfinished, err) + } + dropped = true + } + f.Check(t, compose.InstallImagesCtrd(ctx, c, tree, cfg, atEvent)) defer deleteImage(t, c, tree) - if installLeases() != 0 { - t.Fatalf("expected the next install to remove the lease, found %d", installLeases()) + if !dropped { + t.Error("the resumed install did not report the image loading first") + } + loaded := 0 + for _, e := range events { + if e.State == compose.ImageLoadStateLayerLoaded { + loaded++ + } + } + if loaded != len(missing)-committed { + t.Errorf("expected %d layers loaded by the resumed install, %d skipped, got %d loaded", len(missing)-committed, committed, loaded) + } + if left := installLeasesOf(t, c); len(left) != 0 { + t.Errorf("expected the resumed install to remove every install lease, found %+v", left) } + for _, chainID := range chain { + if info, err := sn.Stat(ctx, chainID); err != nil || info.Kind != snapshots.KindCommitted { + t.Errorf("snapshot %s after the resumed install: %v, %v", chainID, info.Kind, err) + } + } + if leftover := activeExtractSnapshot(t, sn, lastChainID); leftover != "" { + t.Errorf("leftover snapshot %s after the resumed install", leftover) + } + out, err := exec.Command("docker", "run", "--rm", "--pull", "never", "--network", "none", ref, "echo", "ok").CombinedOutput() + f.Checkf(t, err, "docker run %s: %s", ref, out) +} + +// activeExtractSnapshot returns the key of the active "extract- " snapshot of an +// unpack that did not commit the layer with chainID, or "" when there is none. +func activeExtractSnapshot(t *testing.T, sn snapshots.Snapshotter, chainID string) string { + t.Helper() + var key string + f.Check(t, sn.Walk(context.Background(), func(_ context.Context, info snapshots.Info) error { + if info.Kind == snapshots.KindActive && strings.HasSuffix(info.Name, " "+chainID) { + key = info.Name + } + return nil + })) + return key } // TestCtrdLinkedBlobAndGC: containerd's blob-directory cleanup runs after a GC that removed content. diff --git a/pkg/compose/ctrd_install_test.go b/pkg/compose/ctrd_install_test.go index f6a1a02..7c28a91 100644 --- a/pkg/compose/ctrd_install_test.go +++ b/pkg/compose/ctrd_install_test.go @@ -28,19 +28,48 @@ func TestInstallLeaseID(t *testing.T) { } } -func TestInstallLeasesFilter(t *testing.T) { +func TestInstallLeaseOpts(t *testing.T) { + for _, syncFs := range []bool{true, false} { + var l leases.Lease + for _, opt := range installLeaseOpts("composectl/install/app@sha256:aaaa", syncFs) { + if err := opt(&l); err != nil { + t.Fatal(err) + } + } + if l.ID != "composectl/install/app@sha256:aaaa" || (l.Labels[installLeaseSyncedLabel] == "true") != syncFs { + t.Fatalf("sync_fs %v: unexpected lease %+v", syncFs, l) + } + } +} + +func TestStaleInstallLeases(t *testing.T) { + synced := map[string]string{installLeaseSyncedLabel: "true"} + mine := "composectl/install/app1@sha256:aaaa" all := []leases.Lease{ - {ID: "composectl/install/app1@sha256:aaaa"}, + {ID: mine, Labels: synced}, {ID: "moby-pull-123"}, - {ID: "composectl/install/app2@sha256:bbbb"}, + {ID: "composectl/install/app2@sha256:bbbb", Labels: synced}, {ID: "composectl/other"}, - } - got := installLeases(all) - if len(got) != 2 || got[0].ID != all[0].ID || got[1].ID != all[2].ID { - t.Fatalf("unexpected install leases: %+v", got) - } - if got := installLeases(nil); len(got) != 0 { - t.Fatalf("expected no leases, got %+v", got) + {ID: "composectl/install/app3@sha256:cccc"}, + } + stale, kept := staleInstallLeases(all, mine) + if !kept || len(stale) != 2 || stale[0].ID != all[2].ID || stale[1].ID != all[4].ID { + t.Fatalf("expected the synced lease of the app kept and the other install leases stale, got kept %v, stale %+v", kept, stale) + } + // The same app's lease without the label: its snapshots may be torn, so it is stale too. + all[0].Labels = nil + stale, kept = staleInstallLeases(all, mine) + if kept || len(stale) != 3 || stale[0].ID != mine { + t.Fatalf("expected the unlabelled lease of the app stale, got kept %v, stale %+v", kept, stale) + } + // No lease to keep: every install lease is stale. + all[0].Labels = synced + stale, kept = staleInstallLeases(all, "") + if kept || len(stale) != 3 { + t.Fatalf("expected every install lease stale, got kept %v, stale %+v", kept, stale) + } + if stale, kept := staleInstallLeases(nil, mine); kept || len(stale) != 0 { + t.Fatalf("expected nothing from no leases, got kept %v, stale %+v", kept, stale) } } diff --git a/pkg/compose/docker_client_test.go b/pkg/compose/docker_client_test.go index fe945f4..99f5a8f 100644 --- a/pkg/compose/docker_client_test.go +++ b/pkg/compose/docker_client_test.go @@ -95,6 +95,12 @@ func TestVersionAtLeast(t *testing.T) { {"29.5", "29.5.2", false}, {"29.6", "29.5.2", true}, {"", "29.5.2", false}, + // containerd versions against the sync_fs floor + {"v2.4.1", "1.7.14", true}, + {"2.2.3+unknown", "1.7.14", true}, + {"v1.7.14", "1.7.14", true}, + {"v1.7.13", "1.7.14", false}, + {"dev", "1.7.14", false}, {"garbage", "29.5.2", false}, } { if got := versionAtLeast(tc.version, tc.min); got != tc.want { diff --git a/pkg/compose/export_test.go b/pkg/compose/export_test.go index 107cb17..81e00ba 100644 --- a/pkg/compose/export_test.go +++ b/pkg/compose/export_test.go @@ -10,6 +10,10 @@ import ( type CtrdClient = ctrdClient +const InstallLeaseSyncedLabel = installLeaseSyncedLabel + +var UnpackPollInterval = &unpackPollInterval + var ( ConnectCtrdIfInstallTarget = connectCtrdIfInstallTarget InstallImagesCtrd = installImagesCtrd diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 011b69f..cda6f45 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -496,6 +496,18 @@ func randomString(length int) string { // pushes it to the test registry through the publisher dockerd, which the publish step reads the // image layers from. It returns the image reference. func PublishDerivedImage(t *testing.T, base string, sizeMB int) string { + t.Helper() + return publishDerivedImage(t, base, fmt.Sprintf("head -c %d /dev/urandom > /derived", sizeMB<<20)) +} + +// PublishDerivedImageOfFiles is PublishDerivedImage with a layer of the given number of small +// files: its extraction takes a syscall per file, not a disk write, so it stays slow on a fast disk. +func PublishDerivedImageOfFiles(t *testing.T, base string, files int) string { + t.Helper() + return publishDerivedImage(t, base, fmt.Sprintf("mkdir /derived-files && for i in $(seq %d); do echo $i > /derived-files/$i; done", files)) +} + +func publishDerivedImage(t *testing.T, base, command string) string { t.Helper() name := "derived-" + randomString(8) ref := "registry:5000/factory/" + name + ":v1" @@ -504,7 +516,7 @@ func PublishDerivedImage(t *testing.T, base string, sizeMB int) string { Checkf(t, err, "docker %v: %s", args, out) } pub("pull", base) - pub("run", "--name", name, base, "sh", "-c", fmt.Sprintf("head -c %d /dev/urandom > /derived", sizeMB<<20)) + pub("run", "--name", name, base, "sh", "-c", command) pub("commit", name, ref) pub("rm", name) pub("push", ref) diff --git a/test/integration/ctrd_install_test.go b/test/integration/ctrd_install_test.go index 0b82249..c95d36f 100644 --- a/test/integration/ctrd_install_test.go +++ b/test/integration/ctrd_install_test.go @@ -330,8 +330,8 @@ func TestCtrdInstallAfterPrune(t *testing.T) { } // TestCtrdInstallRecoversFromKilledInstall: an install is killed after it created snapshots; its -// lease stays, the next install removes the lease and with it the snapshots it alone held, and -// recreates them, so that nothing a power cut may have broken is reused. +// lease stays, marked as synced, and the next install goes on under it: the snapshots the killed +// install committed are kept, the rest are created, and the lease is gone at the end. func TestCtrdInstallRecoversFromKilledInstall(t *testing.T) { r := newCtrdRig(t) const base = "ghcr.io/foundriesio/busybox:1.36" @@ -342,18 +342,31 @@ func TestCtrdInstallRecoversFromKilledInstall(t *testing.T) { defer app.Remove(t) r.pruneImages(r.topChainIDs(app)) killedAt := r.killInstall(app) - if leftovers := r.installLeases(); len(leftovers) != 1 { - t.Fatalf("expected the lease of the killed install to stay, got %+v", leftovers) + if leftovers := r.installLeases(); len(leftovers) != 1 || leftovers[0].Labels["composectl/install.synced"] != "true" { + t.Fatalf("expected the lease of the killed install to stay, marked as synced, got %+v", leftovers) + } + committed := map[digest.Digest]time.Time{} + for _, img := range r.images(app.PublishedUri) { + for _, chainID := range img.chainIDs { + if info, ok := r.snapshot(chainID); ok { + committed[chainID] = info.Created + } + } } app.Install(t) defer app.Uninstall(t) r.checkInstalled(app) + if leftovers := r.installLeases(); len(leftovers) != 0 { + t.Errorf("expected no install lease after the resumed install, got %+v", leftovers) + } for _, img := range r.images(app.PublishedUri) { for _, chainID := range img.chainIDs { info, ok := r.snapshot(chainID) - if !ok || !info.Created.After(killedAt) { - t.Errorf("snapshot %s was not recreated after the killed install: created %v, killed at %v", chainID, info.Created, killedAt) + if createdAt, kept := committed[chainID]; kept && (!ok || !info.Created.Equal(createdAt)) { + t.Errorf("snapshot %s of the killed install was not kept: created %v, was %v", chainID, info.Created, createdAt) + } else if !kept && (!ok || !info.Created.After(killedAt)) { + t.Errorf("snapshot %s was not created after the killed install: created %v, killed at %v", chainID, info.Created, killedAt) } } }