fix: start and install on dockerd 29; e2e on both image stores

.github/workflows/ci.yml

@@ -26,7 +26,15 @@ jobs:
 
   test:
     name: Build and run all tests
+    strategy:
+      fail-fast: false
+      matrix:
+        dockerd: [stock, fio]
+        containerd-snapshotter: [false, true]
     runs-on: ubuntu-latest
+    env:
+      CONTAINERD_SNAPSHOTTER: ${{ matrix.containerd-snapshotter }}
+      DOCKERD_FIO_PATCH: ${{ matrix.dockerd == 'fio' && '1' || '' }}
     steps:
       - uses: actions/checkout@v3
       - name: build

dev-shell.sh

@@ -1,7 +1,8 @@
 down() {
     docker compose --env-file=test/compose/.env.test -f test/compose/docker-compose.yml down --remove-orphans
 	# remove the docker runtime and compose app store volumes
-    docker volume rm compose_docker-data compose_docker-runtime compose_reset-apps
+    docker volume rm compose_docker-data compose_docker-runtime compose_reset-apps \
+        compose_publisher-data compose_publisher-runtime
 }
 
 trap down EXIT

pkg/compose/image_loader.go

@@ -396,7 +396,7 @@ func reportProgressIfContainerd(body io.ReadCloser, imageURIs []imageURI2RefCoun
 		if strings.HasPrefix(jm.Stream, "Error unpacking image") {
 			return fmt.Errorf("error unpacking image: %s", jm.Stream)
 		}
-		if strings.HasPrefix(jm.Stream, "Loaded image: ") {
+		if reporter != nil && strings.HasPrefix(jm.Stream, "Loaded image: ") {
 			imageRef := strings.TrimPrefix(jm.Stream, "Loaded image: ")
 			// Remove end line character if present
 			imageRef = strings.TrimSuffix(imageRef, "\n")

pkg/compose/start.go

@@ -7,6 +7,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"os"
 	"os/exec"
+	"strings"
 )
 
 type (
@@ -65,32 +66,64 @@ func StartApps(ctx context.Context, cfg *Config, appURIs []string, options ...St
 		if opts.ProgressHandler != nil {
 			opts.ProgressHandler(app, AppStartStatusStarting, nil)
 		}
-		cmd := exec.Command("docker", "compose", "up", "-d", "--remove-orphans")
-		cmd.Dir = cfg.GetAppComposeDir(app.Name())
-		if opts.Verbose {
-			// Directly connect to stdout/stderr, so we can see the output in real time
-			cmd.Stdout = os.Stdout
-			cmd.Stderr = os.Stderr
-		} else {
-			// Capture stdout/stderr for error reporting
-			var stdout, stderr bytes.Buffer
-			cmd.Stdout = &stdout
-			cmd.Stderr = &stderr
-		}
-		if err := cmd.Run(); err != nil {
+		if err := composeUp(cfg.GetAppComposeDir(app.Name()), opts.Verbose); err != nil {
 			if opts.ProgressHandler != nil {
 				opts.ProgressHandler(app, AppStartStatusFailed, err)
 			}
-			if opts.Verbose {
-				return fmt.Errorf("failed to start %s: %w", app, err)
-			} else {
-				return fmt.Errorf("failed to start %s: %w\n\tstdout: %s\n\tstderr: %s", app, err, cmd.Stdout, cmd.Stderr)
-			}
-
+			return fmt.Errorf("failed to start %s: %w", app, err)
 		}
 		if opts.ProgressHandler != nil {
 			opts.ProgressHandler(app, AppStartStatusStarted, nil)
 		}
 	}
 	return nil
 }
+
+// composeUp runs `docker compose up` in the given app compose directory.
+// If it fails, the app containers that were created but not started are removed.
+func composeUp(composeDir string, verbose bool) error {
+	cmd := exec.Command("docker", "compose", "up", "-d", "--remove-orphans")
+	cmd.Dir = composeDir
+	var stdout, stderr bytes.Buffer
+	if verbose {
+		// Directly connect to stdout/stderr, so we can see the output in real time
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	} else {
+		// Capture stdout/stderr for error reporting
+		cmd.Stdout = &stdout
+		cmd.Stderr = &stderr
+	}
+	err := cmd.Run()
+	if err == nil {
+		return nil
+	}
+	if !verbose {
+		err = fmt.Errorf("%w\n\tstdout: %s\n\tstderr: %s", err, stdout.String(), stderr.String())
+	}
+	if errRm := removeCreatedContainers(composeDir); errRm != nil {
+		err = fmt.Errorf("%w\n\tfailed to clean up after the failed start: %s", err, errRm.Error())
+	}
+	return err
+}
+
+// removeCreatedContainers removes the app containers that `docker compose up` created but failed to start.
+// Starting such a container again may succeed without its published ports (Docker 29.7.2 does so after
+// a host port conflict), so the next start attempt could report success for an app that is broken.
+func removeCreatedContainers(composeDir string) error {
+	cmd := exec.Command("docker", "compose", "ps", "--all", "--quiet", "--status", "created")
+	cmd.Dir = composeDir
+	out, err := cmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to list containers in the created state: %w", err)
+	}
+	ids := strings.Fields(string(out))
+	if len(ids) == 0 {
+		return nil
+	}
+	cmd = exec.Command("docker", append([]string{"rm"}, ids...)...)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to remove containers in the created state: %w: %s", err, out)
+	}
+	return nil
+}

test/compose/.env.test

@@ -7,3 +7,8 @@ REG_DIR=$CPS_DIR/registry
 REG_CERT_DIR=$REG_DIR/certs
 # This must be relative to SRC_DIR path
 REG_CERT=test/compose/registry/certs/registry.crt
+
+# Image store of the test dockerd: false - the overlay2 graphdriver, true - the containerd image store
+CONTAINERD_SNAPSHOTTER=false
+# Empty: run tests against the stock dockerd; 1: against the dockerd with the Foundries patches
+DOCKERD_FIO_PATCH=

test/compose/docker-compose.yml

@@ -12,15 +12,36 @@ services:
       - ${REG_DIR}/config.yml:/etc/distribution/config.yml
 
   dockerd:
-    image: ghcr.io/foundriesio/moby:25.0.3_fio
-    command: ["dockerd", "-D", "-H", "unix:///var/run/docker/docker.sock"]
+    image: composeapp-test-dockerd:29.7.2${DOCKERD_FIO_PATCH:+-fio}
+    build:
+      context: ${CPS_DIR}/dockerd
+      args:
+        DOCKERD_FIO_PATCH: ${DOCKERD_FIO_PATCH}
+    command: ["dockerd", "-D", "-H", "unix:///var/run/docker/docker.sock",
+              "--feature", "containerd-snapshotter=${CONTAINERD_SNAPSHOTTER}"]
     privileged: true
     volumes:
       - docker-runtime:/var/run/docker
       - docker-data:/var/lib/docker
       - ${REG_DIR}/daemon.json:/etc/docker/daemon.json:ro
       - reset-apps:/var/sota/reset-apps
 
+  # Plays the role of the FoundriesFactory CI when tests publish apps: test images are pulled into it
+  # and the layer sizes are read from its overlay2 store, whatever image store the dockerd above uses.
+  dockerd-publisher:
+    image: composeapp-test-dockerd:29.7.2
+    build:
+      context: ${CPS_DIR}/dockerd
+      args:
+        DOCKERD_FIO_PATCH: ""
+    command: ["dockerd", "-D", "-H", "unix:///var/run/docker-publisher/docker.sock",
+              "--feature", "containerd-snapshotter=false"]
+    privileged: true
+    volumes:
+      - publisher-runtime:/var/run/docker-publisher
+      - publisher-data:/var/lib/docker
+      - ${REG_DIR}/daemon.json:/etc/docker/daemon.json:ro
+
   composectl:
     build:
       context: ${SRC_DIR}
@@ -34,20 +55,28 @@ services:
       - docker-runtime:/var/run/docker
       - docker-data:/var/lib/docker
       - reset-apps:/var/sota/reset-apps
+      - publisher-runtime:/var/run/docker-publisher
+      - publisher-data:/var/lib/docker-publisher:ro
     working_dir: ${SRC_DIR}
     depends_on:
       - registry
       - registry-org
       - dockerd
+      - dockerd-publisher
     environment:
       - GOCACHE=${SRC_DIR}/.cache
       - DOCKER_HOST=unix:///var/run/docker/docker.sock
       - COMPOSECTL_EXE=${BIN_DIR}/composectl
       - SRC_DIR=${SRC_DIR}
       - STOREROOT=/var/sota/reset-apps
       - COMPOSEROOT=/var/sota/compose-apps
+      - DOCKERD_FIO_PATCH=${DOCKERD_FIO_PATCH}
+      - PUBLISHER_DOCKER_HOST=unix:///var/run/docker-publisher/docker.sock
+      - PUBLISHER_DOCKER_DATA_ROOT=/var/lib/docker-publisher
 
 volumes:
   docker-data:
   docker-runtime:
   reset-apps:
+  publisher-data:
+  publisher-runtime:

test/compose/dockerd/Dockerfile

@@ -0,0 +1,15 @@
+# dockerd with a newer containerd, for the test environment.
+# Built by `docker compose` on first start; rebuild with `docker compose build`.
+ARG DOCKER_VERSION=29.7.2
+# Empty: the stock dockerd; 1: the dockerd with the Foundries patches
+ARG DOCKERD_FIO_PATCH=
+
+FROM docker:${DOCKER_VERSION}-dind AS dockerd
+FROM ghcr.io/foundriesio/moby:${DOCKER_VERSION}_fio AS dockerd-fio
+
+FROM dockerd${DOCKERD_FIO_PATCH:+-fio}
+ARG CONTAINERD_VERSION=2.4.1
+ARG TARGETARCH
+# Replace the containerd shipped with Docker; dockerd starts the one found on its PATH
+RUN wget -qO- https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-static-${CONTAINERD_VERSION}-linux-${TARGETARCH}.tar.gz \
+    | tar -xz -C /usr/local

test/fixtures/composectl_cmds.go

@@ -123,10 +123,18 @@ func (a *App) getAppImages(t *testing.T, composeFilePath string) []string {
 	return images
 }
 
+// publisherDockerCommand returns a docker CLI command run against the publisher dockerd,
+// the daemon that the layers meta script reads image layer sizes from.
+func publisherDockerCommand(args ...string) *exec.Cmd {
+	c := exec.Command("docker", args...)
+	c.Env = append(os.Environ(), "DOCKER_HOST="+os.Getenv("PUBLISHER_DOCKER_HOST"))
+	return c
+}
+
 func (a *App) pullImages(t *testing.T) {
 	images := a.GetAppImages(t)
 	for _, image := range images {
-		c := exec.Command("docker", "pull", image)
+		c := publisherDockerCommand("pull", image)
 		output, cmdErr := c.CombinedOutput()
 		Checkf(t, cmdErr, "failed to pull app images: %s\n", output)
 	}
@@ -139,7 +147,7 @@ func (a *App) removeImages(t *testing.T) {
 		if _, ok := removedImages[image]; ok {
 			continue
 		}
-		c := exec.Command("docker", "image", "rm", image)
+		c := publisherDockerCommand("image", "rm", image)
 		output, cmdErr := c.CombinedOutput()
 		Checkf(t, cmdErr, "failed to pull app images: %s\n", output)
 		removedImages[image] = true

test/fixtures/layers_meta.go

@@ -12,7 +12,7 @@ func GenerateLayersMetaFile(t *testing.T, appsRootDir string) string {
 	tmpDir := t.TempDir()
 	layerSizesFile := filepath.Join(tmpDir, "layers_meta.json")
 	c := exec.Command("python", os.Getenv("LAYERS_SIZE_SCRIPT"), "--apps-root",
-		appsRootDir, "-o", layerSizesFile)
+		appsRootDir, "-o", layerSizesFile, "--docker-data-root", os.Getenv("PUBLISHER_DOCKER_DATA_ROOT"))
 	output, err := c.CombinedOutput()
 	Checkf(t, err, "failed to run command to gather app layer sizes: %s", string(output))
 	b, err := os.ReadFile(layerSizesFile)

test/integration/image_loading_test.go

@@ -45,6 +45,9 @@ services:
 
 	err = loadImages(t, context.Background(), cli, composeApp, layersRoot)
 	f.Check(t, err)
+	if os.Getenv("DOCKERD_FIO_PATCH") != "1" {
+		t.Skip("skipping image loading that requires the Foundries patches to dockerd; set DOCKERD_FIO_PATCH=1 to run it")
+	}
 	err = loadImages(t, context.Background(), cli, composeApp, layersRoot,
 		compose.WithProgressReporting(progressHandler), compose.WithBlobReadingFromStore(), compose.WithRefWithDigest())
 	f.Check(t, err)