diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f78d26f..4b60f72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,15 @@ jobs: test: name: Build and run all tests + strategy: + fail-fast: false + matrix: + dockerd: [stock, fio] + containerd-snapshotter: [false, true] runs-on: ubuntu-latest + env: + CONTAINERD_SNAPSHOTTER: ${{ matrix.containerd-snapshotter }} + DOCKERD_FIO_PATCH: ${{ matrix.dockerd == 'fio' && '1' || '' }} steps: - uses: actions/checkout@v3 - name: build diff --git a/dev-shell.sh b/dev-shell.sh index 0f31ab1..dc2a40f 100755 --- a/dev-shell.sh +++ b/dev-shell.sh @@ -1,7 +1,8 @@ down() { docker compose --env-file=test/compose/.env.test -f test/compose/docker-compose.yml down --remove-orphans # remove the docker runtime and compose app store volumes - docker volume rm compose_docker-data compose_docker-runtime compose_reset-apps + docker volume rm compose_docker-data compose_docker-runtime compose_reset-apps \ + compose_publisher-data compose_publisher-runtime } trap down EXIT diff --git a/pkg/compose/image_loader.go b/pkg/compose/image_loader.go index 3c810e0..c27738e 100644 --- a/pkg/compose/image_loader.go +++ b/pkg/compose/image_loader.go @@ -396,7 +396,7 @@ func reportProgressIfContainerd(body io.ReadCloser, imageURIs []imageURI2RefCoun if strings.HasPrefix(jm.Stream, "Error unpacking image") { return fmt.Errorf("error unpacking image: %s", jm.Stream) } - if strings.HasPrefix(jm.Stream, "Loaded image: ") { + if reporter != nil && strings.HasPrefix(jm.Stream, "Loaded image: ") { imageRef := strings.TrimPrefix(jm.Stream, "Loaded image: ") // Remove end line character if present imageRef = strings.TrimSuffix(imageRef, "\n") diff --git a/pkg/compose/start.go b/pkg/compose/start.go index 88694d0..5989556 100644 --- a/pkg/compose/start.go +++ b/pkg/compose/start.go @@ -7,6 +7,7 @@ import ( "github.com/containerd/containerd/platforms" "os" "os/exec" + "strings" ) type ( @@ -65,28 +66,11 @@ func StartApps(ctx context.Context, cfg *Config, appURIs []string, options ...St if opts.ProgressHandler != nil { opts.ProgressHandler(app, AppStartStatusStarting, nil) } - cmd := exec.Command("docker", "compose", "up", "-d", "--remove-orphans") - cmd.Dir = cfg.GetAppComposeDir(app.Name()) - if opts.Verbose { - // Directly connect to stdout/stderr, so we can see the output in real time - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - } else { - // Capture stdout/stderr for error reporting - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - } - if err := cmd.Run(); err != nil { + if err := composeUp(cfg.GetAppComposeDir(app.Name()), opts.Verbose); err != nil { if opts.ProgressHandler != nil { opts.ProgressHandler(app, AppStartStatusFailed, err) } - if opts.Verbose { - return fmt.Errorf("failed to start %s: %w", app, err) - } else { - return fmt.Errorf("failed to start %s: %w\n\tstdout: %s\n\tstderr: %s", app, err, cmd.Stdout, cmd.Stderr) - } - + return fmt.Errorf("failed to start %s: %w", app, err) } if opts.ProgressHandler != nil { opts.ProgressHandler(app, AppStartStatusStarted, nil) @@ -94,3 +78,52 @@ func StartApps(ctx context.Context, cfg *Config, appURIs []string, options ...St } return nil } + +// composeUp runs `docker compose up` in the given app compose directory. +// If it fails, the app containers that were created but not started are removed. +func composeUp(composeDir string, verbose bool) error { + cmd := exec.Command("docker", "compose", "up", "-d", "--remove-orphans") + cmd.Dir = composeDir + var stdout, stderr bytes.Buffer + if verbose { + // Directly connect to stdout/stderr, so we can see the output in real time + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } else { + // Capture stdout/stderr for error reporting + cmd.Stdout = &stdout + cmd.Stderr = &stderr + } + err := cmd.Run() + if err == nil { + return nil + } + if !verbose { + err = fmt.Errorf("%w\n\tstdout: %s\n\tstderr: %s", err, stdout.String(), stderr.String()) + } + if errRm := removeCreatedContainers(composeDir); errRm != nil { + err = fmt.Errorf("%w\n\tfailed to clean up after the failed start: %s", err, errRm.Error()) + } + return err +} + +// removeCreatedContainers removes the app containers that `docker compose up` created but failed to start. +// Starting such a container again may succeed without its published ports (Docker 29.7.2 does so after +// a host port conflict), so the next start attempt could report success for an app that is broken. +func removeCreatedContainers(composeDir string) error { + cmd := exec.Command("docker", "compose", "ps", "--all", "--quiet", "--status", "created") + cmd.Dir = composeDir + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to list containers in the created state: %w", err) + } + ids := strings.Fields(string(out)) + if len(ids) == 0 { + return nil + } + cmd = exec.Command("docker", append([]string{"rm"}, ids...)...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to remove containers in the created state: %w: %s", err, out) + } + return nil +} diff --git a/test/compose/.env.test b/test/compose/.env.test index 06b1c22..ac81c99 100644 --- a/test/compose/.env.test +++ b/test/compose/.env.test @@ -7,3 +7,8 @@ REG_DIR=$CPS_DIR/registry REG_CERT_DIR=$REG_DIR/certs # This must be relative to SRC_DIR path REG_CERT=test/compose/registry/certs/registry.crt + +# Image store of the test dockerd: false - the overlay2 graphdriver, true - the containerd image store +CONTAINERD_SNAPSHOTTER=false +# Empty: run tests against the stock dockerd; 1: against the dockerd with the Foundries patches +DOCKERD_FIO_PATCH= diff --git a/test/compose/docker-compose.yml b/test/compose/docker-compose.yml index 52cadf0..b2a70e5 100644 --- a/test/compose/docker-compose.yml +++ b/test/compose/docker-compose.yml @@ -12,8 +12,13 @@ services: - ${REG_DIR}/config.yml:/etc/distribution/config.yml dockerd: - image: ghcr.io/foundriesio/moby:25.0.3_fio - command: ["dockerd", "-D", "-H", "unix:///var/run/docker/docker.sock"] + image: composeapp-test-dockerd:29.7.2${DOCKERD_FIO_PATCH:+-fio} + build: + context: ${CPS_DIR}/dockerd + args: + DOCKERD_FIO_PATCH: ${DOCKERD_FIO_PATCH} + command: ["dockerd", "-D", "-H", "unix:///var/run/docker/docker.sock", + "--feature", "containerd-snapshotter=${CONTAINERD_SNAPSHOTTER}"] privileged: true volumes: - docker-runtime:/var/run/docker @@ -21,6 +26,22 @@ services: - ${REG_DIR}/daemon.json:/etc/docker/daemon.json:ro - reset-apps:/var/sota/reset-apps + # Plays the role of the FoundriesFactory CI when tests publish apps: test images are pulled into it + # and the layer sizes are read from its overlay2 store, whatever image store the dockerd above uses. + dockerd-publisher: + image: composeapp-test-dockerd:29.7.2 + build: + context: ${CPS_DIR}/dockerd + args: + DOCKERD_FIO_PATCH: "" + command: ["dockerd", "-D", "-H", "unix:///var/run/docker-publisher/docker.sock", + "--feature", "containerd-snapshotter=false"] + privileged: true + volumes: + - publisher-runtime:/var/run/docker-publisher + - publisher-data:/var/lib/docker + - ${REG_DIR}/daemon.json:/etc/docker/daemon.json:ro + composectl: build: context: ${SRC_DIR} @@ -34,11 +55,14 @@ services: - docker-runtime:/var/run/docker - docker-data:/var/lib/docker - reset-apps:/var/sota/reset-apps + - publisher-runtime:/var/run/docker-publisher + - publisher-data:/var/lib/docker-publisher:ro working_dir: ${SRC_DIR} depends_on: - registry - registry-org - dockerd + - dockerd-publisher environment: - GOCACHE=${SRC_DIR}/.cache - DOCKER_HOST=unix:///var/run/docker/docker.sock @@ -46,8 +70,13 @@ services: - SRC_DIR=${SRC_DIR} - STOREROOT=/var/sota/reset-apps - COMPOSEROOT=/var/sota/compose-apps + - DOCKERD_FIO_PATCH=${DOCKERD_FIO_PATCH} + - PUBLISHER_DOCKER_HOST=unix:///var/run/docker-publisher/docker.sock + - PUBLISHER_DOCKER_DATA_ROOT=/var/lib/docker-publisher volumes: docker-data: docker-runtime: reset-apps: + publisher-data: + publisher-runtime: diff --git a/test/compose/dockerd/Dockerfile b/test/compose/dockerd/Dockerfile new file mode 100644 index 0000000..545f6f1 --- /dev/null +++ b/test/compose/dockerd/Dockerfile @@ -0,0 +1,15 @@ +# dockerd with a newer containerd, for the test environment. +# Built by `docker compose` on first start; rebuild with `docker compose build`. +ARG DOCKER_VERSION=29.7.2 +# Empty: the stock dockerd; 1: the dockerd with the Foundries patches +ARG DOCKERD_FIO_PATCH= + +FROM docker:${DOCKER_VERSION}-dind AS dockerd +FROM ghcr.io/foundriesio/moby:${DOCKER_VERSION}_fio AS dockerd-fio + +FROM dockerd${DOCKERD_FIO_PATCH:+-fio} +ARG CONTAINERD_VERSION=2.4.1 +ARG TARGETARCH +# Replace the containerd shipped with Docker; dockerd starts the one found on its PATH +RUN wget -qO- https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-static-${CONTAINERD_VERSION}-linux-${TARGETARCH}.tar.gz \ + | tar -xz -C /usr/local diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 3a66b4b..5385323 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -123,10 +123,18 @@ func (a *App) getAppImages(t *testing.T, composeFilePath string) []string { return images } +// publisherDockerCommand returns a docker CLI command run against the publisher dockerd, +// the daemon that the layers meta script reads image layer sizes from. +func publisherDockerCommand(args ...string) *exec.Cmd { + c := exec.Command("docker", args...) + c.Env = append(os.Environ(), "DOCKER_HOST="+os.Getenv("PUBLISHER_DOCKER_HOST")) + return c +} + func (a *App) pullImages(t *testing.T) { images := a.GetAppImages(t) for _, image := range images { - c := exec.Command("docker", "pull", image) + c := publisherDockerCommand("pull", image) output, cmdErr := c.CombinedOutput() Checkf(t, cmdErr, "failed to pull app images: %s\n", output) } @@ -139,7 +147,7 @@ func (a *App) removeImages(t *testing.T) { if _, ok := removedImages[image]; ok { continue } - c := exec.Command("docker", "image", "rm", image) + c := publisherDockerCommand("image", "rm", image) output, cmdErr := c.CombinedOutput() Checkf(t, cmdErr, "failed to pull app images: %s\n", output) removedImages[image] = true diff --git a/test/fixtures/layers_meta.go b/test/fixtures/layers_meta.go index c42ab87..2bf4949 100644 --- a/test/fixtures/layers_meta.go +++ b/test/fixtures/layers_meta.go @@ -12,7 +12,7 @@ func GenerateLayersMetaFile(t *testing.T, appsRootDir string) string { tmpDir := t.TempDir() layerSizesFile := filepath.Join(tmpDir, "layers_meta.json") c := exec.Command("python", os.Getenv("LAYERS_SIZE_SCRIPT"), "--apps-root", - appsRootDir, "-o", layerSizesFile) + appsRootDir, "-o", layerSizesFile, "--docker-data-root", os.Getenv("PUBLISHER_DOCKER_DATA_ROOT")) output, err := c.CombinedOutput() Checkf(t, err, "failed to run command to gather app layer sizes: %s", string(output)) b, err := os.ReadFile(layerSizesFile) diff --git a/test/integration/image_loading_test.go b/test/integration/image_loading_test.go index f41eda1..601c19b 100644 --- a/test/integration/image_loading_test.go +++ b/test/integration/image_loading_test.go @@ -45,6 +45,9 @@ services: err = loadImages(t, context.Background(), cli, composeApp, layersRoot) f.Check(t, err) + if os.Getenv("DOCKERD_FIO_PATCH") != "1" { + t.Skip("skipping image loading that requires the Foundries patches to dockerd; set DOCKERD_FIO_PATCH=1 to run it") + } err = loadImages(t, context.Background(), cli, composeApp, layersRoot, compose.WithProgressReporting(progressHandler), compose.WithBlobReadingFromStore(), compose.WithRefWithDigest()) f.Check(t, err)