From 9e8ceeda6817c5f63746b003591be7018b0bde95 Mon Sep 17 00:00:00 2001 From: Nan Liu Date: Sat, 14 Nov 2026 05:12:49 +0000 Subject: [PATCH] fix(test): prevent concurrent scenario image rebuilds The image warm-up did not prevent later container requests from rebuilding the shared latest tag. Parallel tests and package processes could replace that tag while another test was creating a container, causing AlreadyExists and No such image failures. Build each Dockerfile once per test process with a generated unique tag and create containers from the cached image reference. Cache build errors so every caller receives the failure without attempting another build. Add regressions for concurrent callers, separate Dockerfiles, and failed builds, and document the container harness behavior. Validation: unit tests, race-detector unit tests, quality checks, and container harness scenarios passed. The full mage all run encountered no-space-left-on-device failures in other scenarios. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/developer/how-to/testing.md | 5 + .../internal/containertest/containercache.go | 79 ++++++++++++--- .../containercache_internal_test.go | 97 +++++++++++++++++++ scenario/internal/containertest/containers.go | 60 ++---------- .../containertest/containers_internal_test.go | 2 - 5 files changed, 176 insertions(+), 67 deletions(-) create mode 100644 scenario/internal/containertest/containercache_internal_test.go diff --git a/docs/developer/how-to/testing.md b/docs/developer/how-to/testing.md index 987d18c29..0307017f8 100644 --- a/docs/developer/how-to/testing.md +++ b/docs/developer/how-to/testing.md @@ -37,6 +37,11 @@ This document provides guidelines for writing and maintaining tests within the A - Use `mage scenarioUpdate` when test expectations change and snapshots need updating - Use existing test helpers in `scenario/internal/cmdtest` +The container harness builds each selected Dockerfile once per test process and +reuses its generated image reference. Image tags are unique across test processes +so parallel packages cannot replace each other's images. Build failures are +reported to every caller instead of retrying with a potentially missing image. + ## End-to-End Tests Tests tagged `e2e` are excluded from `mage scenario` and `mage all`. Run the diff --git a/scenario/internal/containertest/containercache.go b/scenario/internal/containertest/containercache.go index 8d5fcc343..979cfbac7 100644 --- a/scenario/internal/containertest/containercache.go +++ b/scenario/internal/containertest/containercache.go @@ -4,27 +4,84 @@ package containertest import ( + "fmt" + "io" + "os" + "strconv" "sync" "testing" + + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/testhelpers" + testcontainers "github.com/testcontainers/testcontainers-go" ) //nolint:gochecknoglobals // The testing framework doesn't have a clean way to pass this around. -var containerCacheOnce sync.Once +var containerImages containerImageCache + +type containerImageCache struct { + mutex sync.Mutex + builds map[string]func() (string, error) +} + +func (c *containerImageCache) get(dockerfile string, build func() (string, error)) (string, error) { + c.mutex.Lock() + + if c.builds == nil { + c.builds = make(map[string]func() (string, error)) + } + + buildOnce, ok := c.builds[dockerfile] + if !ok { + buildOnce = sync.OnceValues(build) + c.builds[dockerfile] = buildOnce + } + + c.mutex.Unlock() + + return buildOnce() +} -// cacheContainerImage ensures that the container image is build only once per test (technically once per package that -// imports this package). Otherwise every parallel test would try to build the container image. -func cacheContainerImage(t *testing.T) { +// cacheContainerImage builds each Dockerfile once per test process. The generated tag prevents +// other test processes from replacing the image while containers are being created. +func cacheContainerImage(t *testing.T, dockerfile string, buildLogWriter io.Writer) (string, error) { t.Helper() - containerCacheOnce.Do(func() { - // This is a best-effort attempt to ensure that the container is created only once. If any error occurs during the - // creation, it will be logged but no further attempt will be made. - collateral := NewContainerTestCollateral(t) + return containerImages.get(dockerfile, func() (string, error) { + dockerPath, err := testhelpers.FindTestDockerDirectory() + if err != nil { + return "", fmt.Errorf("find scenario Docker build context:\n%w", err) + } - // Run a simple echo command to cache the container image - _, err := runCmdInContainerImpl(t, collateral, []string{"echo", "ready"}, NoTimeout) + provider, err := testcontainers.NewDockerProvider() if err != nil { - t.Logf("Failed to initialize container: %v", err) + return "", fmt.Errorf("get container provider:\n%w", err) + } + defer provider.Close() + + uidStr := strconv.Itoa(os.Getuid()) + gidStr := strconv.Itoa(os.Getgid()) + containerWorkDir := ContainerWorkDir + + req := testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: dockerPath, + Dockerfile: dockerfile, + Repo: containerURI, + KeepImage: true, + BuildLogWriter: buildLogWriter, + BuildArgs: map[string]*string{ + "UID": &uidStr, + "GID": &gidStr, + "WORK_DIR": &containerWorkDir, + }, + }, } + + image, err := provider.BuildImage(t.Context(), &req) + if err != nil { + return "", fmt.Errorf("build scenario image from %#q:\n%w", dockerfile, err) + } + + return image, nil }) } diff --git a/scenario/internal/containertest/containercache_internal_test.go b/scenario/internal/containertest/containercache_internal_test.go new file mode 100644 index 000000000..0da50a315 --- /dev/null +++ b/scenario/internal/containertest/containercache_internal_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package containertest + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestContainerImageCacheConcurrentBuild(t *testing.T) { + t.Parallel() + + var ( + cache containerImageCache + builds atomic.Int32 + workers sync.WaitGroup + ) + + const callers = 16 + + start := make(chan struct{}) + images := make([]string, callers) + errs := make([]error, callers) + + for i := range callers { + workers.Go(func() { + <-start + + images[i], errs[i] = cache.get(defaultDockerfile, func() (string, error) { + builds.Add(1) + + return "scenario-image", nil + }) + }) + } + + close(start) + workers.Wait() + + for i := range callers { + require.NoError(t, errs[i]) + assert.Equal(t, "scenario-image", images[i]) + } + + assert.EqualValues(t, 1, builds.Load()) +} + +func TestContainerImageCacheSeparateDockerfiles(t *testing.T) { + t.Parallel() + + var cache containerImageCache + + builds := make(map[string]int) + + for range 2 { + for _, dockerfile := range []string{defaultDockerfile, "Dockerfile.image-builder", "Dockerfile.tmt-runner"} { + image, err := cache.get(dockerfile, func() (string, error) { + builds[dockerfile]++ + + return dockerfile + "-image", nil + }) + require.NoError(t, err) + assert.Equal(t, dockerfile+"-image", image) + } + } + + for dockerfile, count := range builds { + assert.Equal(t, 1, count, dockerfile) + } +} + +func TestContainerImageCacheBuildFailure(t *testing.T) { + t.Parallel() + + var cache containerImageCache + + builds := 0 + buildErr := errors.New("image build failed") + + for range 2 { + image, err := cache.get(defaultDockerfile, func() (string, error) { + builds++ + + return "", buildErr + }) + require.ErrorIs(t, err, buildErr) + assert.Empty(t, image) + } + + assert.Equal(t, 1, builds) +} diff --git a/scenario/internal/containertest/containers.go b/scenario/internal/containertest/containers.go index 4c8ab97fe..5d24f32eb 100644 --- a/scenario/internal/containertest/containers.go +++ b/scenario/internal/containertest/containers.go @@ -11,7 +11,6 @@ import ( "io" "os" "path" - "strconv" "strings" "sync" "testing" @@ -45,7 +44,6 @@ const ( // Container image reference. containerURI = "github.com/azldev/azldev-test/scenario/container" - containerTag = "latest" defaultDockerfile = "Dockerfile" ) @@ -235,26 +233,6 @@ func RunCmdInContainer( t.Skipf("skipping test %s because network tests are disabled", t.Name()) } - // Try to pre-provision the default container image. Alternate Dockerfiles - // build their own purpose-specific images and must not depend on this image. - if collateral.dockerfile == defaultDockerfile { - cacheContainerImage(t) - } - - return runCmdInContainerImpl(t, collateral, cmd, timeout) -} - -// runCmdInContainerImpl is the actual implementation of running a command in a container. It makes no attempt to -// cache the container image or check environment variables. It is split out from RunCmdInContainer so that it can be -// called from the cacheContainerImage function without causing a circular dependency. -func runCmdInContainerImpl( - t *testing.T, - collateral *ContainerCmdTestCollateral, - cmd []string, - timeout time.Duration, -) (results testhelpers.TestResults, err error) { - t.Helper() - // Compose the `ContainerFile` list to specify the files to copy into / write into the container. files := composeContainerFileList(collateral) @@ -268,15 +246,12 @@ func runCmdInContainerImpl( buildLogWriter: &bytes.Buffer{}, } - dockerPath, err := testhelpers.FindTestDockerDirectory() - require.NoError(t, err) - - // Need to duplicate the workdir here because the testcontainers library - // expects a string pointer for the build args and consts can't be used. - containerWorkDir := ContainerWorkDir + image, err := cacheContainerImage(t, collateral.dockerfile, logger.buildLogWriter) + if err != nil { + t.Logf("Container image build log: %s", logger.buildLogWriter.String()) + } - uidStr := strconv.Itoa(os.Getuid()) - gidStr := strconv.Itoa(os.Getgid()) + require.NoError(t, err) binds := []string{collateral.workdir + ":" + ContainerWorkDir} if collateral.extraMounts != nil { @@ -290,21 +265,7 @@ func runCmdInContainerImpl( // the azldev command at /azldev, and run the command in the container. req := testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ - // Build the container from the Dockerfile in the "docker" directory. - FromDockerfile: testcontainers.FromDockerfile{ - // This path is relative to this file. - Context: dockerPath, - Dockerfile: collateral.dockerfile, - Repo: containerURI, - Tag: containerImageTag(collateral.dockerfile), - KeepImage: true, - BuildLogWriter: logger.buildLogWriter, - BuildArgs: map[string]*string{ - "UID": &uidStr, - "GID": &gidStr, - "WORK_DIR": &containerWorkDir, - }, - }, + Image: image, // Mount the workdir for the test into the container. HostConfigModifier: hostConfigModifier(collateral, binds), @@ -354,15 +315,6 @@ func hostConfigModifier( } } -func containerImageTag(dockerfile string) string { - suffix := strings.TrimPrefix(dockerfile, defaultDockerfile) - if suffix == "" { - return containerTag - } - - return containerTag + strings.ReplaceAll(suffix, ".", "-") -} - func composeContainerFileList(collateral *ContainerCmdTestCollateral, ) []testcontainers.ContainerFile { files := []testcontainers.ContainerFile{} diff --git a/scenario/internal/containertest/containers_internal_test.go b/scenario/internal/containertest/containers_internal_test.go index c5c81a593..3574222cc 100644 --- a/scenario/internal/containertest/containers_internal_test.go +++ b/scenario/internal/containertest/containers_internal_test.go @@ -28,8 +28,6 @@ func TestContainerImageOptions(t *testing.T) { assert.Equal(t, "/dev/kvm", collateral.devices[0].PathInContainer) assert.Equal(t, "rwm", collateral.devices[0].CgroupPermissions) assert.Equal(t, []string{"123"}, collateral.groupAdd) - assert.Equal(t, "latest-image-builder", containerImageTag(collateral.dockerfile)) - assert.Equal(t, "latest", containerImageTag(defaultDockerfile)) } func TestContainerImageOptionsRejectInvalidValues(t *testing.T) {