From 2799b13099ff0d01f2d51c1feeaa920ba8491320 Mon Sep 17 00:00:00 2001 From: Nan Liu Date: Thu, 3 Sep 2026 00:19:16 +0000 Subject: [PATCH] fix(config): resolve components in config dump output Resolve components before serializing so config dump includes inherited distro, project, and group defaults, expanded overlay-files, and components discovered through group spec globs. Preserve resolved lock data and derived rendered-spec directories in JSON output when available. TOML continues to omit these runtime fields through the existing serialization tags. Keep the loaded configuration unchanged and let consumers filter the JSON fields they need. Retain default configuration blocks alongside resolved component entries. The dump is an inspection snapshot for debugging and scripting, not project configuration intended to be loaded again. Update tests and documentation for the resolved output and JSON/TOML runtime-field behavior. Fixes #264 --- docs/user/explanation/config-system.md | 8 +- docs/user/explanation/repos.md | 4 +- docs/user/reference/cli/azldev_config_dump.md | 8 +- docs/user/reference/config/config-file.md | 2 +- internal/app/azldev/cmds/config/dump.go | 30 +++- internal/app/azldev/cmds/config/dump_test.go | 162 +++++++++++++----- 6 files changed, 160 insertions(+), 54 deletions(-) diff --git a/docs/user/explanation/config-system.md b/docs/user/explanation/config-system.md index ac5059ea9..33e2d9548 100644 --- a/docs/user/explanation/config-system.md +++ b/docs/user/explanation/config-system.md @@ -152,7 +152,13 @@ azldev validates each config file against its schema as it is loaded. Unknown fi azldev config generate-schema ``` -Use `azldev config dump -q -O json` to inspect the fully resolved configuration after all includes are merged and inheritance is applied. +Use `azldev config dump -q -f json` to inspect the fully resolved configuration after all includes are merged, component inheritance is applied, and per-file overlays are expanded. The dump retains the default configuration blocks alongside the resolved component entries, so it is an inspection snapshot and must not be reloaded as project configuration. + +JSON output also includes resolved lock data (`locked`) when available and the derived rendered-spec directory (`renderedSpecDir`) when configured. TOML output omits these runtime fields. To exclude them from JSON output, filter it explicitly: + +```sh +azldev config dump -q -f json | jq 'del(.components[] | .locked, .renderedSpecDir)' +``` ## Related Resources diff --git a/docs/user/explanation/repos.md b/docs/user/explanation/repos.md index d46938536..712006480 100644 --- a/docs/user/explanation/repos.md +++ b/docs/user/explanation/repos.md @@ -181,7 +181,7 @@ azldev repo compare --left koji-build --right pmc-prod --missing-from-right --st Expansion is deterministic and happens once during `ProjectConfig.Validate()`, after **all** config files (project, user, `--config-file` extras) have been merged. This means: - a later config file can override an earlier set's `base-uri` or `gpg-key`; -- the merged config in `azldev config dump` is round-trippable: what you see is what you authored, and re-loading the dumped TOML produces the same effective state; +- `azldev config dump` emits an inspection snapshot of the effective configuration; it is not intended to be reloaded as project configuration; - consumers (mock, kiwi) call `ResourcesConfig.EffectiveRpmRepos()` to get the flat resolved map without re-parsing layouts. `gpg-key` paths follow the usual rule: bare paths are resolved relative to the directory of the file that defines them, then re-emitted as a `file` URI. URI-shaped values (an `http` or `https` URI, or a `file` URI with an absolute path) are passed through unchanged. @@ -274,7 +274,7 @@ image-build = [ Inspect the fully resolved configuration with: ```sh -azldev config dump -q -O json | jq '.resources, .distros.mydistro.versions["4.0"].inputs' +azldev config dump -q -f json | jq '.resources, .distros.mydistro.versions["4.0"].inputs' ``` ## Authoring tips diff --git a/docs/user/reference/cli/azldev_config_dump.md b/docs/user/reference/cli/azldev_config_dump.md index 4e40022d5..1706591cf 100644 --- a/docs/user/reference/cli/azldev_config_dump.md +++ b/docs/user/reference/cli/azldev_config_dump.md @@ -9,9 +9,11 @@ Dump the current configuration Dump the fully resolved project configuration. Shows the merged result of all config files (embedded defaults, project -config, includes, and any extra --config-file arguments) after inheritance -and merge rules have been applied. Useful for debugging configuration -issues or inspecting effective values. +config, includes, and any extra --config-file arguments). Component entries +include inherited defaults and overlays expanded from 'overlay-files'. + +The output is an inspection snapshot for debugging and scripting, not project +configuration intended to be loaded again. ``` azldev config dump [flags] diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index 258e9cf2b..57041fcba 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -76,5 +76,5 @@ includes = ["**/*.comp.toml"] - [Configuration System](../../explanation/config-system.md) — how config files are loaded, merged, and how inheritance works - [JSON Schema](../../../../schemas/azldev.schema.json) — machine-readable schema for editor integration and validation -- Run `azldev config dump -q -O json` to inspect the fully resolved configuration +- Run `azldev config dump -q -f json` to inspect the fully resolved configuration. The output is an inspection snapshot, not project configuration intended to be loaded again. - Run `azldev config generate-schema` to generate the latest schema diff --git a/internal/app/azldev/cmds/config/dump.go b/internal/app/azldev/cmds/config/dump.go index 5630006c3..d463b9b6b 100644 --- a/internal/app/azldev/cmds/config/dump.go +++ b/internal/app/azldev/cmds/config/dump.go @@ -10,6 +10,7 @@ import ( "slices" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/pelletier/go-toml/v2" "github.com/spf13/cobra" @@ -63,9 +64,11 @@ func newDumpCmd() *cobra.Command { Long: `Dump the fully resolved project configuration. Shows the merged result of all config files (embedded defaults, project -config, includes, and any extra --config-file arguments) after inheritance -and merge rules have been applied. Useful for debugging configuration -issues or inspecting effective values.`, +config, includes, and any extra --config-file arguments). Component entries +include inherited defaults and overlays expanded from 'overlay-files'. + +The output is an inspection snapshot for debugging and scripting, not project +configuration intended to be loaded again.`, Example: ` # Dump config as TOML (default) azldev config dump @@ -100,7 +103,10 @@ issues or inspecting effective values.`, } func DumpConfig(env *azldev.Env, format configDumpFormat) (string, error) { - config := portableConfigCopy(env.Config()) + config, err := resolvedConfigCopy(env) + if err != nil { + return "", err + } switch format { case ConfigDumpFormatTOML: @@ -122,6 +128,22 @@ func DumpConfig(env *azldev.Env, format configDumpFormat) (string, error) { } } +func resolvedConfigCopy(env *azldev.Env) (*projectconfig.ProjectConfig, error) { + resolved, err := components.NewResolver(env).FindAllComponents() + if err != nil { + return nil, fmt.Errorf("failed to resolve components:\n%w", err) + } + + config := *env.Config() + + config.Components = make(map[string]projectconfig.ComponentConfig, resolved.Len()) + for _, component := range resolved.Components() { + config.Components[component.GetName()] = *component.GetConfig() + } + + return portableConfigCopy(&config), nil +} + func portableConfigCopy(config *projectconfig.ProjectConfig) *projectconfig.ProjectConfig { result := *config diff --git a/internal/app/azldev/cmds/config/dump_test.go b/internal/app/azldev/cmds/config/dump_test.go index 1bbb0562c..e9e783c43 100644 --- a/internal/app/azldev/cmds/config/dump_test.go +++ b/internal/app/azldev/cmds/config/dump_test.go @@ -4,68 +4,144 @@ package config_test import ( - "context" + "encoding/json" "testing" - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/config" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestDumpConfig(t *testing.T) { - const ( - testProjectRoot = "/non/existent/dir" - testLogDir = "/non/existent/logs" - testWorkDir = "/non/existent/work" - testOutputDir = "/non/existent/output" - ) + testEnv := testutils.NewTestEnv(t) + testEnv.Config.Project.RenderedSpecsDir = "/project/specs" + testEnv.Config.DefaultComponentConfig.Build.With = []string{"project-default"} + testEnv.Config.Components["example"] = projectconfig.ComponentConfig{ + Name: "example", + OverlayFiles: []string{"/project/components/example.overlay.toml"}, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "generated.tar.gz", + Origin: projectconfig.Origin{ + Type: projectconfig.OriginTypeCustom, + Script: "/project/components/generate.sh", + }, + }}, + } + testEnv.Config.ComponentGroups["example-group"] = projectconfig.ComponentGroupConfig{ + Components: []string{"example"}, + DefaultComponentConfig: projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{With: []string{"group-default"}}, + }, + } + testEnv.Config.GroupsByComponent["example"] = []string{"example-group"} + distro := testEnv.Config.Distros["test-distro"] + distroVersion := distro.Versions["1.0"] + distroVersion.DefaultComponentConfig.Build.With = []string{"distro-default"} + distro.Versions["1.0"] = distroVersion + testEnv.Config.Distros["test-distro"] = distro + testEnv.WriteDefaultLock(t, "example") + require.NoError(t, fileutils.WriteFile( + testEnv.TestFS, + "/project/components/example.overlay.toml", + []byte(`[metadata] +category = "azl-branding-policy" +upstream-status = "inapplicable" + +[[overlays]] +type = "spec-set-tag" +tag = "Vendor" +value = "Microsoft" +`), + fileperms.PrivateFile, + )) - cfg := &projectconfig.ProjectConfig{ - Project: projectconfig.ProjectInfo{ - LogDir: testLogDir, - WorkDir: testWorkDir, - OutputDir: testOutputDir, + testCases := []struct { + name string + dump func() (string, error) + unmarshal func([]byte, any) error + expectedLocked *projectconfig.ComponentLockData + expectedRenderedSpecDir string + }{ + { + name: "TOML", + dump: func() (string, error) { + return config.DumpConfig(testEnv.Env, config.ConfigDumpFormatTOML) + }, + unmarshal: toml.Unmarshal, }, - Components: map[string]projectconfig.ComponentConfig{ - "example": { - SourceFiles: []projectconfig.SourceFileReference{{ - Filename: "generated.tar.gz", - Origin: projectconfig.Origin{ - Type: projectconfig.OriginTypeCustom, - Script: "/project/components/generate.sh", - }, - }}, + { + name: "JSON", + dump: func() (string, error) { + return config.DumpConfig(testEnv.Env, config.ConfigDumpFormatJSON) + }, + unmarshal: json.Unmarshal, + expectedLocked: &projectconfig.ComponentLockData{ + UpstreamCommit: testutils.TestUpstreamCommit, }, + expectedRenderedSpecDir: "/project/specs/e/example", }, } - ctx, cancelFunc := context.WithCancel(t.Context()) - defer cancelFunc() + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + configText, err := testCase.dump() + require.NoError(t, err) - testEnv := testutils.NewTestEnv(t) + var dumped projectconfig.ProjectConfig + require.NoError(t, testCase.unmarshal([]byte(configText), &dumped)) - options := azldev.NewEnvOptions() - options.DryRunnable = testEnv.DryRunnable - options.EventListener = testEnv.EventListener - options.Interfaces = testEnv.TestInterfaces - options.ProjectDir = testProjectRoot - options.Config = cfg + dumpedComponent := dumped.Components["example"] + assert.Equal(t, + []string{"distro-default", "project-default", "group-default"}, + dumpedComponent.Build.With, + ) + require.Len(t, dumpedComponent.Overlays, 1) + assert.Equal(t, "Vendor", dumpedComponent.Overlays[0].Tag) + assert.Empty(t, dumpedComponent.OverlayFiles) + assert.Equal(t, testCase.expectedLocked, dumpedComponent.Locked) + assert.Equal(t, testCase.expectedRenderedSpecDir, dumpedComponent.RenderedSpecDir) + assert.Equal(t, []string{"project-default"}, dumped.DefaultComponentConfig.Build.With) + assert.Equal(t, []string{"group-default"}, + dumped.ComponentGroups["example-group"].DefaultComponentConfig.Build.With) + assert.Equal(t, "generate.sh", dumpedComponent.SourceFiles[0].Origin.Script) + }) + } - env := azldev.NewEnv(ctx, options) + originalComponent := testEnv.Config.Components["example"] + assert.Equal(t, []string{"/project/components/example.overlay.toml"}, originalComponent.OverlayFiles) + assert.Empty(t, originalComponent.Overlays) + assert.Nil(t, originalComponent.Locked) + assert.Empty(t, originalComponent.RenderedSpecDir) + assert.Equal(t, "/project/components/generate.sh", originalComponent.SourceFiles[0].Origin.Script) +} - configText, err := config.DumpConfig(env, config.ConfigDumpFormatTOML) - require.NoError(t, err) - require.NotEmpty(t, configText) - require.Contains(t, configText, "generate.sh") - require.NotContains(t, configText, "/project/components") +func TestDumpConfigIncludesSpecDiscoveredComponents(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + testEnv.Config.ComponentGroups["local-specs"] = projectconfig.ComponentGroupConfig{ + SpecPathPatterns: []string{"/project/specs/**/*.spec"}, + } + require.NoError(t, fileutils.WriteFile( + testEnv.TestFS, + "/project/specs/example/example.spec", + []byte("Name: example\n"), + fileperms.PrivateFile, + )) - configText, err = config.DumpConfig(env, config.ConfigDumpFormatJSON) + configText, err := config.DumpConfig(testEnv.Env, config.ConfigDumpFormatJSON) require.NoError(t, err) - require.NotEmpty(t, configText) - require.Contains(t, configText, "generate.sh") - require.NotContains(t, configText, "/project/components") - require.Equal(t, "/project/components/generate.sh", - cfg.Components["example"].SourceFiles[0].Origin.Script) + + var dumped projectconfig.ProjectConfig + require.NoError(t, json.Unmarshal([]byte(configText), &dumped)) + + dumpedComponent, found := dumped.Components["example"] + require.True(t, found) + assert.Equal(t, projectconfig.SpecSourceTypeLocal, dumpedComponent.Spec.SourceType) + assert.Equal(t, "/project/specs/example/example.spec", dumpedComponent.Spec.Path) + assert.Nil(t, dumpedComponent.Locked) + assert.Empty(t, dumpedComponent.RenderedSpecDir) }