From 33f1a8d98d1ae4a41063f76e3dcb7b0df2ffaf8d Mon Sep 17 00:00:00 2001 From: Tobias Brick Date: Thu, 15 Oct 2026 22:49:11 +0000 Subject: [PATCH] feat(mock): scope build environments to configuration directories Generate stable root names from a sanitized canonical configuration path and a short digest so separate worktrees can build concurrently. Reject mismatched configuration/include directories and apply shared naming logic after prod/preview selection in all four architecture configs, preserving upstream channel settings and repositories. Add agent guidance for inventorying and retiring worktree-associated roots and caches while preserving shared resources and diagnostics. Suppress cache allocation only during discovery queries and require approval and exclusive use before targeted cleanup. Treat orphan recovery as unvalidated rather than automatic cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 796d7df5-e235-425f-8a0c-74dbd8b9beab --- .../skills/azldev-worktree-cleanup/SKILL.md | 142 ++++++++++++++++++ AGENTS.md | 11 ++ distro/mock/azl4/azurelinux-4.0-aarch64.cfg | 1 + .../azl4/azurelinux-4.0-preview-aarch64.cfg | 1 + .../azl4/azurelinux-4.0-preview-x86_64.cfg | 1 + distro/mock/azl4/azurelinux-4.0-x86_64.cfg | 1 + distro/mock/azl4/config-directory-root.tpl | 20 +++ 7 files changed, 177 insertions(+) create mode 100644 .agents/skills/azldev-worktree-cleanup/SKILL.md create mode 100644 distro/mock/azl4/config-directory-root.tpl diff --git a/.agents/skills/azldev-worktree-cleanup/SKILL.md b/.agents/skills/azldev-worktree-cleanup/SKILL.md new file mode 100644 index 00000000000..d09146c0ea9 --- /dev/null +++ b/.agents/skills/azldev-worktree-cleanup/SKILL.md @@ -0,0 +1,142 @@ +--- +name: azldev-worktree-cleanup +description: "Read before removing an Azure Linux worktree, cleaning mock roots or caches, reclaiming build disk space, or investigating orphaned mock environments. Inventory worktree-associated resources, preserve diagnostics, obtain approval, and use targeted mock cleanup before worktree removal." +--- + +# Retire worktrees and their mock environments + +Worktree removal does not remove mock roots or caches. Treat cleanup as +explicit, best-effort housekeeping, not automatic garbage collection. +Inventory first; obtain approval for the exact resources before deleting. +Do not run cleanup merely because this skill was loaded. + +## While the worktree still exists + +1. Identify the exact worktree with `git worktree list --porcelain` and check + its `git status --short`. Confirm that commits are pushed, merged, or + intentionally preserved elsewhere before proposing worktree removal. + Do not discard uncommitted files, untracked outputs, or diagnostics. +2. Inspect its distro configuration and build logs to identify mock + selections and overrides actually used. Account for prod (`4.0`), preview + (`4.0-preview`), and each architecture: four current selections in + `distro/mock/azl4/`. Check custom `basedir`, `rootdir`, + root/cache overrides, unique extensions, and separately placed helper + environments. Read the `azldev` skill before running azldev commands. +3. Resolve current selections with mock, not a caller-owned implementation + of the naming algorithm. For example, with an absolute selected config: + + ```bash + cfg=/absolute/worktree/distro/mock/azl4/azurelinux-4.0-preview-x86_64.cfg + mock -r "$cfg" --configdir "$(dirname "$cfg")" \ + --disable-plugin=yum_cache --print-root-path + ``` + + Supply the same effective overrides as the original operation. The + canonical directory containing the selected `.cfg` must match + `--configdir`. Each architecture config applies `config-directory-root.tpl` + after its channel templates. Preserve the complete configuration directory; + moving a lone `.cfg` to a scratch directory loses required includes and + does not resolve the original identity. Prod and preview share a directory + label/digest but have distinct channel-specific roots and caches. + Mock initializes plugins even for path/configuration queries. For the + default Azure Linux plugin set, disabling `yum_cache` during discovery + avoids creating empty caches and lock files. Use the same discovery-only + option with `--debug-config` if unexpanded settings need inspection; + avoid `--debug-config-expanded`. Custom plugins may have other side + effects and need separate inspection. Do not carry the plugin-disable + option into builds or cleanup commands. +4. Inventory the associated live root, bootstrap root, and cache directories. + For the default layout, caches are under `/var/cache/mock/` + and `/var/cache/mock/-bootstrap`; root placement is under + `/var/lib/mock`. Confirm effective settings instead of assuming these + paths for custom configurations. Separately placed helpers may still + share caches. Include disk usage, missing resources, and uncertainties + in the inventory. +5. Check logs and directory listings for previous naming schemes or paths. + In particular, old stage1/stage2/stage2-prod environments are not selected + by the current prod/preview configurations; retain their original + configuration evidence for separately approved cleanup. + Current configuration discovery does not identify roots left by older + templates, a moved directory, or an earlier checkout at the same path. + Readable labels are hints, not evidence that all matching resources + belong to this worktree. +6. Confirm with the user that operations using the identity are stopped and + no new build, render, query, or shell operation will start during cleanup. + Check for active users as supporting evidence, not proof that a race + cannot occur. Mock's locks do not cover the whole root/cache deletion + transaction; if exclusive use cannot be established, defer cleanup. +7. Present exact resource paths, approximate disk savings, retained + diagnostics/outputs, and targeted commands for approval. Ask whether + failed environments must be preserved. Distinguish approval for mock + cleanup from approval to remove the worktree. +8. After approval, use mock's supported cleanup with the still-loadable + configuration and the matching overrides. For one default-layout + stage/architecture selection: + + ```bash + mock -r "$cfg" --configdir "$(dirname "$cfg")" --scrub all + ``` + + This is destructive: it removes the selected live root, bootstrap root, + and their caches. It does not retire every channel/architecture at once. + Do not use it if any targeted cache is shared with an identity that + must remain. Cover approved helper placements separately. +9. Verify each inventoried resource is gone, or explicitly retained. Report + failures and remaining paths; a successful command alone is not proof of + complete reclamation. If cleanup is incomplete, keep the configuration + available and stop before worktree removal unless the user explicitly + approves deferral with a recovery plan. Remove the worktree only after + its separate approval and preservation checks. + +Ordinary successful-build cleanup uses `--scrub chroot` with azldev's +default preserve policy. It intentionally leaves bootstrap roots and caches; +it is not complete retirement. Do not change every build to `--scrub all`. + +## If the worktree is already gone + +There is no authoritative mock environment registry. `mock --list-chroots` +lists available configuration files, not instantiated environments. For +read-only discovery of default locations: + +```bash +find /var/lib/mock /var/cache/mock \ + -mindepth 1 -maxdepth 1 -type d -printf '%p\n' | sort +``` + +Inspect both locations: a live root may be absent while bootstrap roots or +caches remain. Custom placement/cache overrides require separate discovery. +Report permission failures or inaccessible locations as incomplete inventory. + +Use readable path labels to nominate candidates. Corroborate ownership with +surviving configurations, build logs, worktree records, or other explicit +evidence; absence of the apparent owner path is not sufficient. Labels are +sanitized and may be truncated, distinct paths can have the same label, and +path reuse can associate a name with a different checkout. + +Report candidate paths, disk usage, ownership evidence, and uncertainties. +Do not delete ambiguous resources. A shared/external configuration may have +other callers even if the worktree that first used it is gone. + +Recovery through a different loadable configuration plus a recorded root +override is a candidate approach, **not an end-to-end validated orphan +cleanup recipe**. It must reproduce placement, bootstrap, and cache settings +without loading includes from the deleted directory. Do not prescribe it as +routine cleanup until a disposable-environment test demonstrates complete +removal and leaves unrelated resources untouched. Escalate for a separately +reviewed recovery plan instead. + +## Guardrails + +- Never use wildcard/prefix deletion, broad recursive removal, or + `mock --scrub-all-chroots` for this workflow. The latter guesses associations + with installed configuration names and is not reliable for checkout-local + roots. +- A readable-name match, an old timestamp, or an apparently idle lock is not + authorization to delete. Do not infer ownership by recomputing a digest. +- Removing a caller of external/shared configuration does not authorize + retiring that configuration's environments or caches. +- Preserve requested failed environments, caller `--resultdir` outputs, + `chroot_scan` artifacts, and any diagnostics needed for inspection. +- Keep cleanup records or logs outside a worktree scheduled for removal + when they are needed for deferred recovery. Do not claim a durable + inventory exists unless one was actually saved. diff --git a/AGENTS.md b/AGENTS.md index f5a923f539e..f2421fec7bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,6 +60,7 @@ Detailed workflows live in skills (loaded on-demand when relevant): | Remove component(s) from the distro | [`azldev-remove-component`](.agents/skills/azldev-remove-component/SKILL.md) | | Diagnose and fix overlay issues | [`azldev-overlays`](.agents/skills/azldev-overlays/SKILL.md) | | Test and inspect packages in mock chroot | [`azldev-mock`](.agents/skills/azldev-mock/SKILL.md) | +| Remove a worktree or inspect leftover mock roots/caches | [`azldev-worktree-cleanup`](.agents/skills/azldev-worktree-cleanup/SKILL.md) | | Review component for hygiene and best practices | [`azldev-comp-toml` review checklist](.agents/skills/azldev-comp-toml/SKILL.md#review-checklist) | | Build, boot, test, or configure images | [`azldev-image`](.agents/skills/azldev-image/SKILL.md) | | Triage Koji build failures | [`skill-koji-triage`](.agents/skills/skill-koji-triage/SKILL.md) | @@ -76,3 +77,13 @@ Detailed workflows live in skills (loaded on-demand when relevant): - **Components:** [`base/comps/AGENTS.md`](base/comps/AGENTS.md) — file organization rules - **Distro config:** [`distro/AGENTS.md`](distro/AGENTS.md) — build defaults, shared config + +## Worktree Cleanup + +Before removing an Azure Linux worktree, read +[`azldev-worktree-cleanup`](.agents/skills/azldev-worktree-cleanup/SKILL.md) +and check for associated mock roots, bootstrap roots, and caches. Worktree +removal does not remove those resources. Identify candidates and obtain +explicit approval before deleting them. Preserve requested diagnostics and +caller-owned outputs; do not delete shared configuration resources merely +because one caller worktree is being removed. diff --git a/distro/mock/azl4/azurelinux-4.0-aarch64.cfg b/distro/mock/azl4/azurelinux-4.0-aarch64.cfg index a1808c5f8c4..127758c5e12 100644 --- a/distro/mock/azl4/azurelinux-4.0-aarch64.cfg +++ b/distro/mock/azl4/azurelinux-4.0-aarch64.cfg @@ -4,3 +4,4 @@ config_opts['legal_host_arches'] = ('aarch64',) # Include common Azure Linux 4.0 configuration include('azurelinux-4.0.tpl') +include('config-directory-root.tpl') diff --git a/distro/mock/azl4/azurelinux-4.0-preview-aarch64.cfg b/distro/mock/azl4/azurelinux-4.0-preview-aarch64.cfg index 2969f92ce70..c7ae22cf226 100644 --- a/distro/mock/azl4/azurelinux-4.0-preview-aarch64.cfg +++ b/distro/mock/azl4/azurelinux-4.0-preview-aarch64.cfg @@ -4,3 +4,4 @@ config_opts['legal_host_arches'] = ('aarch64',) # Include Azure Linux 4.0 prod + preview configuration include('azurelinux-4.0-preview.tpl') +include('config-directory-root.tpl') diff --git a/distro/mock/azl4/azurelinux-4.0-preview-x86_64.cfg b/distro/mock/azl4/azurelinux-4.0-preview-x86_64.cfg index cf688dbb876..5bc827e349c 100644 --- a/distro/mock/azl4/azurelinux-4.0-preview-x86_64.cfg +++ b/distro/mock/azl4/azurelinux-4.0-preview-x86_64.cfg @@ -4,3 +4,4 @@ config_opts['legal_host_arches'] = ('x86_64',) # Include Azure Linux 4.0 prod + preview configuration include('azurelinux-4.0-preview.tpl') +include('config-directory-root.tpl') diff --git a/distro/mock/azl4/azurelinux-4.0-x86_64.cfg b/distro/mock/azl4/azurelinux-4.0-x86_64.cfg index 4e076cd8e88..dd467502955 100644 --- a/distro/mock/azl4/azurelinux-4.0-x86_64.cfg +++ b/distro/mock/azl4/azurelinux-4.0-x86_64.cfg @@ -4,3 +4,4 @@ config_opts['legal_host_arches'] = ('x86_64',) # Include common Azure Linux 4.0 configuration include('azurelinux-4.0.tpl') +include('config-directory-root.tpl') diff --git a/distro/mock/azl4/config-directory-root.tpl b/distro/mock/azl4/config-directory-root.tpl new file mode 100644 index 00000000000..d2687463218 --- /dev/null +++ b/distro/mock/azl4/config-directory-root.tpl @@ -0,0 +1,20 @@ +import hashlib +import os +import re + +# Apply once from the architecture config after channel-specific root selection. +config_dir = os.path.dirname(os.path.realpath(config_opts['config_file'])) +include_dir = os.path.realpath(config_opts['config_path']) +if config_dir != include_dir: + raise ValueError( + 'Selected mock configuration directory must match --configdir: ' + f'{config_dir!r} != {include_dir!r}' + ) + +config_dir_id = hashlib.sha256(os.fsencode(config_dir)).hexdigest()[:12] +# Keep the end of long paths so the worktree and configuration remain readable. +config_dir_label = re.sub(r'[^A-Za-z0-9._-]+', '-', config_dir).strip('-') +config_dir_label = config_dir_label[-120:].lstrip('-') or 'config' +config_opts['root'] = ( + f"{config_opts['root']}-v2-{config_dir_label}-{config_dir_id}" +)