From 382397f165f936c306a026305af286275fd997ee Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 7 Oct 2026 10:50:25 -0500 Subject: [PATCH 1/5] ci: install ccache 4.14.1 and its HTTPS storage helper in the CI image ccache can only reach an HTTPS remote cache through a storage helper, which it supports from 4.13 on. Ubuntu noble ships 4.9.1, whose built-in HTTP backend is plain HTTP only and is deprecated upstream. Install the official 4.14.1 glibc release binary and the reference ccache-storage-http-go helper (as ccache-storage-https) for amd64 and arm64, with pinned SHA-256 sums. Nothing is configured to use remote storage yet, so builds behave as before; the newer ccache does start from an empty local cache once, because the cache format changed. Co-Authored-By: Claude Opus 5.5 (1M context) --- contrib/containers/ci/ci.Dockerfile | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/contrib/containers/ci/ci.Dockerfile b/contrib/containers/ci/ci.Dockerfile index 42df9b08018c..070e0f26d65f 100644 --- a/contrib/containers/ci/ci.Dockerfile +++ b/contrib/containers/ci/ci.Dockerfile @@ -15,7 +15,6 @@ RUN set -ex; \ bc \ bear \ bison \ - ccache \ cmake \ g++-11 \ g++-14 \ @@ -32,6 +31,38 @@ RUN set -ex; \ zip \ && rm -rf /var/lib/apt/lists/* +# ccache from upstream rather than apt: remote storage over HTTPS needs a +# storage helper, which ccache supports from 4.13 (noble ships 4.9.1). +ARG TARGETARCH +ARG CCACHE_VERSION=4.14.1 +ARG CCACHE_HTTP_HELPER_VERSION=0.10 +RUN set -ex; \ + ARCH_INFERRED="${TARGETARCH}"; \ + if [ -z "${ARCH_INFERRED}" ]; then \ + ARCH_INFERRED="$(dpkg --print-architecture || true)"; \ + fi; \ + case "${ARCH_INFERRED}" in \ + amd64|x86_64) \ + CCACHE_ARCH="x86_64"; HELPER_ARCH="amd64"; \ + CCACHE_SHA256="ad63d19f5d09ea13f749651653a561c98a59994e673504a4266617b8754218f7"; \ + HELPER_SHA256="88963ef2cc21ca588145d46bcb02b8295275975ed75696eedf2cdb0e9543edd0" ;; \ + arm64|aarch64) \ + CCACHE_ARCH="aarch64"; HELPER_ARCH="arm64"; \ + CCACHE_SHA256="4e8f16aa3b55acc57dc24ab3b997bf2bcef049a3bde1c97e38b07397e4ed4f6d"; \ + HELPER_SHA256="f0ee3302b52f87628b69fa53d2cc52a626c4c5c7e43b779ac9e3db481418db40" ;; \ + *) echo "Unsupported architecture for ccache: ${ARCH_INFERRED}"; exit 1 ;; \ + esac; \ + curl -fL "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}-linux-${CCACHE_ARCH}-glibc.tar.xz" -o /tmp/ccache.tar.xz; \ + echo "${CCACHE_SHA256} /tmp/ccache.tar.xz" | sha256sum -c -; \ + tar -xf /tmp/ccache.tar.xz -C /tmp --strip-components=1 "ccache-${CCACHE_VERSION}-linux-${CCACHE_ARCH}-glibc/ccache"; \ + install -m 0755 /tmp/ccache /usr/local/bin/ccache; \ + curl -fL "https://github.com/ccache/ccache-storage-http-go/releases/download/v${CCACHE_HTTP_HELPER_VERSION}/ccache-storage-http-go-${CCACHE_HTTP_HELPER_VERSION}-linux-${HELPER_ARCH}.tar.gz" -o /tmp/helper.tar.gz; \ + echo "${HELPER_SHA256} /tmp/helper.tar.gz" | sha256sum -c -; \ + tar -xf /tmp/helper.tar.gz -C /tmp --strip-components=1 "ccache-storage-http-go-${CCACHE_HTTP_HELPER_VERSION}-linux-${HELPER_ARCH}/ccache-storage-http"; \ + install -m 0755 /tmp/ccache-storage-http /usr/local/bin/ccache-storage-https; \ + rm -f /tmp/ccache.tar.xz /tmp/ccache /tmp/helper.tar.gz /tmp/ccache-storage-http; \ + ccache --version | head -n 1 + # Install Clang + LLVM and set it as default RUN set -ex; \ apt-get update && apt-get install ${APT_ARGS} \ From 6c30dc17a96114f17105750dd324a31ec932d305 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 7 Oct 2026 10:50:25 -0500 Subject: [PATCH 2/5] ci: hand the remote ccache credential only to trusted runs A shared remote ccache must only be written by code we trust, but the whole point is that trusted pull requests write to it too: their author's fork has usually built most of the same objects minutes earlier. check-skip now emits cache-write. Any push qualifies, since it only ever sees its own repository's secret and pushing there already needs write access. A pull_request_target run qualifies only when author, actor and head repository owner pass the existing self-hosted allowlist (SELFHOSTED_LINT_AUTHORS). Nothing else does. The src-* jobs pass CCACHE_REMOTE_AUTH to build-src.yml through a secrets: mapping that yields an empty string when the gate is closed. That mapping is resolved before the job is sent to a runner, so an untrusted pull request never holds the credential at all. Holding it only in a skipped step would not be enough: job containers can reach the host's Docker socket and, from there, the runner process. build-src.yml declares the secret but does not use it yet. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/build-src.yml | 4 ++ .github/workflows/build.yml | 15 ++++++ .github/workflows/select_dynamic_runner.py | 25 ++++++++++ .../workflows/test_select_dynamic_runner.py | 47 +++++++++++++++++++ 4 files changed, 91 insertions(+) diff --git a/.github/workflows/build-src.yml b/.github/workflows/build-src.yml index 47a757d3941e..f44e2e25fcfe 100644 --- a/.github/workflows/build-src.yml +++ b/.github/workflows/build-src.yml @@ -38,6 +38,10 @@ on: description: "Runner label to use (e.g., ubuntu-24.04 or ubuntu-24.04-arm)" required: true type: string + secrets: + CCACHE_REMOTE_AUTH: + description: "user:password allowed to write to the remote ccache; empty means read-only" + required: false outputs: key: description: "Key needed for restoring artifacts bundle" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cf3884775436..63627c38f438 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,10 @@ jobs: use-blacksmith-amd64: ${{ steps.select-runner.outputs.use_blacksmith_amd64 }} use-blacksmith-arm64: ${{ steps.select-runner.outputs.use_blacksmith_arm64 }} backlog-count: ${{ steps.select-runner.outputs.backlog_count }} + # Gates CCACHE_REMOTE_AUTH for the src-* jobs. Their secrets: mapping is + # resolved before a job reaches a runner, so a run that fails this gate + # is handed an empty value rather than a credential it could read. + cache-write: ${{ steps.select-runner.outputs.cache_write }} decision-reason: ${{ steps.select-runner.outputs.decision_reason }} base-image-digest: ${{ steps.base-image.outputs.digest }} runner-check-skip: ${{ vars.RUNNER_CHECK_SKIP || 'ubuntu-24.04-arm' }} @@ -254,6 +258,8 @@ jobs: depends-dep-opts: ${{ needs.depends-aarch64-linux.outputs.dep-opts }} depends-artifact: ${{ needs.depends-aarch64-linux.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: &remote-ccache-auth + CCACHE_REMOTE_AUTH: ${{ needs.check-skip.outputs.cache-write == 'true' && secrets.CCACHE_REMOTE_AUTH || '' }} src-linux64: name: linux64-build @@ -268,6 +274,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth src-linux64_asan: name: linux64_asan-build @@ -282,6 +289,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth src-linux64_fuzz: name: linux64_fuzz-build @@ -296,6 +304,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth src-linux64_multiprocess: name: linux64_multiprocess-build @@ -310,6 +319,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64_multiprocess.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} + secrets: *remote-ccache-auth src-linux64_nowallet: name: linux64_nowallet-build @@ -323,6 +333,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64_nowallet.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64_nowallet.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth src-linux64_sqlite: name: linux64_sqlite-build @@ -337,6 +348,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth src-linux64_tsan: name: linux64_tsan-build @@ -351,6 +363,7 @@ jobs: depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }} depends-artifact: ${{ needs.depends-linux64_multiprocess.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} + secrets: *remote-ccache-auth src-mac: name: mac-build @@ -365,6 +378,7 @@ jobs: depends-artifact: ${{ needs.depends-mac.outputs.built-artifact }} sdk-artifact: ${{ needs.depends-mac.outputs.sdk-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth src-win64: name: win64-build @@ -378,6 +392,7 @@ jobs: depends-dep-opts: ${{ needs.depends-win64.outputs.dep-opts }} depends-artifact: ${{ needs.depends-win64.outputs.built-artifact }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + secrets: *remote-ccache-auth test-linux64: name: linux64-test diff --git a/.github/workflows/select_dynamic_runner.py b/.github/workflows/select_dynamic_runner.py index e1f4d2b70382..f6da231efa94 100644 --- a/.github/workflows/select_dynamic_runner.py +++ b/.github/workflows/select_dynamic_runner.py @@ -234,6 +234,24 @@ def is_selfhosted_allowed( return False +def allows_cache_write( + event_name: str, + event: Dict, + actor: str, + allowed_authors: Set[str], +) -> bool: + """Whether this run may receive the remote ccache write credential. + + The credential is the repository's own secret, so any push may have it: + pushing already requires write access to that repository. A pull request + executes its head, so it gets the credential only when the people and the + repository behind that head are trusted to run on our own hardware too. + """ + return event_name == "push" or is_selfhosted_allowed( + event_name, event, actor, allowed_authors + ) + + def select_prelint_runner( event_name: str, event: Dict, @@ -407,6 +425,7 @@ def prelint( runner_prelint_arm64, prelint_arm64_decision_reason = prelint( runner_selfhosted_arm64_var, runner_arm64, "selfhosted-arm64-disabled" ) + cache_write = allows_cache_write(event_name, event, actor, allowed_authors) return { "runner_amd64": runner_amd64, @@ -419,6 +438,7 @@ def prelint( "use_blacksmith_amd64": "true" if use_blacksmith_amd64 else "false", "use_blacksmith_arm64": "true" if use_blacksmith_arm64 else "false", "backlog_count": backlog_count, + "cache_write": "true" if cache_write else "false", "decision_reason": ";".join(decision_parts), "label_override": "true" if label_override else "false", } @@ -465,6 +485,11 @@ def write_step_summary(path: Optional[str], outputs: Dict[str, str]) -> None: fh.write("- Pre-lint amd64 decision: `{}`\n".format(outputs["prelint_decision_reason"])) fh.write("- Pre-lint arm64 decision: `{}`\n".format( outputs["prelint_arm64_decision_reason"])) + fh.write( + "- Remote ccache writes: {}\n".format( + "yes" if outputs["cache_write"] == "true" else "no" + ) + ) def env_int(name: str, default: int) -> int: diff --git a/.github/workflows/test_select_dynamic_runner.py b/.github/workflows/test_select_dynamic_runner.py index 3d67a4e5af29..5d71db24e6a7 100644 --- a/.github/workflows/test_select_dynamic_runner.py +++ b/.github/workflows/test_select_dynamic_runner.py @@ -523,6 +523,53 @@ def test_lint_inherits_blacksmith_when_rung_unavailable(self): self.assertEqual(outputs["runner_amd64"], "blacksmith-amd64") self.assertEqual(outputs["runner_prelint"], "blacksmith-amd64") + # --- remote ccache write gate -------------------------------------------- + + def test_cache_write_on_any_push(self): + # A push only ever sees its own repository's secret, and pushing there + # already requires write access, so even an unlisted actor qualifies. + for actor in ("knst", "mallory"): + with self.subTest(actor=actor): + outputs = self._select(event_name="push", event={}, actor=actor) + self.assertEqual(outputs["cache_write"], "true") + + def test_cache_write_for_trusted_pull_requests(self): + for head_owner in ("PastaPastaPasta", "dashpay"): + with self.subTest(head_owner=head_owner): + outputs = self._select( + event=pull_request_event("PastaPastaPasta", head_owner=head_owner), + ) + self.assertEqual(outputs["cache_write"], "true") + + def test_no_cache_write_for_untrusted_pull_requests(self): + for author, actor, head_owner in ( + ("mallory", "PastaPastaPasta", "mallory"), + ("PastaPastaPasta", "mallory", "PastaPastaPasta"), + ("PastaPastaPasta", "PastaPastaPasta", "mallory"), + ): + with self.subTest(author=author, actor=actor, head_owner=head_owner): + outputs = self._select( + event=pull_request_event(author, head_owner=head_owner), + actor=actor, + ) + self.assertEqual(outputs["cache_write"], "false") + + def test_cache_write_does_not_depend_on_the_prelint_rung(self): + # The write gate shares the allowlist, not the self-hosted decision. + outputs = self._select(runner_selfhosted_var="", queued_jobs=0) + + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["cache_write"], "true") + + def test_no_cache_write_for_other_events(self): + for event_name in ("workflow_dispatch", "schedule", "pull_request"): + with self.subTest(event_name=event_name): + outputs = self._select( + event_name=event_name, + event=pull_request_event("PastaPastaPasta"), + ) + self.assertEqual(outputs["cache_write"], "false") + if __name__ == "__main__": unittest.main() From 7376c363e175473e843f4e4aa9041dbfc0a47453 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 7 Oct 2026 10:51:00 -0500 Subject: [PATCH 3/5] ci: use a shared remote ccache behind the GitHub cache Pull request runs cannot save to the GitHub Actions cache, and develop snapshots are often missing or evicted, so sampled PR builds hit only 2-35% of their objects in ccache even when the author's fork had built the same tree minutes earlier. Add a second-level ccache at https://cache.thepasta.org (bazel-remote, see PastaPastaPasta/dash-ci-cache). ccache checks the local cache restored from GitHub first and the remote only on a local miss. - Every run reads. Runs that received CCACHE_REMOTE_AUTH also write, through a netrc file, and reshare local hits the remote is missing. - A failed health check, CCACHE_REMOTE_DISABLE, or an image without the storage helper leaves the build on the local cache alone. The last case covers pull requests opened before the image change: pull_request_target runs this workflow from the base branch but builds the image from the head, and apt's ccache 4.9 rejects the new configuration outright. CCACHE_REMOTE_URL overrides the server. - The netrc is deleted when the build step ends, whether or not the compile succeeded, so unit tests never run binaries linked from cached objects while the credential is on disk. - Requests time out after 5s without data or 30s in total, so a slow server cannot stall a build for long. - The ccache stats, now including remote hits and misses, go to the job summary. Upstream and fork checkouts share the same absolute paths (/__w/dash/dash), so their objects are interchangeable without base_dir. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/build-src.yml | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/build-src.yml b/.github/workflows/build-src.yml index f44e2e25fcfe..0785595f0f2e 100644 --- a/.github/workflows/build-src.yml +++ b/.github/workflows/build-src.yml @@ -142,8 +142,50 @@ jobs: restore-keys: | ccache-${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'depends/packages/*') }}-${{ inputs.build-target }}- + - name: Configure remote ccache + # Second-level cache behind the GitHub one restored above. Every run + # reads from it; only runs given CCACHE_REMOTE_AUTH (see check-skip in + # build.yml) write. Any problem here leaves the build on the local + # cache alone rather than failing it. + if: vars.CCACHE_REMOTE_DISABLE == '' + env: + REMOTE_URL: ${{ vars.CCACHE_REMOTE_URL || 'https://cache.thepasta.org' }} + REMOTE_AUTH: ${{ secrets.CCACHE_REMOTE_AUTH }} + run: | + # Pull requests opened before the image gained ccache 4.14 still + # build with apt's 4.9, which rejects this configuration outright. + if ! command -v ccache-storage-https > /dev/null; then + echo "Remote ccache: off (this image has no HTTPS storage helper)" + exit 0 + fi + if ! curl -fsS --max-time 5 -o /dev/null "${REMOTE_URL}/status"; then + echo "::warning::Remote ccache: off (${REMOTE_URL} is unreachable)" + exit 0 + fi + mode="read-only" + options="read-only=true" + if [[ -n "${REMOTE_AUTH}" ]]; then + host="${REMOTE_URL#*://}" + host="${host%%/*}" + host="${host%%:*}" + netrc="${RUNNER_TEMP}/ccache-netrc" + (umask 077 && printf 'machine %s login %s password %s\n' \ + "${host}" "${REMOTE_AUTH%%:*}" "${REMOTE_AUTH#*:}" > "${netrc}") + mode="read-write" + options="@netrc-file=${netrc}" + # Also upload local hits the remote lacks, so it refills after a wipe. + echo "CCACHE_RESHARE=true" >> "${GITHUB_ENV}" + fi + echo "CCACHE_REMOTE_STORAGE=${REMOTE_URL} @layout=bazel data-timeout=5s request-timeout=30s ${options}" >> "${GITHUB_ENV}" + echo "Remote ccache: ${mode} (${REMOTE_URL})" + shell: bash + - name: Build source run: | + # Nothing after this step needs to write to the remote cache, and the + # tests run code built from cached objects; drop the credential even + # if the compile fails. + trap 'rm -f "${RUNNER_TEMP}/ccache-netrc"' EXIT CCACHE_MAXSIZE="600M" CACHE_DIR="/cache" mkdir /output @@ -151,6 +193,7 @@ jobs: BUILD_TARGET="${{ inputs.build-target }}" source ./ci/dash/matrix.sh ./ci/dash/build_src.sh + { echo "### ccache: ${BUILD_TARGET}"; echo '```'; ccache --show-stats; echo '```'; } >> "${GITHUB_STEP_SUMMARY}" ccache -X 9 ccache -c du -hd0 "${BASE_OUTDIR}" From 7d20f8e85d4d07edc25297c572826a9584085fe9 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 7 Oct 2026 10:52:01 -0500 Subject: [PATCH 4/5] ci: let develop builds finish and cancel only superseded tests Develop pushes shared one run-wide concurrency group with cancel-in-progress, so every merge killed the previous run. 55 of 81 develop runs between 2026-09-22 and 2026-10-06 were cancelled, most before their builds reached the ccache and depends cache saves. The next run, and every pull request based on it, then started cold. Give each develop run its own run-wide group, so nothing cancels or queues behind an older run, and every build finishes and saves its caches. With the shared remote ccache those builds are cheap. Tests are the expensive part of a stale run, so they alone can be cancelled. Concurrency groups order jobs by when they queue, not by commit age, so a delayed older run could otherwise cancel the newest run's tests. The last step of Build source and of Run linters therefore asks the remote where develop's tip is and records this run as current or superseded: - superseded (by either check): the run's tests are skipped; - current (by both checks): the tests join a develop-wide group per target with cancel-in-progress, so a newer run's tests replace them; - unknown, because a lookup failed: the tests still run, but in a group of their own run, so they cannot cancel anything. A newer run's test cannot queue before its own build finishes, which is after the older run's check already saw the newer tip, so a verified older run never reaches the shared group late. On every other ref the test group includes the run id and the check does not run, so pull requests and branches behave exactly as before. Re-running the tests of an older develop run still enters the shared group and cancels the newest run's test for that target; re-run the newest run instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/build-src.yml | 19 +++++++++++++++++++ .github/workflows/build.yml | 19 +++++++++++++++++++ .github/workflows/lint.yml | 21 +++++++++++++++++++++ .github/workflows/test-src.yml | 9 +++++++++ 4 files changed, 68 insertions(+) diff --git a/.github/workflows/build-src.yml b/.github/workflows/build-src.yml index 0785595f0f2e..58c9a03b300d 100644 --- a/.github/workflows/build-src.yml +++ b/.github/workflows/build-src.yml @@ -46,6 +46,9 @@ on: key: description: "Key needed for restoring artifacts bundle" value: ${{ jobs.build-src.outputs.key }} + freshness: + description: "current or superseded relative to develop's tip at the end of the build; empty if unknown" + value: ${{ jobs.build-src.outputs.freshness }} jobs: build-src: @@ -53,6 +56,7 @@ jobs: runs-on: ${{ inputs.runs-on }} outputs: key: ${{ steps.bundle.outputs.key }} + freshness: ${{ steps.freshness.outputs.freshness }} container: image: ${{ inputs.container-path }} # SYS_PTRACE is required by LeakSanitizer's stop-the-world @@ -266,3 +270,18 @@ jobs: compression-level: 0 overwrite: true retention-days: 3 + + - name: Check whether develop has moved on + id: freshness + if: github.event_name == 'push' && github.ref_name == github.event.repository.default_branch + run: | + # Left unset when the lookup fails: tests still run, but only a + # verified-current run may cancel another run's tests. + tip="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | cut -f1)" || true + if [[ "${tip}" == "${GITHUB_SHA}" ]]; then + echo "freshness=current" >> "${GITHUB_OUTPUT}" + elif [[ -n "${tip}" ]]; then + echo "freshness=superseded" >> "${GITHUB_OUTPUT}" + echo "Superseded by ${tip}: this run's tests will be skipped" + fi + shell: bash diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63627c38f438..f5153d1bfe17 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,11 +9,18 @@ permissions: contents: read packages: read +# Develop pushes get a group of their own, so a newer push neither cancels nor +# queues behind an older run: every build finishes and saves its caches. Only +# tests share a cancelling group (test-src.yml), and only when the run's build +# and lint both verified it was still develop's tip as they finished; a run +# found superseded skips its tests, and one whose check failed runs them in its +# own group. Otherwise a delayed older run could cancel the newest run's tests. concurrency: group: | ${{ github.workflow }}-${{ (vars.ALLOW_CONCURRENCY != '' && github.sha) || (github.event_name == 'pull_request_target' && github.event.pull_request.number) || + (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && github.run_id) || (github.event_name == 'push' && github.ref) }} cancel-in-progress: ${{ github.ref_type != 'tag' }} @@ -398,58 +405,70 @@ jobs: name: linux64-test uses: ./.github/workflows/test-src.yml needs: [check-skip, container-slim, src-linux64, lint] + if: needs.src-linux64.outputs.freshness != 'superseded' && needs.lint.outputs.freshness != 'superseded' with: bundle-key: ${{ needs.src-linux64.outputs.key }} build-target: linux64 container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + cancel-stale: ${{ needs.src-linux64.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_asan: name: linux64_asan-test uses: ./.github/workflows/test-src.yml needs: [check-skip, container-slim, src-linux64_asan, lint] + if: needs.src-linux64_asan.outputs.freshness != 'superseded' && needs.lint.outputs.freshness != 'superseded' with: bundle-key: ${{ needs.src-linux64_asan.outputs.key }} build-target: linux64_asan container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + cancel-stale: ${{ needs.src-linux64_asan.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_multiprocess: name: linux64_multiprocess-test uses: ./.github/workflows/test-src.yml needs: [check-skip, container-slim, src-linux64_multiprocess, lint] + if: needs.src-linux64_multiprocess.outputs.freshness != 'superseded' && needs.lint.outputs.freshness != 'superseded' with: bundle-key: ${{ needs.src-linux64_multiprocess.outputs.key }} build-target: linux64_multiprocess container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} + cancel-stale: ${{ needs.src-linux64_multiprocess.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_nowallet: name: linux64_nowallet-test uses: ./.github/workflows/test-src.yml needs: [check-skip, container-slim, src-linux64_nowallet, lint] + if: needs.src-linux64_nowallet.outputs.freshness != 'superseded' && needs.lint.outputs.freshness != 'superseded' with: bundle-key: ${{ needs.src-linux64_nowallet.outputs.key }} build-target: linux64_nowallet container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + cancel-stale: ${{ needs.src-linux64_nowallet.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_sqlite: name: linux64_sqlite-test uses: ./.github/workflows/test-src.yml needs: [check-skip, container-slim, src-linux64_sqlite, lint] + if: needs.src-linux64_sqlite.outputs.freshness != 'superseded' && needs.lint.outputs.freshness != 'superseded' with: bundle-key: ${{ needs.src-linux64_sqlite.outputs.key }} build-target: linux64_sqlite container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }} + cancel-stale: ${{ needs.src-linux64_sqlite.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} test-linux64_tsan: name: linux64_tsan-test uses: ./.github/workflows/test-src.yml needs: [check-skip, container-slim, src-linux64_tsan, lint] + if: needs.src-linux64_tsan.outputs.freshness != 'superseded' && needs.lint.outputs.freshness != 'superseded' with: bundle-key: ${{ needs.src-linux64_tsan.outputs.key }} build-target: linux64_tsan container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} + cancel-stale: ${{ needs.src-linux64_tsan.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 982736169621..2b859d86847f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,11 +11,17 @@ on: description: "Runner label to use" required: true type: string + outputs: + freshness: + description: "current or superseded relative to develop's tip at the end of lint; empty if unknown" + value: ${{ jobs.lint.outputs.freshness }} jobs: lint: name: Run linters runs-on: ${{ inputs.runs-on }} + outputs: + freshness: ${{ steps.freshness.outputs.freshness }} container: image: ${{ inputs.container-path }} options: --user root @@ -82,3 +88,18 @@ jobs: with: path: /cache/cppcheck key: cppcheck-${{ hashFiles('contrib/containers/ci/ci-slim.Dockerfile') }}-${{ github.sha }} + + - name: Check whether develop has moved on + id: freshness + if: github.event_name == 'push' && github.ref_name == github.event.repository.default_branch + run: | + # Left unset when the lookup fails: tests still run, but only a + # verified-current run may cancel another run's tests. + tip="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | cut -f1)" || true + if [[ "${tip}" == "${GITHUB_SHA}" ]]; then + echo "freshness=current" >> "${GITHUB_OUTPUT}" + elif [[ -n "${tip}" ]]; then + echo "freshness=superseded" >> "${GITHUB_OUTPUT}" + echo "Superseded by ${tip}: this run's tests will be skipped" + fi + shell: bash diff --git a/.github/workflows/test-src.yml b/.github/workflows/test-src.yml index 795a665f03e4..347d5e0aebc2 100644 --- a/.github/workflows/test-src.yml +++ b/.github/workflows/test-src.yml @@ -19,6 +19,11 @@ on: description: "Runner label to use (e.g., ubuntu-24.04 or ubuntu-24.04-arm)" required: true type: string + cancel-stale: + description: "Share a develop-wide group so a newer run's tests cancel these; only for runs verified current" + required: false + type: boolean + default: false env: INTEGRATION_TESTS_ARGS: "--extended --exclude feature_pruning,feature_dbcrash" @@ -28,6 +33,10 @@ jobs: test-src: name: Test source runs-on: ${{ inputs.runs-on }} + # A newer develop run's tests cancel these; see the concurrency comment in build.yml. + concurrency: + group: ${{ github.workflow }}-${{ inputs.cancel-stale && github.ref || github.run_id }}-test-src-${{ inputs.build-target }} + cancel-in-progress: true container: image: ${{ inputs.container-path }} # SYS_PTRACE is required by LeakSanitizer's stop-the-world From 7e34b1ff5f9c7c5c4e4a6c3195c1794a9a6e2590 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 7 Oct 2026 10:52:39 -0500 Subject: [PATCH 5/5] ci: keep one ccache generation per target in the GitHub cache Each develop push saves a ccache entry (and a ctcache entry for linux64_multiprocess) keyed by commit, and nothing deleted the older ones. At roughly 3.7 GB per push the repository's 10 GB cache stayed full, so GitHub evicted entries within hours and pull requests often found no develop snapshot to restore. After the src jobs of a develop push, delete every ccache-/ctcache- entry on that ref except the newest per target. The job runs no checked-out code, needs actions: write only for itself, and never runs for pull requests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/build.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5153d1bfe17..bf14e4dc9106 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -472,3 +472,35 @@ jobs: container-path: ${{ needs.container-slim.outputs.path }} runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} cancel-stale: ${{ needs.src-linux64_tsan.outputs.freshness == 'current' && needs.lint.outputs.freshness == 'current' }} + + prune-ccache: + name: Prune old ccache generations + # Each develop push saves a new ccache/ctcache entry per target; keep only + # the newest so the 10 GB repository cache stops evicting fresh ones. + needs: [src-aarch64-linux, src-linux64, src-linux64_asan, src-linux64_fuzz, src-linux64_multiprocess, src-linux64_nowallet, src-linux64_sqlite, src-linux64_tsan, src-mac, src-win64] + if: | + !cancelled() && + github.event_name == 'push' && + github.ref_name == github.event.repository.default_branch + runs-on: ubuntu-24.04-arm + permissions: + actions: write + steps: + - name: Delete all but the newest entry per target + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + REF: ${{ github.ref }} + run: | + for prefix in ccache ctcache; do + gh cache list --ref "${REF}" --key "${prefix}-" --sort created_at --order desc \ + --limit 1000 --json id,key \ + | jq -r --arg p "${prefix}" ' + map(.target = (.key | capture("^" + $p + "-[0-9a-f]{64}-(?.+)-[0-9a-f]{40}$").t)) + | group_by(.target)[][1:][] | "\(.id) \(.key)"' \ + | while read -r id key; do + echo "Deleting ${key}" + # A concurrent prune may have deleted it already. + gh cache delete "${id}" || true + done + done