ci: run pre-lint jobs on self-hosted runners when hosted runners are backed up

.github/workflows/build-container.yml

@@ -23,14 +23,48 @@ on:
         description: "Runner label to use for arm64 jobs"
         required: true
         type: string
+      runs-on-manifest:
+        description: "Runner label for the job that combines the amd64 and arm64 builds into one multi-arch manifest"
+        required: true
+        type: string
+      existing-image:
+        description: "Digest reference (ghcr.io/...@sha256:...) of a trusted container already built from this exact context. When set, nothing is built and this container is used."
+        required: false
+        type: string
+        default: ""
+      runs-on-existing:
+        description: "Runner label for the one-step job that runs instead of the build when existing-image is set"
+        required: true
+        type: string
+      context-tag:
+        description: "Tag naming this build context (ctx-...). Trusted (push) builds add it to the container so later runs can find it as existing-image."
+        required: false
+        type: string
+        default: ""
     outputs:
       path:
-        description: "Digest reference to the container built by this run"
-        value: ${{ jobs.create-manifest.outputs.path }}
+        description: "Digest reference to the container this run uses: existing-image, or the one it just built"
+        value: ${{ inputs.existing-image || jobs.create-manifest.outputs.path }}
 
 jobs:
+  use-existing:
+    # Runs instead of the build jobs. A called workflow must run at least one
+    # job: if all of them are skipped, the caller counts as skipped, and so
+    # does every job that needs it (lint, depends, src, test).
+    name: Use existing container
+    if: ${{ inputs.existing-image != '' }}
+    runs-on: ${{ inputs.runs-on-existing }}
+    steps:
+      - name: Report existing container
+        env:
+          NAME: ${{ inputs.name }}
+          EXISTING_IMAGE: ${{ inputs.existing-image }}
+        run: |
+          echo "::notice title=${NAME}::Using existing container ${EXISTING_IMAGE}, build skipped"
+
   build-amd64:
     name: Build container (amd64)
+    if: ${{ inputs.existing-image == '' }}
     runs-on: ${{ inputs.runs-on-amd64 }}
     outputs:
       tag: ${{ steps.prepare.outputs.tag }}
@@ -79,6 +113,7 @@ jobs:
 
   build-arm64:
     name: Build container (arm64)
+    if: ${{ inputs.existing-image == '' }}
     runs-on: ${{ inputs.runs-on-arm64 }}
     outputs:
       digest: ${{ steps.build.outputs.digest }}
@@ -125,7 +160,8 @@ jobs:
 
   create-manifest:
     name: Create multi-arch manifest
-    runs-on: ${{ inputs.runs-on-arm64 }}
+    if: ${{ inputs.existing-image == '' }}
+    runs-on: ${{ inputs.runs-on-manifest }}
     needs: [build-amd64, build-arm64]
     outputs:
       path: ${{ steps.manifest.outputs.path }}
@@ -151,13 +187,18 @@ jobs:
         id: manifest
         env:
           TRUSTED: ${{ github.event_name == 'push' }}
+          CONTEXT_TAG: ${{ inputs.context-tag }}
         run: |
           REPO="ghcr.io/${{ needs.build-amd64.outputs.repo }}/${{ inputs.name }}"
           TAG="${{ needs.build-amd64.outputs.tag }}"
           HASH_TAG="${{ hashFiles(inputs.file) }}"
 
           if [[ "$TRUSTED" == "true" ]]; then
             TAGS=(-t "${REPO}:${HASH_TAG}" -t "${REPO}:${TAG}" -t "${REPO}:latest")
+            # Lets later runs with the same context use this container (resolve_ci_images.py)
+            if [[ -n "$CONTEXT_TAG" ]]; then
+              TAGS+=(-t "${REPO}:${CONTEXT_TAG}")
+            fi
           else
             TAGS=(-t "${REPO}:untrusted")
           fi

.github/workflows/build.yml

@@ -27,13 +27,18 @@ jobs:
       skip: ${{ steps.skip-check.outputs.skip }}
       runner-amd64: ${{ steps.select-runner.outputs.runner_amd64 }}
       runner-arm64: ${{ steps.select-runner.outputs.runner_arm64 }}
-      runner-lint: ${{ steps.select-runner.outputs.runner_lint }}
+      runner-prelint: ${{ steps.select-runner.outputs.runner_prelint }}
+      runner-prelint-arm64: ${{ steps.select-runner.outputs.runner_prelint_arm64 }}
       use-blacksmith: ${{ steps.select-runner.outputs.use_blacksmith }}
       use-blacksmith-amd64: ${{ steps.select-runner.outputs.use_blacksmith_amd64 }}
       use-blacksmith-arm64: ${{ steps.select-runner.outputs.use_blacksmith_arm64 }}
       backlog-count: ${{ steps.select-runner.outputs.backlog_count }}
       decision-reason: ${{ steps.select-runner.outputs.decision_reason }}
       base-image-digest: ${{ steps.base-image.outputs.digest }}
+      runner-check-skip: ${{ vars.RUNNER_CHECK_SKIP || 'ubuntu-24.04-arm' }}
+      ci-context-tag: ${{ steps.ci-images.outputs.context_tag }}
+      existing-runner-image: ${{ steps.ci-images.outputs.runner_image }}
+      existing-slim-image: ${{ steps.ci-images.outputs.slim_image }}
     steps:
       - name: Check skip environment variables
         id: skip-check
@@ -61,10 +66,17 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           RUNNER_AMD64_VAR: ${{ vars.RUNNER_AMD64 }}
           RUNNER_ARM64_VAR: ${{ vars.RUNNER_ARM64 }}
-          # Self-hosted rung for the lint job only. Unset RUNNER_SELFHOSTED (or
-          # rename it) to send lint straight back to the hosted/Blacksmith ladder.
+          # Self-hosted rung for the pre-lint jobs (lint, cache-sources, the
+          # container builds and manifests). All but the manifest run pull
+          # request head code, so all of them are offered our own hardware only
+          # for the logins in SELFHOSTED_LINT_AUTHORS. Unset RUNNER_SELFHOSTED
+          # or RUNNER_SELFHOSTED_ARM64 to send that arch back to the
+          # hosted/Blacksmith ladder; the long builds and tests never use it.
           RUNNER_SELFHOSTED_VAR: ${{ vars.RUNNER_SELFHOSTED }}
+          RUNNER_SELFHOSTED_ARM64_VAR: ${{ vars.RUNNER_SELFHOSTED_ARM64 }}
           SELFHOSTED_LINT_AUTHORS: ${{ vars.SELFHOSTED_LINT_AUTHORS }}
+          # Fall back to hosted once more jobs than this wait for a self-hosted label
+          SELFHOSTED_MAX_QUEUED: "6"
         run: |
           python3 .github/workflows/select_dynamic_runner.py
 
@@ -88,14 +100,30 @@ jobs:
           fi
           echo "digest=${DIGEST:-unknown}" >> "$GITHUB_OUTPUT"
 
+      - name: Find existing CI containers
+        id: ci-images
+        if: ${{ steps.skip-check.outputs.skip == 'false' }}
+        # Reads the commit under test only through the API, never checks it
+        # out. A failure here only means the containers get built.
+        continue-on-error: true
+        timeout-minutes: 2
+        env:
+          BASE_IMAGE_DIGEST: ${{ steps.base-image.outputs.digest }}
+          GH_TOKEN: ${{ github.token }}
+          SKIP_CI_IMAGE_REUSE: ${{ vars.SKIP_CI_IMAGE_REUSE }}
+        run: |
+          python3 .github/workflows/resolve_ci_images.py
+
   cache-sources:
     name: Cache depends sources
     needs: [check-skip]
     if: ${{ needs.check-skip.outputs.skip == 'false' }}
     uses: ./.github/workflows/cache-depends-sources.yml
     with:
-      runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }}
+      runs-on: ${{ needs.check-skip.outputs['runner-prelint'] }}
 
+  # Pushes build the trusted-* images that later runs reuse, so they stay on
+  # GitHub's runners; only pull request builds use the pre-lint runners.
   container:
     permissions:
       contents: read
@@ -108,8 +136,12 @@ jobs:
       context: ./contrib/containers/ci
       file: ./contrib/containers/ci/ci.Dockerfile
       name: dashcore-ci-runner
-      runs-on-amd64: ${{ needs.check-skip.outputs['runner-amd64'] }}
-      runs-on-arm64: ${{ needs.check-skip.outputs['runner-arm64'] }}
+      runs-on-amd64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-amd64'] || needs.check-skip.outputs['runner-prelint'] }}
+      runs-on-arm64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint-arm64'] }}
+      runs-on-manifest: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint'] }}
+      existing-image: ${{ needs.check-skip.outputs['existing-runner-image'] }}
+      runs-on-existing: ${{ needs.check-skip.outputs['runner-check-skip'] }}
+      context-tag: ${{ needs.check-skip.outputs['ci-context-tag'] }}
 
   container-slim:
     permissions:
@@ -123,8 +155,12 @@ jobs:
       context: ./contrib/containers/ci
       file: ./contrib/containers/ci/ci-slim.Dockerfile
       name: dashcore-ci-slim
-      runs-on-amd64: ${{ needs.check-skip.outputs['runner-amd64'] }}
-      runs-on-arm64: ${{ needs.check-skip.outputs['runner-arm64'] }}
+      runs-on-amd64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-amd64'] || needs.check-skip.outputs['runner-prelint'] }}
+      runs-on-arm64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint-arm64'] }}
+      runs-on-manifest: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint'] }}
+      existing-image: ${{ needs.check-skip.outputs['existing-slim-image'] }}
+      runs-on-existing: ${{ needs.check-skip.outputs['runner-check-skip'] }}
+      context-tag: ${{ needs.check-skip.outputs['ci-context-tag'] }}
 
   depends-aarch64-linux:
     name: aarch64-linux-gnu
@@ -204,7 +240,7 @@ jobs:
     needs: [check-skip, container-slim]
     with:
       container-path: ${{ needs.container-slim.outputs.path }}
-      runs-on: ${{ needs.check-skip.outputs['runner-lint'] }}
+      runs-on: ${{ needs.check-skip.outputs['runner-prelint'] }}
 
   src-aarch64-linux:
     name: aarch64-linux-build

.github/workflows/cache-depends-sources.yml

@@ -17,6 +17,8 @@ jobs:
     name: Cache depends sources
     # Intentionally keep scheduled cache warming on GitHub-hosted ARM runners.
     # Blacksmith caches are expected to persist long enough without a warmup cron.
+    # build.yml passes its pre-lint runner, which may be self-hosted; the cache
+    # key does not name the runner, so any arch can fill it.
     runs-on: ${{ inputs.runs-on || 'ubuntu-24.04-arm' }}
     steps:
       - name: Checkout code