diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index fe2724361d70..693bcf815d7f 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -23,14 +23,48 @@ on: description: "Runner label to use for arm64 jobs" required: true type: string + runs-on-manifest: + description: "Runner label for the job that combines the amd64 and arm64 builds into one multi-arch manifest" + required: true + type: string + existing-image: + description: "Digest reference (ghcr.io/...@sha256:...) of a trusted container already built from this exact context. When set, nothing is built and this container is used." + required: false + type: string + default: "" + runs-on-existing: + description: "Runner label for the one-step job that runs instead of the build when existing-image is set" + required: true + type: string + context-tag: + description: "Tag naming this build context (ctx-...). Trusted (push) builds add it to the container so later runs can find it as existing-image." + required: false + type: string + default: "" outputs: path: - description: "Digest reference to the container built by this run" - value: ${{ jobs.create-manifest.outputs.path }} + description: "Digest reference to the container this run uses: existing-image, or the one it just built" + value: ${{ inputs.existing-image || jobs.create-manifest.outputs.path }} jobs: + use-existing: + # Runs instead of the build jobs. A called workflow must run at least one + # job: if all of them are skipped, the caller counts as skipped, and so + # does every job that needs it (lint, depends, src, test). + name: Use existing container + if: ${{ inputs.existing-image != '' }} + runs-on: ${{ inputs.runs-on-existing }} + steps: + - name: Report existing container + env: + NAME: ${{ inputs.name }} + EXISTING_IMAGE: ${{ inputs.existing-image }} + run: | + echo "::notice title=${NAME}::Using existing container ${EXISTING_IMAGE}, build skipped" + build-amd64: name: Build container (amd64) + if: ${{ inputs.existing-image == '' }} runs-on: ${{ inputs.runs-on-amd64 }} outputs: tag: ${{ steps.prepare.outputs.tag }} @@ -79,6 +113,7 @@ jobs: build-arm64: name: Build container (arm64) + if: ${{ inputs.existing-image == '' }} runs-on: ${{ inputs.runs-on-arm64 }} outputs: digest: ${{ steps.build.outputs.digest }} @@ -125,7 +160,8 @@ jobs: create-manifest: name: Create multi-arch manifest - runs-on: ${{ inputs.runs-on-arm64 }} + if: ${{ inputs.existing-image == '' }} + runs-on: ${{ inputs.runs-on-manifest }} needs: [build-amd64, build-arm64] outputs: path: ${{ steps.manifest.outputs.path }} @@ -151,6 +187,7 @@ jobs: id: manifest env: TRUSTED: ${{ github.event_name == 'push' }} + CONTEXT_TAG: ${{ inputs.context-tag }} run: | REPO="ghcr.io/${{ needs.build-amd64.outputs.repo }}/${{ inputs.name }}" TAG="${{ needs.build-amd64.outputs.tag }}" @@ -158,6 +195,10 @@ jobs: if [[ "$TRUSTED" == "true" ]]; then TAGS=(-t "${REPO}:${HASH_TAG}" -t "${REPO}:${TAG}" -t "${REPO}:latest") + # Lets later runs with the same context use this container (resolve_ci_images.py) + if [[ -n "$CONTEXT_TAG" ]]; then + TAGS+=(-t "${REPO}:${CONTEXT_TAG}") + fi else TAGS=(-t "${REPO}:untrusted") fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 616d127834ea..cf3884775436 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,13 +27,18 @@ jobs: skip: ${{ steps.skip-check.outputs.skip }} runner-amd64: ${{ steps.select-runner.outputs.runner_amd64 }} runner-arm64: ${{ steps.select-runner.outputs.runner_arm64 }} - runner-lint: ${{ steps.select-runner.outputs.runner_lint }} + runner-prelint: ${{ steps.select-runner.outputs.runner_prelint }} + runner-prelint-arm64: ${{ steps.select-runner.outputs.runner_prelint_arm64 }} use-blacksmith: ${{ steps.select-runner.outputs.use_blacksmith }} use-blacksmith-amd64: ${{ steps.select-runner.outputs.use_blacksmith_amd64 }} use-blacksmith-arm64: ${{ steps.select-runner.outputs.use_blacksmith_arm64 }} backlog-count: ${{ steps.select-runner.outputs.backlog_count }} decision-reason: ${{ steps.select-runner.outputs.decision_reason }} base-image-digest: ${{ steps.base-image.outputs.digest }} + runner-check-skip: ${{ vars.RUNNER_CHECK_SKIP || 'ubuntu-24.04-arm' }} + ci-context-tag: ${{ steps.ci-images.outputs.context_tag }} + existing-runner-image: ${{ steps.ci-images.outputs.runner_image }} + existing-slim-image: ${{ steps.ci-images.outputs.slim_image }} steps: - name: Check skip environment variables id: skip-check @@ -61,10 +66,17 @@ jobs: GH_TOKEN: ${{ github.token }} RUNNER_AMD64_VAR: ${{ vars.RUNNER_AMD64 }} RUNNER_ARM64_VAR: ${{ vars.RUNNER_ARM64 }} - # Self-hosted rung for the lint job only. Unset RUNNER_SELFHOSTED (or - # rename it) to send lint straight back to the hosted/Blacksmith ladder. + # Self-hosted rung for the pre-lint jobs (lint, cache-sources, the + # container builds and manifests). All but the manifest run pull + # request head code, so all of them are offered our own hardware only + # for the logins in SELFHOSTED_LINT_AUTHORS. Unset RUNNER_SELFHOSTED + # or RUNNER_SELFHOSTED_ARM64 to send that arch back to the + # hosted/Blacksmith ladder; the long builds and tests never use it. RUNNER_SELFHOSTED_VAR: ${{ vars.RUNNER_SELFHOSTED }} + RUNNER_SELFHOSTED_ARM64_VAR: ${{ vars.RUNNER_SELFHOSTED_ARM64 }} SELFHOSTED_LINT_AUTHORS: ${{ vars.SELFHOSTED_LINT_AUTHORS }} + # Fall back to hosted once more jobs than this wait for a self-hosted label + SELFHOSTED_MAX_QUEUED: "6" run: | python3 .github/workflows/select_dynamic_runner.py @@ -88,14 +100,30 @@ jobs: fi echo "digest=${DIGEST:-unknown}" >> "$GITHUB_OUTPUT" + - name: Find existing CI containers + id: ci-images + if: ${{ steps.skip-check.outputs.skip == 'false' }} + # Reads the commit under test only through the API, never checks it + # out. A failure here only means the containers get built. + continue-on-error: true + timeout-minutes: 2 + env: + BASE_IMAGE_DIGEST: ${{ steps.base-image.outputs.digest }} + GH_TOKEN: ${{ github.token }} + SKIP_CI_IMAGE_REUSE: ${{ vars.SKIP_CI_IMAGE_REUSE }} + run: | + python3 .github/workflows/resolve_ci_images.py + cache-sources: name: Cache depends sources needs: [check-skip] if: ${{ needs.check-skip.outputs.skip == 'false' }} uses: ./.github/workflows/cache-depends-sources.yml with: - runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} + runs-on: ${{ needs.check-skip.outputs['runner-prelint'] }} + # Pushes build the trusted-* images that later runs reuse, so they stay on + # GitHub's runners; only pull request builds use the pre-lint runners. container: permissions: contents: read @@ -108,8 +136,12 @@ jobs: context: ./contrib/containers/ci file: ./contrib/containers/ci/ci.Dockerfile name: dashcore-ci-runner - runs-on-amd64: ${{ needs.check-skip.outputs['runner-amd64'] }} - runs-on-arm64: ${{ needs.check-skip.outputs['runner-arm64'] }} + runs-on-amd64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-amd64'] || needs.check-skip.outputs['runner-prelint'] }} + runs-on-arm64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint-arm64'] }} + runs-on-manifest: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint'] }} + existing-image: ${{ needs.check-skip.outputs['existing-runner-image'] }} + runs-on-existing: ${{ needs.check-skip.outputs['runner-check-skip'] }} + context-tag: ${{ needs.check-skip.outputs['ci-context-tag'] }} container-slim: permissions: @@ -123,8 +155,12 @@ jobs: context: ./contrib/containers/ci file: ./contrib/containers/ci/ci-slim.Dockerfile name: dashcore-ci-slim - runs-on-amd64: ${{ needs.check-skip.outputs['runner-amd64'] }} - runs-on-arm64: ${{ needs.check-skip.outputs['runner-arm64'] }} + runs-on-amd64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-amd64'] || needs.check-skip.outputs['runner-prelint'] }} + runs-on-arm64: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint-arm64'] }} + runs-on-manifest: ${{ github.event_name == 'push' && needs.check-skip.outputs['runner-arm64'] || needs.check-skip.outputs['runner-prelint'] }} + existing-image: ${{ needs.check-skip.outputs['existing-slim-image'] }} + runs-on-existing: ${{ needs.check-skip.outputs['runner-check-skip'] }} + context-tag: ${{ needs.check-skip.outputs['ci-context-tag'] }} depends-aarch64-linux: name: aarch64-linux-gnu @@ -204,7 +240,7 @@ jobs: needs: [check-skip, container-slim] with: container-path: ${{ needs.container-slim.outputs.path }} - runs-on: ${{ needs.check-skip.outputs['runner-lint'] }} + runs-on: ${{ needs.check-skip.outputs['runner-prelint'] }} src-aarch64-linux: name: aarch64-linux-build diff --git a/.github/workflows/cache-depends-sources.yml b/.github/workflows/cache-depends-sources.yml index 122851d8cd72..d4eba66056df 100644 --- a/.github/workflows/cache-depends-sources.yml +++ b/.github/workflows/cache-depends-sources.yml @@ -17,6 +17,8 @@ jobs: name: Cache depends sources # Intentionally keep scheduled cache warming on GitHub-hosted ARM runners. # Blacksmith caches are expected to persist long enough without a warmup cron. + # build.yml passes its pre-lint runner, which may be self-hosted; the cache + # key does not name the runner, so any arch can fill it. runs-on: ${{ inputs.runs-on || 'ubuntu-24.04-arm' }} steps: - name: Checkout code diff --git a/.github/workflows/resolve_ci_images.py b/.github/workflows/resolve_ci_images.py new file mode 100644 index 000000000000..d93f3301451a --- /dev/null +++ b/.github/workflows/resolve_ci_images.py @@ -0,0 +1,326 @@ +# Copyright (c) 2026 The Dash Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +"""Find trusted CI container images that this run can reuse instead of rebuilding. + +Runs in build.yml's check-skip job, which only checks out base branch code, +even under pull_request_target. The commit under test is read only as data +through the GitHub API; nothing from it is checked out or executed here. + +The reuse key covers everything that decides what gets built: the git tree id +of contrib/containers/ci at the commit under test, the ubuntu:noble manifest +digest, and the workflows in this checkout that drive the build (the same +revision GitHub runs). Only trusted (push) builds tag images with that key, so +a hit is an image that a trusted run built the same way from the same context. +Any failure means "build as usual", never a failed run. +""" + +import argparse +import base64 +import hashlib +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from typing import Callable, Dict, Optional, Sequence, Tuple + + +CONTEXT_DIR = "contrib/containers/ci" +# Workflows that decide how the images are built from CONTEXT_DIR +RECIPE_FILES = ( + ".github/workflows/build.yml", + ".github/workflows/build-container.yml", +) +TAG_PREFIX = "ctx" +REGISTRY = "ghcr.io" +# (output key, image name) for every image built from CONTEXT_DIR +IMAGES = ( + ("runner", "dashcore-ci-runner"), + ("slim", "dashcore-ci-slim"), +) +INDEX_MEDIA_TYPES = ( + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +) +REQUIRED_PLATFORMS = frozenset({("linux", "amd64"), ("linux", "arm64")}) +REQUEST_TIMEOUT_SECONDS = 10 +# Commit and tree ids, SHA-1 or SHA-256 repositories +OBJECT_ID_RE = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") +DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") + + +class HttpError(Exception): + def __init__(self, status: int, url: str): + super().__init__("HTTP {} for {}".format(status, url)) + self.status = status + + +# (url, request headers) -> (lowercased response headers, body) +HttpGet = Callable[[str, Dict[str, str]], Tuple[Dict[str, str], bytes]] + + +def http_get(url: str, headers: Dict[str, str]) -> Tuple[Dict[str, str], bytes]: + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + return {k.lower(): v for k, v in response.headers.items()}, response.read() + except urllib.error.HTTPError as exc: + raise HttpError(exc.code, url) from exc + + +def recipe_digest(paths: Sequence[str]) -> str: + """Hash of the local files that drive the build.""" + digest = hashlib.sha256() + for path in paths: + with open(path, "rb") as fh: + content = fh.read() + digest.update("{}\0{}\0".format(path, len(content)).encode()) + digest.update(content) + return digest.hexdigest() + + +def commit_under_test(event_name: str, event: Dict, github_sha: str) -> str: + """The commit whose container context the build jobs would use.""" + if event_name == "pull_request_target": + # build-container.yml checks out the pull request head, not github.sha + # (which is the base branch tip for pull_request_target). + return ((event.get("pull_request") or {}).get("head") or {}).get("sha") or "" + return github_sha + + +def context_tree_id(get: HttpGet, api_url: str, repo: str, commit: str, token: str) -> str: + """Tree id of CONTEXT_DIR at commit, read from the API without a checkout.""" + if not OBJECT_ID_RE.match(commit): + raise ValueError("invalid commit id {!r}".format(commit)) + parent, name = CONTEXT_DIR.rsplit("/", 1) + url = "{}/repos/{}/contents/{}?ref={}".format(api_url, repo, parent, commit) + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "dash-ci-image-resolver", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = "Bearer {}".format(token) + _, body = get(url, headers) + for entry in json.loads(body): + if entry.get("name") == name and entry.get("type") == "dir": + tree = entry.get("sha") or "" + if not OBJECT_ID_RE.match(tree): + raise ValueError("invalid tree id {!r}".format(tree)) + return tree + raise LookupError("{} not found at {}".format(CONTEXT_DIR, commit)) + + +def registry_token(get: HttpGet, image_path: str, token: str, actor: str) -> str: + """Pull token for image_path, authenticated if possible, else anonymous.""" + url = "https://{}/token?{}".format(REGISTRY, urllib.parse.urlencode({ + "scope": "repository:{}:pull".format(image_path), + "service": REGISTRY, + })) + attempts = [] + if token: + credentials = "{}:{}".format(actor or "github-actions", token).encode() + attempts.append({"Authorization": "Basic " + base64.b64encode(credentials).decode()}) + attempts.append({}) + + last_error: Exception = LookupError("no registry token for {}".format(image_path)) + for headers in attempts: + try: + _, body = get(url, headers) + value = json.loads(body).get("token") + if value: + return value + except Exception as exc: # noqa: BLE001 + last_error = exc + raise last_error + + +def multi_arch_digest(get: HttpGet, image_path: str, tag: str, token: str) -> str: + """Digest of the multi-arch index tagged `tag`, after checking it is usable.""" + url = "https://{}/v2/{}/manifests/{}".format(REGISTRY, image_path, tag) + headers = { + "Accept": ", ".join(INDEX_MEDIA_TYPES), + "Authorization": "Bearer {}".format(token), + } + response_headers, body = get(url, headers) + + media_type = response_headers.get("content-type", "").split(";")[0].strip() + if media_type not in INDEX_MEDIA_TYPES: + raise ValueError("{} is {}, not a multi-arch index".format(tag, media_type or "untyped")) + + # Hash what we received rather than trusting the header, so the reference + # handed to the build jobs names exactly the index that was checked. + digest = "sha256:" + hashlib.sha256(body).hexdigest() + header_digest = response_headers.get("docker-content-digest", "") + if header_digest and header_digest != digest: + raise ValueError("{} digest mismatch: header {} body {}".format(tag, header_digest, digest)) + + platforms = set() + for manifest in json.loads(body).get("manifests", []): + platform = manifest.get("platform") or {} + platforms.add((str(platform.get("os")), str(platform.get("architecture")))) + missing = REQUIRED_PLATFORMS - platforms + if missing: + raise ValueError("{} lacks {}".format( + tag, ", ".join(sorted("/".join(p) for p in missing)))) + return digest + + +def resolve_images( + event_name: str, + event: Dict, + github_sha: str, + repo: str, + base_digest: str, + recipe: str, + get: HttpGet, + api_url: str = "https://api.github.com", + token: str = "", + actor: str = "", + disabled: bool = False, +) -> Dict[str, str]: + outputs = {"context_tag": "", "commit": "", "tree": ""} + for key, _ in IMAGES: + outputs["{}_image".format(key)] = "" + outputs["{}_reason".format(key)] = "" + + def build_all(reason: str) -> Dict[str, str]: + for key, _ in IMAGES: + outputs["{}_reason".format(key)] = reason + return outputs + + if not DIGEST_RE.match(base_digest): + return build_all("build: ubuntu:noble digest unknown ({!r})".format(base_digest)) + if not DIGEST_RE.match(recipe): + return build_all("build: cannot hash the build workflows") + + commit = commit_under_test(event_name, event, github_sha) + outputs["commit"] = commit + try: + tree = context_tree_id(get, api_url, repo, commit, token) + except Exception as exc: # noqa: BLE001 + return build_all("build: cannot read {} tree ({}: {})".format( + CONTEXT_DIR, type(exc).__name__, exc)) + outputs["tree"] = tree + + tag = "{}-{}-{}-{}".format(TAG_PREFIX, tree, base_digest[:12], recipe[:12]) + # Handed to the container builds even when nothing is reused: trusted + # builds publish it so later runs can find the image. + outputs["context_tag"] = tag + + if disabled: + return build_all("build: reuse disabled by SKIP_CI_IMAGE_REUSE") + + for key, name in IMAGES: + image_path = "{}/{}".format(repo.lower(), name) + try: + registry = registry_token(get, image_path, token, actor) + digest = multi_arch_digest(get, image_path, tag, registry) + except Exception as exc: # noqa: BLE001 + if isinstance(exc, HttpError) and exc.status == 404: + reason = "build: no trusted image tagged {}".format(tag) + else: + reason = "build: registry lookup failed ({}: {})".format(type(exc).__name__, exc) + outputs["{}_reason".format(key)] = reason + else: + outputs["{}_image".format(key)] = "{}/{}@{}".format(REGISTRY, image_path, digest) + outputs["{}_reason".format(key)] = "reuse: trusted image tagged {}".format(tag) + + return outputs + + +def write_github_output(path: Optional[str], outputs: Dict[str, str]) -> None: + if not path: + return + with open(path, "a", encoding="utf-8") as fh: + for key in ["context_tag"] + ["{}_image".format(key) for key, _ in IMAGES]: + fh.write("{}={}\n".format(key, outputs[key])) + + +def summary_lines(outputs: Dict[str, str], base_digest: str, recipe: str) -> Sequence[str]: + lines = [ + "### CI container images", + "- Context: `{}` tree `{}` at `{}`, ubuntu:noble `{}`, build workflows `{}`".format( + CONTEXT_DIR, outputs["tree"][:12] or "?", outputs["commit"][:12] or "?", + base_digest[:12] or "?", recipe[:12] or "?"), + "- Context tag: `{}`".format(outputs["context_tag"] or "none"), + ] + for key, name in IMAGES: + line = "- {}: {}".format(name, outputs["{}_reason".format(key)]) + path = outputs["{}_image".format(key)] + if path: + line += " `{}`".format(path) + lines.append(line) + return lines + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Find reusable trusted CI container images.") + parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME", "")) + parser.add_argument("--event-path", default=os.environ.get("GITHUB_EVENT_PATH", "")) + parser.add_argument("--sha", default=os.environ.get("GITHUB_SHA", "")) + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) + parser.add_argument("--api-url", default=os.environ.get("GITHUB_API_URL", "https://api.github.com")) + parser.add_argument("--actor", default=os.environ.get("GITHUB_ACTOR", "")) + parser.add_argument("--token", default=os.environ.get("GH_TOKEN", "")) + parser.add_argument( + "--base-image-digest", + default=os.environ.get("BASE_IMAGE_DIGEST", ""), + help="ubuntu:noble manifest digest without the sha256: prefix", + ) + parser.add_argument( + "--disable", + action="store_true", + default=os.environ.get("SKIP_CI_IMAGE_REUSE", "") != "", + help="Always build; still report the context tag for trusted builds to publish", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + event: Dict = {} + if args.event_path: + try: + with open(args.event_path, "r", encoding="utf-8") as fh: + event = json.load(fh) + except (OSError, ValueError) as exc: + print("warning: cannot read event payload: {}".format(exc), file=sys.stderr) + if not isinstance(event, dict): + event = {} + try: + recipe = recipe_digest(RECIPE_FILES) + except OSError as exc: + print("warning: cannot hash build workflows: {}".format(exc), file=sys.stderr) + recipe = "" + + outputs = resolve_images( + event_name=args.event_name, + event=event, + github_sha=args.sha, + repo=args.repo, + base_digest=args.base_image_digest, + recipe=recipe, + get=http_get, + api_url=args.api_url, + token=args.token, + actor=args.actor, + disabled=args.disable, + ) + + lines = summary_lines(outputs, args.base_image_digest, recipe) + print("\n".join(lines)) + write_github_output(os.environ.get("GITHUB_OUTPUT"), outputs) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/.github/workflows/select_dynamic_runner.py b/.github/workflows/select_dynamic_runner.py index 9f4a56569225..e1f4d2b70382 100644 --- a/.github/workflows/select_dynamic_runner.py +++ b/.github/workflows/select_dynamic_runner.py @@ -25,8 +25,12 @@ # Prefixes — any label starting with one of these indicates a non-GitHub-hosted runner NON_GITHUB_HOSTED_RUNNER_PREFIXES = ("blacksmith-",) SELF_HOSTED_LABEL = "self-hosted" -# Any queued GitHub-hosted job at all is enough to prefer our own hardware for lint. +# Any queued GitHub-hosted job at all is enough to prefer our own hardware for +# the pre-lint jobs. SELFHOSTED_BACKLOG_THRESHOLD = 0 +# Above this many queued jobs for a self-hosted label, our own runners are the +# bottleneck and the pre-lint jobs go back to the hosted ladder. +DEFAULT_SELFHOSTED_MAX_QUEUED = 6 def parse_next_link(link_header: str) -> Optional[str]: @@ -111,8 +115,14 @@ def count_queued_jobs( repos: Sequence[str], statuses: Sequence[str] = DEFAULT_STATUSES, non_hosted_labels: Sequence[str] = (), -) -> int: +) -> Tuple[int, Dict[str, int]]: + """Count queued jobs in one scan of repos. + + Returns the number of queued GitHub-hosted jobs, and for each label in + non_hosted_labels (lowercased) the number of queued jobs asking for it. + """ queued_jobs = 0 + queued_by_label = {label.lower(): 0 for label in non_hosted_labels if label} for repo in repos: run_ids = set() @@ -133,12 +143,20 @@ def count_queued_jobs( ).format(repo, run_id) for payload in iter_pages(fetch_json, jobs_url): for job in payload.get("jobs", []): - if job.get("status") == "queued" and targets_github_hosted_runner( - job, non_hosted_labels - ): + if job.get("status") != "queued": + continue + if targets_github_hosted_runner(job, non_hosted_labels): queued_jobs += 1 + labels = { + label.lower() + for label in job.get("labels", []) + if isinstance(label, str) + } + for label in queued_by_label: + if label in labels: + queued_by_label[label] += 1 - return queued_jobs + return queued_jobs, queued_by_label def load_event(event_path: str) -> Dict: @@ -216,7 +234,7 @@ def is_selfhosted_allowed( return False -def select_lint_runner( +def select_prelint_runner( event_name: str, event: Dict, actor: str, @@ -226,16 +244,28 @@ def select_lint_runner( allowed_authors: Set[str], fallback_runner: str, label_override: bool = False, + selfhosted_queued: int = 0, + selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED, + disabled_reason: str = "selfhosted-disabled", ) -> Tuple[str, str]: - """Pick the runner for the lint job, and the reason for the pick. + """Pick the runner for the pre-lint jobs of one arch, and the reason. + + The pre-lint jobs are the short ones that gate lint and the long builds: + lint itself, cache-sources and the container builds and manifest. They + take seconds but can wait most of an hour for a GitHub-hosted runner. Inserts a self-hosted rung into the hosted -> Blacksmith ladder: once any - GitHub-hosted job is queued, lint goes to our own hardware instead of - competing for the account-wide concurrency limit. When the rung is not - available lint keeps the caller's existing amd64 decision unchanged. + GitHub-hosted job is queued, these jobs go to our own hardware instead of + competing for the account-wide concurrency limit, unless our own runners + already have more than selfhosted_max_queued jobs waiting. When the rung + is not available they keep the caller's existing decision for the arch. + + Every job but the manifest runs pull request head code (lint scripts, + `make -C depends download`, Dockerfile RUN steps), so all of them go + through the same allowlist as lint did on its own. """ if not selfhosted_label: - return fallback_runner, "selfhosted-disabled" + return fallback_runner, disabled_reason if label_override: # Someone asked for Blacksmith explicitly; do not quietly send them to @@ -253,8 +283,14 @@ def select_lint_runner( if not is_selfhosted_allowed(event_name, event, actor, allowed_authors): return fallback_runner, "actor-not-allowed" - return selfhosted_label, "selfhosted:backlog:{}>{}".format( - backlog_count_value, SELFHOSTED_BACKLOG_THRESHOLD + if selfhosted_queued > selfhosted_max_queued: + return fallback_runner, "selfhosted-saturated:queued:{}>{}".format( + selfhosted_queued, selfhosted_max_queued + ) + + return selfhosted_label, "selfhosted:backlog:{}>{};queued:{}<={}".format( + backlog_count_value, SELFHOSTED_BACKLOG_THRESHOLD, + selfhosted_queued, selfhosted_max_queued, ) @@ -270,6 +306,8 @@ def select_runners( runner_selfhosted_var: str = "", selfhosted_authors: str = "", actor: str = "", + runner_selfhosted_arm64_var: str = "", + selfhosted_max_queued: int = DEFAULT_SELFHOSTED_MAX_QUEUED, ) -> Dict[str, str]: label_names = [ label.get("name", "") @@ -281,11 +319,14 @@ def select_runners( backlog_count = "unknown" backlog_count_value = None + selfhosted_queued: Dict[str, int] = {} measurement_error = None try: - backlog_count_value = count_queued_jobs( - fetch_json, repos, non_hosted_labels=(runner_selfhosted_var,) + backlog_count_value, selfhosted_queued = count_queued_jobs( + fetch_json, + repos, + non_hosted_labels=(runner_selfhosted_var, runner_selfhosted_arm64_var), ) backlog_count = str(backlog_count_value) except Exception as exc: # noqa: BLE001 @@ -340,23 +381,40 @@ def select_runners( decision_parts.append("error:{}".format(measurement_error[:180])) decision_parts.extend(fallback_parts) - runner_lint, lint_decision_reason = select_lint_runner( - event_name=event_name, - event=event, - actor=actor, - backlog_count_value=backlog_count_value, - measurement_error=measurement_error, - selfhosted_label=runner_selfhosted_var, - allowed_authors=parse_author_allowlist(selfhosted_authors), - fallback_runner=runner_amd64, - label_override=label_override, + allowed_authors = parse_author_allowlist(selfhosted_authors) + + def prelint( + selfhosted_label: str, fallback_runner: str, disabled_reason: str + ) -> Tuple[str, str]: + return select_prelint_runner( + event_name=event_name, + event=event, + actor=actor, + backlog_count_value=backlog_count_value, + measurement_error=measurement_error, + selfhosted_label=selfhosted_label, + allowed_authors=allowed_authors, + fallback_runner=fallback_runner, + label_override=label_override, + selfhosted_queued=selfhosted_queued.get(selfhosted_label.lower(), 0), + selfhosted_max_queued=selfhosted_max_queued, + disabled_reason=disabled_reason, + ) + + runner_prelint, prelint_decision_reason = prelint( + runner_selfhosted_var, runner_amd64, "selfhosted-disabled" + ) + runner_prelint_arm64, prelint_arm64_decision_reason = prelint( + runner_selfhosted_arm64_var, runner_arm64, "selfhosted-arm64-disabled" ) return { "runner_amd64": runner_amd64, "runner_arm64": runner_arm64, - "runner_lint": runner_lint, - "lint_decision_reason": lint_decision_reason, + "runner_prelint": runner_prelint, + "prelint_decision_reason": prelint_decision_reason, + "runner_prelint_arm64": runner_prelint_arm64, + "prelint_arm64_decision_reason": prelint_arm64_decision_reason, "use_blacksmith": "true" if use_blacksmith_amd64 or use_blacksmith_arm64 else "false", "use_blacksmith_amd64": "true" if use_blacksmith_amd64 else "false", "use_blacksmith_arm64": "true" if use_blacksmith_arm64 else "false", @@ -401,9 +459,12 @@ def write_step_summary(path: Optional[str], outputs: Dict[str, str]) -> None: ) fh.write("- amd64 runner: `{}`\n".format(outputs["runner_amd64"])) fh.write("- arm64 runner: `{}`\n".format(outputs["runner_arm64"])) - fh.write("- lint runner: `{}`\n".format(outputs["runner_lint"])) + fh.write("- pre-lint amd64 runner: `{}`\n".format(outputs["runner_prelint"])) + fh.write("- pre-lint arm64 runner: `{}`\n".format(outputs["runner_prelint_arm64"])) fh.write("- Decision: `{}`\n".format(outputs["decision_reason"])) - fh.write("- Lint decision: `{}`\n".format(outputs["lint_decision_reason"])) + fh.write("- Pre-lint amd64 decision: `{}`\n".format(outputs["prelint_decision_reason"])) + fh.write("- Pre-lint arm64 decision: `{}`\n".format( + outputs["prelint_arm64_decision_reason"])) def env_int(name: str, default: int) -> int: @@ -451,7 +512,18 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace: parser.add_argument( "--runner-selfhosted-var", default=os.environ.get("RUNNER_SELFHOSTED_VAR", ""), - help="Self-hosted runner label for the lint job; empty disables the rung", + help="Self-hosted amd64 runner label for the pre-lint jobs; empty disables the rung", + ) + parser.add_argument( + "--runner-selfhosted-arm64-var", + default=os.environ.get("RUNNER_SELFHOSTED_ARM64_VAR", ""), + help="Self-hosted arm64 runner label for the pre-lint jobs; empty disables the rung", + ) + parser.add_argument( + "--selfhosted-max-queued", + type=int, + default=env_int("SELFHOSTED_MAX_QUEUED", DEFAULT_SELFHOSTED_MAX_QUEUED), + help="Use a self-hosted label only while at most this many jobs are queued for it", ) parser.add_argument( "--selfhosted-authors", @@ -508,6 +580,8 @@ def main(argv: Sequence[str]) -> int: runner_selfhosted_var=args.runner_selfhosted_var, selfhosted_authors=args.selfhosted_authors, actor=args.actor, + runner_selfhosted_arm64_var=args.runner_selfhosted_arm64_var, + selfhosted_max_queued=args.selfhosted_max_queued, ) write_github_output(os.environ.get("GITHUB_OUTPUT"), outputs) diff --git a/.github/workflows/test_select_dynamic_runner.py b/.github/workflows/test_select_dynamic_runner.py index c300da3d4793..3d67a4e5af29 100644 --- a/.github/workflows/test_select_dynamic_runner.py +++ b/.github/workflows/test_select_dynamic_runner.py @@ -36,6 +36,14 @@ def backlog_responses(queued_jobs): } +def add_queued_jobs(responses, label, count): + """Add `count` queued jobs asking for `label` to backlog_responses() output.""" + jobs_url = next(url for url in responses if "/jobs?" in url) + payload, headers = responses[jobs_url] + payload["jobs"] = payload["jobs"] + [{"status": "queued", "labels": [label]}] * count + return responses + + def pull_request_event(author, head_owner=None, labels=()): """A pull_request_target payload, by default a fork PR owned by `author`.""" head_owner = author if head_owner is None else head_owner @@ -85,7 +93,7 @@ def test_count_queued_jobs_deduplicates_runs_across_status_queries(self): def fetch_json(url): return responses[url] - self.assertEqual(MODULE.count_queued_jobs(fetch_json, [repo]), 2) + self.assertEqual(MODULE.count_queued_jobs(fetch_json, [repo])[0], 2) def test_count_queued_jobs_excludes_blacksmith_jobs(self): repo = "dashpay/dash" @@ -118,7 +126,7 @@ def test_count_queued_jobs_excludes_blacksmith_jobs(self): def fetch_json(url): return responses[url] - self.assertEqual(MODULE.count_queued_jobs(fetch_json, [repo]), 1) + self.assertEqual(MODULE.count_queued_jobs(fetch_json, [repo])[0], 1) def test_label_override_selects_blacksmith_even_with_low_backlog(self): outputs = MODULE.select_runners( @@ -254,7 +262,7 @@ def test_missing_runner_vars_fall_back_per_arch(self): self.assertEqual(outputs["runner_arm64"], "blacksmith-arm64") self.assertIn("amd64-github-fallback", outputs["decision_reason"]) - # --- lint runner selection (self-hosted rung) --------------------------- + # --- pre-lint runner selection (self-hosted rung) ----------------------- def _select(self, **overrides): kwargs = dict( @@ -279,10 +287,33 @@ def _select(self, **overrides): def test_lint_uses_selfhosted_when_backlog_and_author_allowed(self): outputs = self._select() - self.assertEqual(outputs["runner_lint"], "ubuntu-core") - self.assertEqual(outputs["lint_decision_reason"], "selfhosted:backlog:3>0") + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + self.assertEqual( + outputs["prelint_decision_reason"], "selfhosted:backlog:3>0;queued:0<=6" + ) # The rest of the ladder is untouched. self.assertEqual(outputs["runner_amd64"], MODULE.DEFAULT_RUNNER_AMD64) + # No arm64 label configured: arm64 pre-lint jobs stay on runner_arm64. + self.assertEqual(outputs["runner_prelint_arm64"], MODULE.DEFAULT_RUNNER_ARM64) + self.assertEqual( + outputs["prelint_arm64_decision_reason"], "selfhosted-arm64-disabled" + ) + + def test_prelint_arm64_uses_its_own_label(self): + outputs = self._select(runner_selfhosted_arm64_var="ubuntu-core-arm64") + + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + self.assertEqual(outputs["runner_prelint_arm64"], "ubuntu-core-arm64") + + def test_prelint_arm64_follows_the_same_allowlist(self): + outputs = self._select( + runner_selfhosted_arm64_var="ubuntu-core-arm64", + event=pull_request_event("mallory"), + actor="mallory", + ) + + self.assertEqual(outputs["runner_prelint_arm64"], MODULE.DEFAULT_RUNNER_ARM64) + self.assertEqual(outputs["prelint_arm64_decision_reason"], "actor-not-allowed") def test_lint_allowlist_is_case_insensitive(self): outputs = self._select( @@ -291,7 +322,7 @@ def test_lint_allowlist_is_case_insensitive(self): selfhosted_authors="pastapastapasta, KWVG , udjinm6", ) - self.assertEqual(outputs["runner_lint"], "ubuntu-core") + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") def test_lint_falls_back_when_author_not_allowed(self): # An allowlisted maintainer may not lend our hardware to a fork PR @@ -301,8 +332,8 @@ def test_lint_falls_back_when_author_not_allowed(self): actor="PastaPastaPasta", ) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_falls_back_when_actor_not_allowed_even_if_author_is(self): # A fork branch can be pushed to by someone other than the PR author. @@ -311,26 +342,26 @@ def test_lint_falls_back_when_actor_not_allowed_even_if_author_is(self): actor="mallory", ) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_falls_back_without_backlog(self): outputs = self._select(queued_jobs=0) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "backlog:0<=0") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "backlog:0<=0") def test_lint_falls_back_when_selfhosted_label_unset(self): outputs = self._select(runner_selfhosted_var="") - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "selfhosted-disabled") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "selfhosted-disabled") def test_lint_falls_back_when_allowlist_empty(self): outputs = self._select(selfhosted_authors="") - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_falls_back_on_measurement_error(self): def fetch_json(_url): @@ -338,19 +369,19 @@ def fetch_json(_url): outputs = self._select(fetch_json=fetch_json) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "metric-unavailable") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "metric-unavailable") def test_lint_uses_selfhosted_on_push_from_allowed_actor(self): outputs = self._select(event_name="push", event={}, actor="knst") - self.assertEqual(outputs["runner_lint"], "ubuntu-core") + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") def test_lint_ignores_unknown_event_types(self): outputs = self._select(event_name="workflow_dispatch", event={}) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_falls_back_for_cross_fork_head(self): # A pull request may be opened from any readable fork, so an allowlisted @@ -359,15 +390,15 @@ def test_lint_falls_back_for_cross_fork_head(self): event=pull_request_event("PastaPastaPasta", head_owner="mallory"), ) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_allows_same_repo_head(self): outputs = self._select( event=pull_request_event("PastaPastaPasta", head_owner="dashpay"), ) - self.assertEqual(outputs["runner_lint"], "ubuntu-core") + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") def test_lint_requires_an_exact_login_match(self): # "knstfoo" must not be accepted on the strength of "knst" being listed. @@ -376,8 +407,8 @@ def test_lint_requires_an_exact_login_match(self): actor="knstfoo", ) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_each_identity_is_matched_exactly(self): # Author, actor and head repository owner are three separate gates; @@ -417,8 +448,8 @@ def test_lint_falls_back_when_head_and_base_are_both_missing(self): outputs = self._select(event=event) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_falls_back_when_head_repo_missing(self): event = pull_request_event("PastaPastaPasta") @@ -426,16 +457,16 @@ def test_lint_falls_back_when_head_repo_missing(self): outputs = self._select(event=event) - self.assertEqual(outputs["runner_lint"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["lint_decision_reason"], "actor-not-allowed") + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual(outputs["prelint_decision_reason"], "actor-not-allowed") def test_lint_yields_to_an_explicit_blacksmith_label(self): outputs = self._select( event=pull_request_event("PastaPastaPasta", labels=["blacksmith-ci"]), ) - self.assertEqual(outputs["runner_lint"], "blacksmith-amd64") - self.assertEqual(outputs["lint_decision_reason"], "label:blacksmith-ci") + self.assertEqual(outputs["runner_prelint"], "blacksmith-amd64") + self.assertEqual(outputs["prelint_decision_reason"], "label:blacksmith-ci") def test_selfhosted_jobs_do_not_count_as_hosted_backlog(self): # Queued lint jobs on our own hardware must not inflate the figure that @@ -447,14 +478,8 @@ def test_selfhosted_jobs_do_not_count_as_hosted_backlog(self): self.assertFalse(MODULE.targets_github_hosted_runner(job, ["UBUNTU-CORE"])) self.assertTrue(MODULE.targets_github_hosted_runner(job, [""])) - def test_backlog_excludes_queued_selfhosted_lint_jobs(self): - responses = backlog_responses(3) - jobs_url = next(url for url in responses if "/jobs?" in url) - payload, headers = responses[jobs_url] - payload["jobs"] = payload["jobs"] + [ - {"status": "queued", "labels": ["ubuntu-core"]} - ] * 9 - responses[jobs_url] = (payload, headers) + def test_backlog_excludes_queued_selfhosted_jobs(self): + responses = add_queued_jobs(backlog_responses(3), "ubuntu-core", 9) outputs = self._select(fetch_json=lambda url: responses[url]) @@ -462,7 +487,29 @@ def test_backlog_excludes_queued_selfhosted_lint_jobs(self): # threshold of 10 is not crossed and Blacksmith is not paid for. self.assertEqual(outputs["backlog_count"], "3") self.assertEqual(outputs["runner_amd64"], MODULE.DEFAULT_RUNNER_AMD64) - self.assertEqual(outputs["runner_lint"], "ubuntu-core") + # The 9 waiting for our own runners exceed the cap of 6, so pre-lint + # goes back to the hosted ladder rather than queue behind them. + self.assertEqual(outputs["runner_prelint"], MODULE.DEFAULT_RUNNER_AMD64) + self.assertEqual( + outputs["prelint_decision_reason"], "selfhosted-saturated:queued:9>6" + ) + + def test_saturation_is_counted_per_label(self): + responses = add_queued_jobs(backlog_responses(3), "ubuntu-core-arm64", 7) + add_queued_jobs(responses, "ubuntu-core", 6) + + outputs = self._select( + runner_selfhosted_arm64_var="ubuntu-core-arm64", + fetch_json=lambda url: responses[url], + ) + + self.assertEqual(outputs["backlog_count"], "3") + # Exactly at the cap is still allowed. + self.assertEqual(outputs["runner_prelint"], "ubuntu-core") + self.assertEqual(outputs["runner_prelint_arm64"], MODULE.DEFAULT_RUNNER_ARM64) + self.assertEqual( + outputs["prelint_arm64_decision_reason"], "selfhosted-saturated:queued:7>6" + ) def test_lint_inherits_blacksmith_when_rung_unavailable(self): # Declining self-hosted must leave lint on whatever amd64 decided, @@ -474,7 +521,7 @@ def test_lint_inherits_blacksmith_when_rung_unavailable(self): ) self.assertEqual(outputs["runner_amd64"], "blacksmith-amd64") - self.assertEqual(outputs["runner_lint"], "blacksmith-amd64") + self.assertEqual(outputs["runner_prelint"], "blacksmith-amd64") if __name__ == "__main__":