diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 2e5cf063b..808607762 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,9 @@ -# Labels of the self-hosted NPU runners, for actionlint's runs-on check. +# Labels of the self-hosted runners, for actionlint's runs-on check: the NPU +# runners, and the GPU machines in .github/hipep-machines.json. self-hosted-runner: labels: - flm - npu - sweep + - oga-hipep + - stx-halo diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index c4171faea..40538b87d 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -35,8 +35,11 @@ outputs: description: '"true" when the chat suite runs, the only suite the models-from-source and model-extension tests run.' value: ${{ steps.tier.outputs.source }} hipep: - description: '"true" when every suite runs or the change reaches the model Test Windows HIPEP tests.' + description: '"true" when every suite runs or the change reaches an oga-hipep model, which Test Windows HIPEP tests one per family.' value: ${{ steps.tier.outputs.hipep }} + hipep-models: + description: Comma-separated oga-hipep models whose catalog entries the change added or edited, which Test Windows HIPEP tests too. + value: ${{ steps.changes.outputs.hipep_models }} tier: description: install, pr or full. value: ${{ steps.tier.outputs.tier }} diff --git a/.github/actions/npu-test/action.yml b/.github/actions/npu-test/action.yml index e0de8c6e4..5365fe564 100644 --- a/.github/actions/npu-test/action.yml +++ b/.github/actions/npu-test/action.yml @@ -29,9 +29,17 @@ inputs: required: false default: "" backend: - description: A GPU backend (oga-hipep) whose models to test instead of the NPU models. + description: A GPU backend (oga-hipep) whose models to test instead of the NPU models, one per family (every one on the full tier). required: false default: "" + changed-models: + description: With backend, comma-separated models whose catalog entries the change touched, which the pr tier tests too. + required: false + default: "" + max-model-gb: + description: Skip the models larger than this many GB (catalog size), which the runner's memory cannot hold; 0 for no limit. + required: false + default: "0" model-extension: description: Optional model-extension manifest passed to FLM. required: false @@ -88,7 +96,10 @@ runs: # Models persist on the runner between jobs, so only the first job on a # runner pays for a download. The harness pulls missing models before the - # suites, within a time budget on the pr tier (--pull-budget). + # suites, within a time budget on the pr tier (--pull-budget). A GPU + # backend run pulls every model it tests, which verifies its files, except + # one a pull on this runner verified before whose catalog pins and files + # are unchanged (npu_test/model_cache.py). - name: Run tests (Linux) id: linux if: runner.os == 'Linux' @@ -102,6 +113,8 @@ runs: FAMILIES: ${{ inputs.families }} MODELS: ${{ inputs.models }} BACKEND: ${{ inputs.backend }} + CHANGED_MODELS: ${{ inputs.changed-models }} + MAX_MODEL_GB: ${{ inputs.max-model-gb || '0' }} MODEL_EXTENSION: ${{ inputs.model-extension }} ASR_FULL: ${{ inputs.asr-full }} ASR_KEY: ${{ inputs.asr-expected-key }} @@ -109,7 +122,8 @@ runs: export FLM_MODEL_PATH="${FLM_MODEL_PATH:-/scratch}" args=(--flm-bin "${FLM_BIN}" --label "${LABEL}" --tier "${TIER}" --suites "${SUITES}" --families "${FAMILIES}" --models "${MODELS}" --backend "${BACKEND}" - --asr-expected-key "${ASR_KEY}" + --changed-models "${CHANGED_MODELS}" + --max-model-gb "${MAX_MODEL_GB}" --asr-expected-key "${ASR_KEY}" --output-dir "${GITHUB_WORKSPACE}/npu-test-results/${LABEL}") [ "${ASR_FULL}" = "true" ] && args+=(--asr-full) [ -n "${MODEL_EXTENSION}" ] && args+=(--model-extension "${MODEL_EXTENSION}") @@ -130,6 +144,8 @@ runs: FAMILIES: ${{ inputs.families }} MODELS: ${{ inputs.models }} BACKEND: ${{ inputs.backend }} + CHANGED_MODELS: ${{ inputs.changed-models }} + MAX_MODEL_GB: ${{ inputs.max-model-gb || '0' }} MODEL_EXTENSION: ${{ inputs.model-extension }} ASR_FULL: ${{ inputs.asr-full }} ASR_KEY: ${{ inputs.asr-expected-key }} @@ -146,8 +162,9 @@ runs: $outDir = Join-Path $env:GITHUB_WORKSPACE "npu-test-results\$env:LABEL" $cmdArgs = @('-m', 'npu_test', "--flm-bin=$env:FLM_BIN", "--label=$env:LABEL", "--tier=$env:TIER", "--suites=$env:SUITES", "--families=$env:FAMILIES", - "--models=$env:MODELS", "--backend=$env:BACKEND", "--asr-expected-key=$env:ASR_KEY", - "--output-dir=$outDir") + "--models=$env:MODELS", "--backend=$env:BACKEND", + "--changed-models=$env:CHANGED_MODELS", "--max-model-gb=$env:MAX_MODEL_GB", + "--asr-expected-key=$env:ASR_KEY", "--output-dir=$outDir") if ($env:ASR_FULL -eq 'true') { $cmdArgs += '--asr-full' } if ($env:MODEL_EXTENSION) { $cmdArgs += "--model-extension=$env:MODEL_EXTENSION" diff --git a/.github/hipep-machines.json b/.github/hipep-machines.json new file mode 100644 index 000000000..d8d8ce1fe --- /dev/null +++ b/.github/hipep-machines.json @@ -0,0 +1,18 @@ +{ + "machines": [ + { + "id": "stx-halo", + "runs-on": ["self-hosted", "windows", "x64", "oga-hipep", "stx-halo"], + "enabled": true, + "max-model-gb": 0 + }, + { + "id": "npu-runners", + "runs-on": ["self-hosted", "windows", "x64", "npu", "sweep", "oga-hipep"], + "enabled": true, + "max-model-gb": 32, + "max-tier": "pr", + "changed-models": false + } + ] +} diff --git a/.github/script/ci/changes.py b/.github/script/ci/changes.py index 4bc7d24f6..f99bbd374 100644 --- a/.github/script/ci/changes.py +++ b/.github/script/ci/changes.py @@ -19,10 +19,14 @@ test of the shutdown suite, which no NPU job runs; a change to a model_list.json field outside its model entries, such as model_path, gives "all" - hipep true when the change can reach qwen3.5-oga-hipep:9b, which Test - Windows HIPEP runs on the GPU: when every suite runs, or a changed - file is oga-hipep code, a GPU catalog entry, the qwen3.5 frontend, - or a test of the chat, api and shutdown suites that job runs + hipep true when the change can reach an oga-hipep model, which Test + Windows HIPEP runs on the GPU (one per family): when every suite + runs, or a changed file is oga-hipep code, a GPU catalog entry, + the frontend of a family with a GPU entry, or a test of the chat, + api and shutdown suites that job runs + hipep_models + comma-separated oga-hipep models whose catalog entries were added + or edited, which Test Windows HIPEP tests too summary one line for the job log When in doubt it tests more: an all-zero or missing base (a new branch), or a @@ -142,17 +146,16 @@ # catalog entries whose effective backend is oga-hipep: their "backend" key, # or without one, a name ending in GPU_SUFFIX (model_list::backend_for_family). # No NPU model runs its code (bar the header above) or those entries, so they -# name no family and run no NPU job: Test Windows HIPEP tests them, on -# qwen3.5-oga-hipep:9b. +# name no family and run no NPU job: Test Windows HIPEP tests them, on every +# GPU entry. GPU_ONLY = re.compile(r"^(src/(common|include)/oga/|src/test/oga_hipep/)") GPU_SUFFIX = "-oga-hipep" # An entry's "backend" key (flm::backend::kBackendKey), when present, overrides # the suffix rule; GPU_BACKEND is the oga-hipep backend's id. GPU_BACKEND_KEY = "backend" GPU_BACKEND = "oga-hipep" -# The family of qwen3.5-oga-hipep:9b: a change to its frontend reaches that -# model too, one to its NPU engine does not. -GPU_FAMILIES = {"qwen3.5"} +# A family's frontend also drives its GPU entries, and its NPU engine does not: +# a change to the frontend of a family with one (gpu_families) reaches them. GPU_FRONTEND = re.compile(r"^src/(include|common)/AutoModel/") # The suites Test Windows HIPEP runs. GPU_SUITES = {"chat", "api", "shutdown"} @@ -274,8 +277,10 @@ class Decision: asr_full: bool = False # None: every suite. Otherwise the suites every changed file is scoped to. suites: Optional[set] = field(default_factory=set) - # A changed file reaches qwen3.5-oga-hipep:9b (every suite running does too). + # A changed file reaches an oga-hipep model (every suite running does too). hipep: bool = False + # The oga-hipep entries added or edited in the head catalog. + gpu_models: set = field(default_factory=set) reasons: list = field(default_factory=list) def all_suites(self, path: str) -> None: @@ -320,6 +325,7 @@ def outputs(self) -> dict[str, str]: "asr_full": str(self.asr_full).lower(), "suites": self.suites_value(), "hipep": str(self.runs_hipep()).lower(), + "hipep_models": ",".join(sorted(self.gpu_models)), "summary": self.summary()} @@ -350,6 +356,14 @@ def runs_on_gpu(model_id: str, entry: Optional[dict] = None) -> bool: return model_id.split(":", 1)[0].endswith(GPU_SUFFIX) +def gpu_families(catalog: dict[str, dict]) -> Optional[set]: + """The families with a GPU entry, whose frontends drive the oga-hipep + backend too; None without a catalog, when any frontend might be one.""" + if not catalog: + return None + return {family_of(e) for mid, e in catalog.items() if runs_on_gpu(mid, e)} - {""} + + def catalog_diff(base: dict[str, dict], head: dict[str, dict]) -> tuple[set, list]: """Families with an NPU entry added, removed or edited between two catalogs, and the GPU entries that were. An entry is a GPU one only if it is on @@ -408,7 +422,8 @@ def model_info_diff(base_text: Optional[str], head_text: Optional[str], def add_catalog_change(d: Decision, path: str, fams: set, gpu: list, catalog: dict[str, dict], neither: str) -> None: """A catalog file's changed entries: their NPU families in full, and Test - Windows HIPEP for GPU entries. A change to neither is not scoped.""" + Windows HIPEP for GPU entries, which also tests those the head catalog + still runs on the GPU. A change to neither is not scoped.""" d.families |= fams if fams: d.add_suites(family_suites(fams, catalog)) @@ -417,6 +432,7 @@ def add_catalog_change(d: Decision, path: str, fams: set, gpu: list, what = ", ".join(sorted(fams)) or ("no NPU family" if gpu else neither) if gpu: d.hipep = True + d.gpu_models |= {mid for mid in gpu if mid in catalog and runs_on_gpu(mid, catalog[mid])} what += (f"; {len(gpu)} oga-hipep entr{'y' if len(gpu) == 1 else 'ies'}, e.g. {gpu[0]}," " which only Test Windows HIPEP runs") d.reasons.append(f"{path}: entries changed in {what}") @@ -488,6 +504,7 @@ def decide(files: Optional[list[str]], git: Optional[Git] = None, d.reasons.append(f"{len(code)} of {len(files)} changed file(s) are code, e.g. {code[0]}") head_catalog = load_catalog(head_catalog_text) + gpu_fams = gpu_families(head_catalog) xclbin_families: dict[str, set] = {} for entry in head_catalog.values(): if entry.get("name"): @@ -508,8 +525,8 @@ def decide(files: Optional[list[str]], git: Optional[Git] = None, if new: d.families |= new d.reasons.append(f"{path}: {hit[0]} -> {', '.join(sorted(new))}") - if GPU_FAMILIES & set(hit[1]) and GPU_FRONTEND.match(path): - d.reach_hipep(f"{path}: {hit[0]} also runs qwen3.5-oga-hipep:9b," + if GPU_FRONTEND.match(path) and (gpu_fams is None or gpu_fams & set(hit[1])): + d.reach_hipep(f"{path}: {hit[0]} also runs oga-hipep models," " which Test Windows HIPEP tests") # Which suites this file can affect. diff --git a/.github/script/ci/hipep_machines.py b/.github/script/ci/hipep_machines.py new file mode 100644 index 000000000..ab86968ff --- /dev/null +++ b/.github/script/ci/hipep_machines.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Lists the machines Test Windows HIPEP runs on, as its job matrix. + + hipep_machines.py [--tier pr] [--changed-models a,b] [--only stx-halo,npu-runners] + [--github-output] + +.github/hipep-machines.json has one entry per kind of machine, each tested +by a job of its own: + + id short name (stx-halo, npu-runners), in the job's name and + in its report's label + runs-on the labels its runners register with. Several runners may + share them; a run takes whichever is free. With the NPU + test runners' own (npu, sweep) the job shares those + runners, holding one for its run as an NPU test job does; + a label only some of them carry (oga-hipep) picks those + enabled false leaves it out of every run: a kind with no runner + yet, or with all of them down (a job no runner takes waits + a day before it fails, and holds up the run's verdict) + max-model-gb the largest oga-hipep model, by its catalog size, that the + machine's memory holds; larger ones are skipped there. 0 + (the default) for no limit + max-tier the highest tier its job runs: install, pr or full (the + default). A run of a higher tier runs this one there, so + runners that cannot hold every model, or that the NPU + nightly needs, test the pr tier's models nightly + changed-models false leaves the models whose catalog entries the change + added or edited to the kinds that test them (true, the + default): a change to many entries downloads each new + revision within the job, which would hold a shared runner + for hours + +--tier is the run's tier; each machine in the matrix carries the tier its job +runs. --changed-models, the change's (changes.py's hipep-models), goes to the +machines that test them, and an empty list to the rest. --only, from the +FLM_HIPEP_MACHINES repository variable, runs the named kinds instead of the +enabled ones ("none" for none), to take one out or put it back without a +commit. Prints the matrix, a JSON list that is empty when no kind runs; +--github-output also writes it to $GITHUB_OUTPUT as "machines". + +Standard library only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from pathlib import Path +from typing import Optional + +REPO_ROOT = Path(__file__).resolve().parents[3] +CONFIG = ".github/hipep-machines.json" +ID = re.compile(r"^[a-z0-9][a-z0-9-]*$") +# npu_test's tiers, lowest first. +TIERS = ("install", "pr", "full") + + +def load(text: str) -> list[dict]: + """The config's machines, checked; ValueError names the first bad one.""" + machines = json.loads(text).get("machines") + if not isinstance(machines, list): + raise ValueError('no "machines" list') + out, seen = [], set() + for entry in machines: + if not isinstance(entry, dict): + raise ValueError(f"not an object: {entry!r}") + mid = entry.get("id") + if not isinstance(mid, str) or not ID.match(mid) or mid in seen: + raise ValueError(f"id {mid!r}: missing, repeated, or not lower-case letters, digits and -") + seen.add(mid) + labels = entry.get("runs-on") + if (not isinstance(labels, list) or not all(isinstance(l, str) and l for l in labels) + or "self-hosted" not in labels): + raise ValueError(f"{mid}: runs-on must be a list of labels, self-hosted among them") + enabled = entry.get("enabled", True) + limit = entry.get("max-model-gb", 0) + max_tier = entry.get("max-tier", "full") + changed = entry.get("changed-models", True) + if not isinstance(enabled, bool): + raise ValueError(f"{mid}: enabled must be true or false") + if isinstance(limit, bool) or not isinstance(limit, (int, float)) or limit < 0: + raise ValueError(f"{mid}: max-model-gb must be a number of GB, 0 for no limit") + if max_tier not in TIERS: + raise ValueError(f"{mid}: max-tier must be one of {', '.join(TIERS)}") + if not isinstance(changed, bool): + raise ValueError(f"{mid}: changed-models must be true or false") + out.append({"id": mid, "runs-on": labels, "enabled": enabled, "max-model-gb": limit, + "max-tier": max_tier, "changed-models": changed}) + return out + + +def pick(machines: list[dict], only: str) -> tuple[list[dict], list[str]]: + """The machines to run on, and the names in only that match none.""" + names = [n.strip() for n in only.split(",") if n.strip()] + if not names: + return [m for m in machines if m["enabled"]], [] + if names == ["none"]: + return [], [] + known = {m["id"] for m in machines} + return [m for m in machines if m["id"] in names], [n for n in names if n not in known] + + +def tier_on(machine: dict, tier: str) -> str: + """The tier a machine's job runs: the run's, or its max-tier if lower.""" + return min(tier, machine["max-tier"], key=TIERS.index) + + +def matrix(machines: list[dict], tier: str, changed: str = "") -> str: + """The job matrix's include list, on one line for $GITHUB_OUTPUT.""" + return json.dumps([{"id": m["id"], "runs-on": m["runs-on"], "max-model-gb": m["max-model-gb"], + "tier": tier_on(m, tier), + "changed-models": changed if m["changed-models"] else ""} + for m in machines], + separators=(",", ":")) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--config", type=Path, default=REPO_ROOT / CONFIG) + parser.add_argument("--tier", choices=TIERS, default="pr", help="The run's tier (default: pr).") + parser.add_argument("--changed-models", default="", + help="Comma-separated models whose catalog entries the change touched.") + parser.add_argument("--only", default="", + help='Comma-separated kinds to run instead of the enabled ones; "none" for none.') + parser.add_argument("--github-output", action="store_true", + help="Also append the matrix to $GITHUB_OUTPUT.") + args = parser.parse_args(argv) + + try: + machines = load(args.config.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + print(f"::error file={CONFIG}::{exc}") + return 2 + chosen, unknown = pick(machines, args.only) + for name in unknown: + print(f"::warning::FLM_HIPEP_MACHINES names {name}, which {CONFIG} does not list") + value = matrix(chosen, args.tier, args.changed_models.strip()) + names = [f"{m['id']} ({tier_on(m, args.tier)} tier)" for m in chosen] + print(f"HIPEP machines: {', '.join(names) or 'none'}") + print(f"machines={value}") + if args.github_output: + target = os.environ.get("GITHUB_OUTPUT") + if not target: + print("--github-output given but $GITHUB_OUTPUT is not set", file=sys.stderr) + return 2 + with open(target, "a", encoding="utf-8") as out: + out.write(f"machines={value}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/script/ci/test_changes.py b/.github/script/ci/test_changes.py index 0abf164eb..083df882d 100644 --- a/.github/script/ci/test_changes.py +++ b/.github/script/ci/test_changes.py @@ -4,7 +4,6 @@ import json import os -import re import shutil import subprocess import sys @@ -131,7 +130,8 @@ def test_github_output(self): lines = target.read_text(encoding="utf-8").splitlines() self.assertIn("run_tests=false", lines) self.assertEqual([l.split("=")[0] for l in lines], - ["run_tests", "families", "asr_full", "suites", "hipep", "summary"]) + ["run_tests", "families", "asr_full", "suites", "hipep", "hipep_models", + "summary"]) def test_suites_scoped_to_the_change(self): cases = { @@ -174,14 +174,18 @@ def test_oga_hipep_code_runs_only_hipep(self): self.assertEqual((out["run_tests"], out["families"], out["suites"], out["hipep"]), ("true", "", "none", "true")) - def test_hipep_runs_when_the_change_reaches_its_model(self): + def test_hipep_runs_when_the_change_reaches_its_models(self): cases = { - # The backend, and the qwen3.5 frontend's code. + # The backend, and the frontends of the families with oga-hipep models. ("src/test/oga_hipep/test_qwen_frontend.cpp",): "true", ("src/include/oga/oga_vision.hpp",): "true", ("src/common/AutoModel/modeling_qwen3_5vl.cpp",): "true", ("src/include/AutoModel/modeling_qwen3_5vl.hpp",): "true", ("src/common/AutoModel/modeling_qwen3_5vl_image.cpp",): "true", + ("src/common/AutoModel/modeling_qwen2.cpp",): "true", + ("src/common/AutoModel/modeling_gemma3.cpp",): "true", + ("src/common/AutoModel/modeling_gemma4e.cpp",): "true", + ("src/common/AutoModel/modeling_gpt_oss.cpp",): "true", # The tests of the suites it runs, and anything that runs every suite. (".github/script/npu_test/suites/chat.py",): "true", (".github/script/npu_test/suites/shutdown.py",): "true", @@ -192,7 +196,8 @@ def test_hipep_runs_when_the_change_reaches_its_model(self): # The qwen3.5 NPU engine, other frontends, other suites, docs. ("src/lib/xrt/qwen3_5vl_npu.dll",): "false", ("src/common/models/qwen35/core/dev04/a.cpp",): "false", - ("src/common/AutoModel/modeling_gemma4e.cpp",): "false", + ("src/common/AutoModel/modeling_lfm2.cpp",): "false", + ("src/common/AutoModel/modeling_gemma3_text.cpp",): "false", (".github/script/npu_test/suites/serving.py",): "false", ("src/common/whisper/whisper.cpp",): "false", ("docs/index.md",): "false", @@ -221,12 +226,17 @@ def test_oga_hipep_entries_name_no_family(self): git = FakeGit(base={changes.MODEL_LIST: base}, head={changes.MODEL_LIST: gpu_only}) d = changes.decide([changes.MODEL_LIST], git, gpu_only) self.assertEqual((d.families, d.suites_value(), d.runs_hipep()), (set(), "none", True)) + self.assertEqual(d.outputs()["hipep_models"], "gemma3-oga-hipep:4b") # An NPU entry changed beside it still names its family. head["models"]["lfm2"]["1b"]["default_context_length"] = 4096 both = json.dumps(head) git = FakeGit(base={changes.MODEL_LIST: base}, head={changes.MODEL_LIST: both}) d = changes.decide([changes.MODEL_LIST], git, both) self.assertEqual((d.families, d.suites_value(), d.runs_hipep()), ({"lfm2"}, "chat,prefix_cache", True)) + # A removed one still runs HIPEP, but is no model to test. + git = FakeGit(base={changes.MODEL_LIST: gpu_only}, head={changes.MODEL_LIST: base}) + d = changes.decide([changes.MODEL_LIST], git, base) + self.assertEqual((d.runs_hipep(), d.outputs()["hipep_models"]), (True, "")) def test_backend_key_overrides_the_oga_hipep_suffix(self): base = catalog(**{"gemma3-oga-hipep__4b": ("Gemma3-4B_gpuep", "gemma3"), "lfm2__1b": ("LFM2", "lfm2")}) @@ -234,15 +244,17 @@ def test_backend_key_overrides_the_oga_hipep_suffix(self): def outcome(base, head): git = FakeGit(base={changes.MODEL_LIST: base}, head={changes.MODEL_LIST: head}) d = changes.decide([changes.MODEL_LIST], git, head) - return d.families, d.suites_value() + return d.families, d.suites_value(), d.outputs()["hipep_models"] # An -oga-hipep entry moved to an NPU backend runs on the NPU, and - # Test Windows HIPEP still runs, since it ran there before. + # Test Windows HIPEP still runs, since it ran there before, though + # no longer on that model. head = json.loads(base) head["models"]["gemma3-oga-hipep"]["4b"]["backend"] = "flm" moved = json.dumps(head) self.assertEqual(outcome(base, moved)[0], {"gemma3"}) self.assertNotEqual(outcome(base, moved)[1], "none") + self.assertEqual(outcome(base, moved)[2], "") git = FakeGit(base={changes.MODEL_LIST: base}, head={changes.MODEL_LIST: moved}) self.assertTrue(changes.decide([changes.MODEL_LIST], git, moved).runs_hipep()) # And an edit to it, once there, names its family too. @@ -252,7 +264,7 @@ def outcome(base, head): head = json.loads(base) head["models"]["lfm2"]["1b"]["backend"] = "oga-hipep" self.assertEqual(outcome(json.dumps(head), json.dumps(head).replace("LFM2", "LFM2b")), - (set(), "none")) + (set(), "none", "lfm2:1b")) def test_model_info_of_an_entry_moved_off_oga_hipep_names_its_family(self): base = catalog(**{"gemma3-oga-hipep__4b": ("x", "gemma3"), "lfm2__1b": ("y", "lfm2")}) @@ -294,6 +306,7 @@ def test_model_info_of_oga_hipep_entries_names_no_family(self): head={changes.MODEL_LIST: cat, changes.MODEL_INFO: '{"lfm2:1b": 1, "gemma3-oga-hipep:4b": 2}'}) d = changes.decide([changes.MODEL_INFO], git, cat) self.assertEqual((d.families, d.suites_value(), d.runs_hipep()), (set(), "none", True)) + self.assertEqual(d.outputs()["hipep_models"], "gemma3-oga-hipep:4b") def test_model_list_without_git_adds_nothing(self): d = changes.decide([changes.MODEL_LIST], None, catalog(a__b=("A", "qwen3"))) @@ -326,15 +339,16 @@ def test_table_matches_catalog(self): self.assertEqual(named - known, set()) self.assertEqual(known - named, set()) - def test_gpu_families_are_those_test_windows_hipep_tests(self): - """GPU_FAMILIES are the families of the models Test Windows HIPEP tests - when a dispatch names none.""" - workflow = (changes.REPO_ROOT / ".github/workflows/build-and-test.yml").read_text(encoding="utf-8") - job = re.search(r"\n test-win-hipep:\n(.*?)\n\n", workflow, re.S).group(1) - tested = re.search(r"inputs\.models \|\| '([^']+)'", job).group(1).split(",") - cat = changes.load_catalog((changes.REPO_ROOT / changes.MODEL_LIST).read_text(encoding="utf-8")) - self.assertTrue(all(changes.runs_on_gpu(m) for m in tested), tested) - self.assertEqual({changes.family_of(cat[m]) for m in tested}, changes.GPU_FAMILIES) + def test_gpu_families_come_from_the_head_catalog(self): + """A frontend reaches Test Windows HIPEP when its family has a GPU entry + in the head catalog, and any frontend does when there is no catalog.""" + cat = catalog(**{"gemma3-oga-hipep__4b": ("Gemma3-4B_gpuep", "gemma3"), "lfm2__1b": ("LFM2", "lfm2")}) + self.assertEqual(changes.gpu_families(changes.load_catalog(cat)), {"gemma3"}) + self.assertTrue(changes.decide(["src/common/AutoModel/modeling_gemma3.cpp"], None, cat).runs_hipep()) + self.assertFalse(changes.decide(["src/common/AutoModel/modeling_lfm2.cpp"], None, cat).runs_hipep()) + self.assertTrue(changes.decide(["src/common/AutoModel/modeling_lfm2.cpp"], None, None).runs_hipep()) + # The NPU engine of a family with a GPU entry does not reach it. + self.assertFalse(changes.decide(["src/lib/xrt/gemma3_npu.dll"], None, cat).runs_hipep()) @unittest.skipUnless(shutil.which("git"), "git not installed") diff --git a/.github/script/ci/test_hipep_machines.py b/.github/script/ci/test_hipep_machines.py new file mode 100644 index 000000000..5d44bdeca --- /dev/null +++ b/.github/script/ci/test_hipep_machines.py @@ -0,0 +1,149 @@ +"""Tests for hipep_machines.py. Run: python3 .github/script/ci/test_hipep_machines.py""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import hipep_machines # noqa: E402 + +SCRIPT = HERE / "hipep_machines.py" + + +def config(*machines: dict) -> str: + return json.dumps({"machines": list(machines)}) + + +def machine(mid: str, *labels: str, **fields) -> dict: + return {"id": mid, "runs-on": ["self-hosted", "windows", "x64", "oga-hipep", *labels], **fields} + + +NPU_RUNNERS = {"id": "npu-runners", "runs-on": ["self-hosted", "windows", "x64", "npu", "sweep", "oga-hipep"], + "max-model-gb": 32, "max-tier": "pr", "changed-models": False} + + +class Load(unittest.TestCase): + def test_repository_config(self): + """The committed config loads, and enables at least one kind.""" + text = (hipep_machines.REPO_ROOT / hipep_machines.CONFIG).read_text(encoding="utf-8") + machines = hipep_machines.load(text) + self.assertTrue(any(m["enabled"] for m in machines)) + + def test_defaults(self): + (m,) = hipep_machines.load(config(machine("stx-halo", "stx-halo"))) + self.assertEqual((m["enabled"], m["max-model-gb"], m["max-tier"], m["changed-models"]), + (True, 0, "full", True)) + + def test_shares_the_npu_runners(self): + (m,) = hipep_machines.load(config(NPU_RUNNERS)) + self.assertEqual((m["runs-on"][3:], m["max-model-gb"], m["max-tier"], m["changed-models"]), + (["npu", "sweep", "oga-hipep"], 32, "pr", False)) + + def test_rejects(self): + bad = { + "no self-hosted": {"id": "stx", "runs-on": ["windows", "oga-hipep"]}, + "labels not a list": {"id": "stx", "runs-on": "self-hosted"}, + "upper-case id": machine("STX", "stx"), + "string limit": machine("stx", "stx", **{"max-model-gb": "16"}), + "negative limit": machine("stx", "stx", **{"max-model-gb": -1}), + "enabled as a string": machine("stx", "stx", enabled="false"), + "unknown max-tier": machine("stx", "stx", **{"max-tier": "nightly"}), + "changed-models as a string": machine("stx", "stx", **{"changed-models": "false"}), + } + for why, entry in bad.items(): + with self.subTest(why): + with self.assertRaises(ValueError): + hipep_machines.load(config(entry)) + with self.assertRaises(ValueError): + hipep_machines.load(config(machine("stx", "stx"), machine("stx", "stx"))) + + +class Pick(unittest.TestCase): + MACHINES = hipep_machines.load(config( + machine("stx-halo", "stx-halo"), + machine("stx", "stx", enabled=False, **{"max-model-gb": 16}), + NPU_RUNNERS)) + + def ids(self, only: str) -> tuple[list[str], list[str]]: + chosen, unknown = hipep_machines.pick(self.MACHINES, only) + return [m["id"] for m in chosen], unknown + + def tiers(self, tier: str) -> list[tuple[str, str]]: + chosen, _ = hipep_machines.pick(self.MACHINES, "") + return [(m["id"], m["tier"]) for m in json.loads(hipep_machines.matrix(chosen, tier))] + + def test_enabled_by_default(self): + self.assertEqual(self.ids(""), (["stx-halo", "npu-runners"], [])) + self.assertEqual(self.ids(" , "), (["stx-halo", "npu-runners"], [])) + + def test_only(self): + self.assertEqual(self.ids("stx"), (["stx"], [])) + self.assertEqual(self.ids("npu-runners, stx-halo"), (["stx-halo", "npu-runners"], [])) + self.assertEqual(self.ids("none"), ([], [])) + self.assertEqual(self.ids("stx,strix"), (["stx"], ["strix"])) + + def test_matrix(self): + chosen, _ = hipep_machines.pick(self.MACHINES, "npu-runners") + self.assertEqual(json.loads(hipep_machines.matrix(chosen, "pr")), + [{"id": "npu-runners", + "runs-on": ["self-hosted", "windows", "x64", "npu", "sweep", "oga-hipep"], + "max-model-gb": 32, "tier": "pr", "changed-models": ""}]) + self.assertEqual(hipep_machines.matrix([], "full"), "[]") + + def test_changed_models(self): + """Only the kinds that test the change's models get them.""" + chosen, _ = hipep_machines.pick(self.MACHINES, "") + rows = json.loads(hipep_machines.matrix(chosen, "pr", "gemma3-oga-hipep:4b,gpt-oss-oga-hipep:120b")) + self.assertEqual([(m["id"], m["changed-models"]) for m in rows], + [("stx-halo", "gemma3-oga-hipep:4b,gpt-oss-oga-hipep:120b"), ("npu-runners", "")]) + + def test_max_tier(self): + self.assertEqual(self.tiers("full"), [("stx-halo", "full"), ("npu-runners", "pr")]) + self.assertEqual(self.tiers("pr"), [("stx-halo", "pr"), ("npu-runners", "pr")]) + self.assertEqual(self.tiers("install"), [("stx-halo", "install"), ("npu-runners", "install")]) + + +class Cli(unittest.TestCase): + def run_cli(self, *args: str, text: str) -> tuple[int, str, list[str]]: + with tempfile.TemporaryDirectory() as tmp: + path, target = Path(tmp) / "machines.json", Path(tmp) / "out" + path.write_text(text, encoding="utf-8") + res = subprocess.run([sys.executable, str(SCRIPT), "--config", str(path), "--github-output", *args], + capture_output=True, text=True, + env={**os.environ, "GITHUB_OUTPUT": str(target)}) + lines = target.read_text(encoding="utf-8").splitlines() if target.exists() else [] + return res.returncode, res.stdout, lines + + def test_github_output(self): + code, _, lines = self.run_cli("--tier", "full", "--only", "", "--changed-models", "gemma3-oga-hipep:4b", + text=config(machine("stx-halo", "stx-halo"), NPU_RUNNERS)) + self.assertEqual(code, 0) + self.assertEqual(len(lines), 1) + key, value = lines[0].split("=", 1) + self.assertEqual((key, [(m["id"], m["tier"], m["changed-models"]) for m in json.loads(value)]), + ("machines", [("stx-halo", "full", "gemma3-oga-hipep:4b"), ("npu-runners", "pr", "")])) + + def test_none_writes_an_empty_matrix(self): + code, _, lines = self.run_cli("--only", "none", text=config(machine("stx-halo", "stx-halo"))) + self.assertEqual((code, lines), (0, ["machines=[]"])) + + def test_unknown_name_warns(self): + code, out, lines = self.run_cli("--only", "strix", text=config(machine("stx-halo", "stx-halo"))) + self.assertEqual((code, lines), (0, ["machines=[]"])) + self.assertIn("::warning::", out) + + def test_bad_config_fails(self): + code, out, lines = self.run_cli(text=config(machine("stx", "stx", **{"max-tier": "nightly"}))) + self.assertEqual((code, lines), (2, [])) + self.assertIn("::error", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/script/npu_test/README.md b/.github/script/npu_test/README.md index faecf111a..ebbaf7704 100644 --- a/.github/script/npu_test/README.md +++ b/.github/script/npu_test/README.md @@ -6,8 +6,9 @@ report. Python 3.9+ standard library only, so a runner needs nothing installed. CI runs it from `.github/workflows/build-and-test.yml` through the `.github/actions/npu-test` action, once per package (XRT and HRX portable, the -Ubuntu 24.04 .deb, the MSI, the models-from-source builds), and once more for -the Windows XRT zip's GPU models (Test Windows HIPEP). +Ubuntu 24.04 .deb, the MSI, the models-from-source builds), and once per kind +of GPU machine for the Windows XRT zip's GPU models (Test Windows HIPEP; see +[HIPEP machines](#hipep-machines)). ## Running it @@ -23,8 +24,10 @@ python3 -m npu_test --flm-bin flm --label dev --suites chat --models qwen3:0.6b, | `--tier install\|pr\|full` | How much to run; see below | | `--suites a,b` | Only these suites (default: all) | | `--models a,b` | Restrict every suite to these model ids | -| `--backend oga-hipep` | Test every model of this GPU backend instead of the NPU models | -| `--families a,b` | Families a change touches; the pr tier tests all their models | +| `--backend oga-hipep` | Test this GPU backend's models instead of the NPU models: one per family, every one on the `full` tier | +| `--changed-models a,b` | With `--backend`: models whose catalog entries the change touched, which the `install` and `pr` tiers test too | +| `--max-model-gb GB` | Skip the models larger than this (catalog size), which the runner's memory cannot hold (default 0, no limit) | +| `--families a,b` | Families a change touches; the pr tier tests all their models (with `--backend`, the model of each family it tests only when touched) | | `--asr-full` | The whole ASR evaluation set on the pr tier | | `--asr-expected-key KEY` | Check WER against this row of `assets/asr_eval/expected.json` | | `--pull-budget MINUTES` | Time for downloading missing models, all together (default 30 on `install`/`pr`, else no limit; 0 for no limit) | @@ -32,16 +35,27 @@ python3 -m npu_test --flm-bin flm --label dev --suites chat --models qwen3:0.6b, Models live in `$FLM_MODEL_PATH`. Before any suite, the run pulls the models it will use that `flm list --json` reports as not installed (`flm pull`, one -`setup`/`pull` case each, smallest download first; a `--backend` run pulls all -of its models, which also checks their files against the pinned hashes), so a -download counts against no -request's timeout: `flm serve` would otherwise download inside the request -that names the model, and every request behind it would wait out its own -timeout. A model that does not pull is not tested. On the `install` and `pr` -tiers the downloads share a 30-minute budget (`--pull-budget MINUTES`, 0 for no -limit), so a model new to the catalog cannot spend a job's time limit on one -runner; a model the budget leaves undownloaded is a warning and goes untested -on that runner. The `full` tier and `--backend` runs wait for every download. +`setup`/`pull` case each, smallest download first), so a download counts +against no request's timeout: `flm serve` would otherwise download inside the +request that names the model, and every request behind it would wait out its +own timeout. A model that does not pull is not tested. On the `install` and +`pr` tiers the downloads share a 30-minute budget (`--pull-budget MINUTES`, 0 +for no limit), so a model new to the catalog cannot spend a job's time limit +on one runner; a model the budget leaves undownloaded is a warning and goes +untested on that runner. The `full` tier and `--backend` runs wait for every +download. + +A `--backend` run pulls every model it tests, which also checks their files +against the pinned hashes: minutes per oga-hipep package, 14 for the 124 GB +one. So once a pull succeeds, `model_cache.py` stamps the model in +`$FLM_MODEL_PATH/npu-test-verified`, with its pins in the catalog (Hugging Face +revision, files, file sources) and the size and modification time of each of +its files. A later run does not pull a model while `flm list` calls it +installed and neither its pins nor its files changed; its `setup`/`pull` case +says so. A new revision in the catalog, or a file changed on the runner, pulls +it again, which downloads what changed and verifies the rest. An entry pinned +to a branch rather than a revision is pulled every time. Delete a stamp, or +the folder, to have the next run verify again. A request that times out restarts that suite's server, since flm keeps working on a request its client gave up on, and the model's remaining cases are @@ -59,12 +73,22 @@ warnings and skips do not fail a run. Which models each suite runs at each tier is in `plan.py`, in one place. -The tiers pick NPU models only. A model whose `backend` in `flm list --json` runs -on the GPU (`oga-hipep`, in the Windows XRT package) is tested by a run with -`--backend oga-hipep`, which tests all of them whatever the tier. The Test -Windows HIPEP job runs the `chat`, `api` and `shutdown` suites that way on -`qwen3.5-oga-hipep:9b` alone, since the other packages are 4 to 124 GB each, -unless a manual run names the models (below). Such a run gives `api` the +The tiers above pick NPU models. A model whose `backend` in `flm list --json` +runs on the GPU (`oga-hipep`, in the Windows XRT package) is tested by a run +with `--backend oga-hipep`, and the Test Windows HIPEP job runs the `chat`, +`api` and `shutdown` suites that way. On the `install` and `pr` tiers it tests +one model per family (`PR_GPU` in `plan.py`): a family's models share its +frontend and features, and differ only in weights. That is the newest Qwen +(`qwen3.8-oga-hipep:27b`, for the Qwen3.5, 3.6 and 3.8 models) and Gemma 4 +(`gemma4-it-oga-hipep:26b-a4b`), about 33 GB; Qwen2.5, Gemma 3, the Gemma 4 +12B on its own frontend (`gemma4-12b`) and gpt-oss (69 GB) run there only +when the change touches their family (`--families`). +The run also tests the models whose catalog entries the change added or +edited (`--changed-models`), on the kinds of machine that test them (see +[HIPEP machines](#hipep-machines)). The `full` tier tests every oga-hipep model (4 to +124 GB each, about 410 GB in all). +Models larger than the machine holds are skipped either way, and a manual run +that names the models (below) tests only those. Such a run gives `api` the backend's models in place of its NPU list, so the OpenAI contract tests, tool calls included, run on the GPU too. @@ -73,11 +97,12 @@ those the changed files can affect (an ASR-only change runs `asr`, a d3-only change `decision`, an engine change `chat` and `prefix_cache`), or all of them when any changed file is shared code. The HRX packages never run `api` or `serving`, which test server logic the backend does not touch. Test Windows -HIPEP runs when every suite does, or when the change reaches -`qwen3.5-oga-hipep:9b`: the oga-hipep backend or its catalog entries, the -qwen3.5 frontend, the `chat`, `api` and `shutdown` tests. A change only to the -backend, its entries or the `shutdown` tests touches no NPU family and runs no -NPU test job. +HIPEP runs when every suite does, or when the change reaches an oga-hipep +model: the oga-hipep backend or its catalog entries, the frontend of a family +with an oga-hipep entry (qwen3.5, qwen2, gemma3, gemma4e, gpt-oss today, read +from the catalog), the `chat`, `api` and `shutdown` tests. A change only to +the backend, its entries or the `shutdown` tests touches no NPU family and +runs no NPU test job. A manual run of Build and Test can name the models to test, for example `gh workflow run build-and-test.yml --ref -f models=qwen3.5-oga-hipep:9b`: @@ -85,6 +110,76 @@ every package then runs only those, and one whose catalog lacks them runs nothing. Add `-f fresh-hip-ep=true` to have Test Windows HIPEP install hip-ep from scratch, instead of reusing the runner's copy. +## HIPEP machines + +Test Windows HIPEP runs one job per kind of machine listed in +`.github/hipep-machines.json`, each reporting as `windows-hipep-`; +`.github/script/ci/hipep_machines.py` checks the file and turns it into the +job matrix. The job first prints the runner's GPUs, its memory, and the +memory dedicated to each GPU, which Windows leaves out of its own figure +(96 GB of an STX Halo's 128 GB). Two kinds today: + +- `stx-halo`: an STX Halo of its own (xconucstrhalo86), labelled `oga-hipep` + and `stx-halo`, which no NPU job asks for, so it runs beside the NPU tests. + It tests every model nightly, and on a pull request also the models whose + catalog entries it added or edited, with 6 hours instead of 2: their new + revisions download within the job, and a change may add or re-pin many. +- `npu-runners`: the shared NPU test runners (`npu`, `sweep`: STX, Krackan and + STX Halo machines) that also carry `oga-hipep`, whichever is free. It holds + that runner for its run, as an NPU test job does. It tests the `pr` tier's + models even nightly (`max-tier`), since each runner would otherwise + download all 410 GB and the NPU nightly needs them, and skips the models + above 32 GB (`max-model-gb`): gpt-oss and the 120B-class Qwen models. It + leaves the changed entries' models to `stx-halo` (`changed-models`), so a + change to many of them does not hold an NPU runner for hours. + +GitHub assigns a job by labels alone, without looking at a runner's memory or +drivers, so a repository admin adds `oga-hipep` (Settings > Actions > +Runners, then the runner's labels) only to the shared runners that can run +the `pr` tier's 17 GB models: at least 32 GB the GPU can use, which is the +memory dedicated to it plus half the memory Windows reports (64 GB on a +machine that dedicates little; the job prints both), a current NPU driver, +and room for those models in `FLM_MODEL_PATH`. NPU jobs do not ask for the +label, so they still use every runner. Never add `stx-halo` to a shared +runner: the nightly's every-model job would then land on it. + +| Field | Meaning | +|---|---| +| `id` | Short name, in the job's name and its report's label | +| `runs-on` | The labels its runners register with; several runners may share them, and a run takes whichever is free. The NPU test runners' own (`npu`, `sweep`) share those runners, and a label only some of them carry (`oga-hipep`) picks those | +| `enabled` | `false` leaves the kind out of every run | +| `max-model-gb` | The largest model, by catalog size, its memory holds; larger ones are skipped there, each a `setup`/`size` case (0: no limit) | +| `max-tier` | The highest tier its job runs (`install`, `pr` or `full`, the default); a run of a higher tier runs this one there | +| `changed-models` | `false` leaves the models whose catalog entries the change added or edited to the kinds that test them (`true`, the default), whose job then gets 6 hours | + +A runner of a kind of its own (xconucstrhalo86 for `stx-halo`): + +1. Windows 11 with the NPU driver (the package's `flm.exe` loads XRT from it), + an AMD GPU driver hip-ep supports, and Python 3.9 or later on `PATH`. +2. A GitHub Actions runner registered for the repository with the labels + `self-hosted`, `windows`, `x64`, `oga-hipep` and the kind's label + (`stx-halo`), and neither `npu` nor `sweep`, or NPU test jobs would take it. +3. `FLM_MODEL_PATH`, in the runner's `.env`, on a disk with room for every + model it tests (about 410 GB for all of them, which the nightly tests, and + room for a new revision); otherwise they go to the machine's own + `FLM_MODEL_PATH`, which installing flm may have set, or else to + `%USERPROFILE%\.flm`. +4. Optionally, the models pulled once by hand (`flm pull `): a CI pull + gets 90 minutes, and the 124 GB model needs about 23 MB/s for that. An + interrupted pull resumes on the next run. Either way each model is verified + once more, by the first run's pull, before it is stamped. + +Another kind of its own is another entry with its own label and the largest +model its memory holds, its runners registered with that label. Another +runner of a kind already listed needs no change. + +A job no runner takes waits a day before it fails, and the run's verdict +waits with it. While a kind has no runner up (`stx-halo` until xconucstrhalo86 +is registered, `npu-runners` until a shared runner carries `oga-hipep`), set +the repository variable `FLM_HIPEP_MACHINES` to the kinds to run instead of +the enabled ones, or to `none`; unset, the enabled ones run. While only +`npu-runners` runs, no job tests the changed entries' models. + ## Suites | Suite | Checks | @@ -101,15 +196,16 @@ from scratch, instead of reusing the runner's copy. ## Layout ``` -run.py entry point and arguments -plan.py catalog and per-tier model selection -harness.py context, cases (pass/fail/warn/skip), report -server.py one flm serve process -client.py HTTP: JSON, SSE streams, multipart -console.py a program in a hidden console of its own: Ctrl+C, typed lines, the screen (Windows) -suites/ one module per suite: NAME, DESCRIPTION, run(ctx) -assets/ images, audio, the ASR evaluation set, d3 goldens -tools/ scripts that regenerate assets +run.py entry point and arguments +plan.py catalog and per-tier model selection +model_cache.py stamps of the GPU models a pull verified, so later runs skip the pull +harness.py context, cases (pass/fail/warn/skip), report +server.py one flm serve process +client.py HTTP: JSON, SSE streams, multipart +console.py a program in a hidden console of its own: Ctrl+C, typed lines, the screen (Windows) +suites/ one module per suite: NAME, DESCRIPTION, run(ctx) +assets/ images, audio, the ASR evaluation set, d3 goldens +tools/ scripts that regenerate assets ``` ## Adding a suite diff --git a/.github/script/npu_test/model_cache.py b/.github/script/npu_test/model_cache.py new file mode 100644 index 000000000..95cddef0c --- /dev/null +++ b/.github/script/npu_test/model_cache.py @@ -0,0 +1,133 @@ +"""Remembers the GPU-backend models a runner has verified, so that later runs +need not verify them again. + +`flm pull` of a model it already has hashes every file against the pinned +hashes: minutes for each oga-hipep package (14 for the 124 GB one), on every +run. Once a pull succeeds, a stamp in $FLM_MODEL_PATH/npu-test-verified +records what it verified: the model's pins in the catalog flm reads (its +Hugging Face revision, its files and where each comes from) and the size and +modification time of each of those files. A later run skips the pull while +`flm list` still calls the model installed, its pins are the same and none of +its files changed; anything else pulls again, which downloads what changed +and verifies the rest. Deleting a stamp, or the folder, makes the next run +verify again. + +An entry that names a branch instead of a revision can change on Hugging Face +with the catalog unchanged, so it is pulled every time. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import time +from pathlib import Path + +STAMP_DIR = "npu-test-verified" +# The catalog fields that decide which files a model has. +PIN_FIELDS = ("name", "url", "files", "file_sources") +REVISION = re.compile(r"^[0-9a-f]{40}$") + + +def find_model_list(flm_bin: str) -> Path | None: + """The model_list.json flm reads (utils::find_model_list), bar its last resorts.""" + configured = os.environ.get("FLM_CONFIG_PATH") + if configured and Path(configured).is_file(): + return Path(configured) + binary = shutil.which(flm_bin) + if not binary: + return None + exe_dir = Path(binary).resolve().parent + for candidate in (exe_dir / "model_list.json", exe_dir.parent / "share" / "flm" / "model_list.json"): + if candidate.is_file(): + return candidate + return None + + +def pins(entry: dict) -> dict | None: + """The fields that fix a catalog entry's files, or None when the entry + does not pin them all to a revision.""" + revision = re.search(r"/resolve/([^/]+)", str(entry.get("url") or "")) + sources = entry.get("file_sources") or {} + if (not revision or not REVISION.match(revision.group(1)) + or not entry.get("name") or not entry.get("files") or not isinstance(sources, dict) + or not all(isinstance(s, dict) and REVISION.match(str(s.get("revision", ""))) + for s in sources.values())): + return None + return {k: entry.get(k) for k in PIN_FIELDS} + + +class ModelCache: + def __init__(self, root: Path, models_dir: Path, entries: dict[str, dict]): + self.dir = root / STAMP_DIR + self.models_dir = models_dir + self.entries = entries # model id -> its model_list.json entry + + @classmethod + def open(cls, flm_bin: str) -> "ModelCache | None": + """The stamps for this binary's catalog; None without $FLM_MODEL_PATH + or a model_list.json to read the pins from.""" + root = os.environ.get("FLM_MODEL_PATH") + path = find_model_list(flm_bin) + if not root or path is None: + return None + try: + catalog = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + entries = {f"{name}:{tag}": entry + for name, tags in (catalog.get("models") or {}).items() if isinstance(tags, dict) + for tag, entry in tags.items() if isinstance(entry, dict)} + return cls(Path(root), Path(root) / str(catalog.get("model_path") or "models"), entries) + + def _path(self, model_id: str) -> Path: + return self.dir / (re.sub(r"[^A-Za-z0-9._-]+", "_", model_id) + ".json") + + def _state(self, model_id: str) -> dict | None: + """The model's pins, and the size and modification time of each of its + files; None when it has no pins or lacks a file.""" + entry = self.entries.get(model_id) + pinned = pins(entry) if entry else None + if pinned is None: + return None + folder = self.models_dir / entry["name"] + files = {} + for name in entry["files"]: + try: + stat = (folder / name).stat() + except OSError: + return None + files[name] = [stat.st_size, stat.st_mtime_ns] + return {"pins": pinned, "files": files} + + def verified(self, model_id: str) -> str | None: + """When the model was verified, if nothing about it changed since.""" + state = self._state(model_id) + if state is None: + return None + try: + stamp = json.loads(self._path(model_id).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(stamp, dict) or {k: stamp.get(k) for k in state} != state: + return None + return str(stamp.get("verified") or "earlier") + + def stamp(self, model_id: str) -> str: + """Records the model as verified now; returns why not, else "".""" + state = self._state(model_id) + if state is None: + return "not pinned to a revision, or a file is missing" + path = self._path(model_id) + stamp = {"model": model_id, "verified": time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime()), + **state} + try: + self.dir.mkdir(parents=True, exist_ok=True) + partial = path.with_name(path.name + ".partial") + partial.write_text(json.dumps(stamp, indent=1), encoding="utf-8") + os.replace(partial, path) + except OSError as exc: + return f"{type(exc).__name__}: {exc}" + return "" diff --git a/.github/script/npu_test/plan.py b/.github/script/npu_test/plan.py index b19de910f..4ed17df85 100644 --- a/.github/script/npu_test/plan.py +++ b/.github/script/npu_test/plan.py @@ -9,6 +9,11 @@ The chat suite covers language models family by family, so there is no separate sweep of every model on a pull request: one model per engine family exercises the engine, and the families a change touches run in full. + +A GPU backend run (--backend) tests that backend's models instead: one per +family on the install and pr tiers (PR_GPU; some families only when the +change touches them), plus those whose catalog entries the change touched +(--changed-models), and every one on the full tier. """ from __future__ import annotations @@ -63,9 +68,32 @@ NON_CHAT_FAMILIES = {"embed-gemma", "d3", "whisper-v3", "r2t2"} # Backends that run their models on the GPU rather than the NPU. Only a run -# that asks for one (--backend) tests its models, and then all of them. +# that asks for one (--backend) tests its models. GPU_BACKENDS = {"oga-hipep"} +# One model per family for a GPU backend run on the install and pr tiers. A +# family's models share its frontend and the backend, and the catalog gives +# them the same features (vision, reasoning, tool calling), so the rest add +# only other weights, which the full tier tests. A family missing from this +# table (one added to the catalog later) gets its smallest model, so it is +# never silently untested. +PR_GPU = { + "qwen3.5": "qwen3.8-oga-hipep:27b", # the newest Qwen; Qwen3.5 and 3.6 too + # Not 12b, whose tool-call round trip fails on OGA 0.19 (the Gemma 4 tool + # grammar), which would fail every pull request. + "gemma4e": "gemma4-it-oga-hipep:26b-a4b", + "qwen2": "qwen2.5-it-oga-hipep:14b", + "gemma3": "gemma3-oga-hipep:4b", + "gemma4-12b": "gemma4-12b-oga-hipep:12b", + "gpt-oss": "gpt-oss-oga-hipep:120b", +} + +# Families whose PR_GPU model the install and pr tiers test only when the +# change touches the family (--families), so that every pull request tests +# just the newest Qwen and Gemma: Qwen2.5 and Gemma 3, the generations before; +# gemma4-12b, a second Gemma 4 family; and gpt-oss, whose one model is 69 GB. +PR_GPU_WHEN_TOUCHED = {"qwen2", "gemma3", "gemma4-12b", "gpt-oss"} + # Suites with a fixed NPU model list that a GPU backend run gives its own chat # models instead: the OpenAI contract (tools, streaming, errors, usage) holds # for every backend the server drives, and flm must still end once it has @@ -195,7 +223,8 @@ def load_catalog(flm_bin: str, class Plan: def __init__(self, catalog: dict[str, Model], tier: str, families: list[str], - only_models: list[str] | None = None, backend: str = ""): + only_models: list[str] | None = None, backend: str = "", + changed_models: list[str] | None = None): if tier not in TIERS: raise ValueError(f"unknown tier {tier!r}") if backend and backend not in GPU_BACKENDS: @@ -205,7 +234,9 @@ def __init__(self, catalog: dict[str, Model], tier: str, families: list[str], self.families = set(families) self.only_models = only_models self.backend = backend # empty: the NPU backends - # Models the run could not pull (Context.pull), which no suite tests. + self.changed_models = list(changed_models or []) + # Models no suite tests: ones the run could not pull (Context.pull), or + # larger than the runner holds (--max-model-gb). self.unavailable: set[str] = set() def _serves(self, model_id: str) -> bool: @@ -237,8 +268,18 @@ def chat(self) -> list[str]: return [m for m in self.only_models if m in self.catalog and self._serves(m) and self.catalog[m].family not in NON_CHAT_FAMILIES] if self.backend: - return self._known(m.id for m in self.catalog.values() - if m.family not in NON_CHAT_FAMILIES) + models = [m for m in self.catalog.values() + if m.backend == self.backend and m.family not in NON_CHAT_FAMILIES] + if self.tier == "full": + return self._known(m.id for m in models) + picked = [] + for family in sorted({m.family for m in models}): + if family in PR_GPU_WHEN_TOUCHED and family not in self.families: + continue + smallest = min((m for m in models if m.family == family), + key=lambda m: (m.size, m.id)) + picked.append(PR_GPU.get(family, smallest.id)) + return self._known(picked + self.changed_models) if self.tier == "install": return self._known(INSTALL_CHAT) chat_models = [m for m in self.catalog.values() if m.family not in NON_CHAT_FAMILIES] diff --git a/.github/script/npu_test/run.py b/.github/script/npu_test/run.py index d8af085ea..cdfbfa2f6 100644 --- a/.github/script/npu_test/run.py +++ b/.github/script/npu_test/run.py @@ -15,7 +15,8 @@ import sys import time -from .harness import FAIL, Context, Result, configure_stdout +from .harness import FAIL, PASS, SKIP, Context, Result, configure_stdout +from .model_cache import ModelCache from .plan import GPU_BACKENDS, TIERS, Plan, load_catalog from .suites import SUITES, ORDER @@ -44,14 +45,22 @@ def parse_args(argv=None) -> argparse.Namespace: help=f"Comma-separated subset of: {', '.join(ORDER)} (default: all).") parser.add_argument("--families", default="", help="Comma-separated model families the change touches; " - "the pr tier tests every model in them.") + "the pr tier tests every model in them, or with --backend " + "one of each that it tests only when touched.") parser.add_argument("--asr-full", action="store_true", help="Run the whole ASR evaluation set even on the pr tier.") parser.add_argument("--models", default="", help="Comma-separated model ids; restricts every suite to these.") parser.add_argument("--backend", choices=["", *sorted(GPU_BACKENDS)], default="", - help="Test every model of this GPU backend instead of the NPU models " - "(default: the NPU models).") + help="Test this GPU backend's models instead of the NPU models: one " + "per family, or every one on the full tier (default: the NPU " + "models).") + parser.add_argument("--changed-models", default="", + help="With --backend: comma-separated models whose catalog entries " + "the change touched, which the install and pr tiers test too.") + parser.add_argument("--max-model-gb", type=float, default=0.0, metavar="GB", + help="Skip the models larger than this (catalog size), which this " + "runner's memory cannot hold (default: 0, no limit).") parser.add_argument("--model-extension", action="append", default=[], help="Model-extension manifest passed to `flm list` and every " "server (repeatable).") @@ -108,7 +117,9 @@ def main(argv=None) -> int: catalog, catalog_error = {}, f"`flm list --json` failed: {type(exc).__name__}: {exc}" families = [f.strip() for f in args.families.split(",") if f.strip()] only = [m.strip() for m in args.models.split(",") if m.strip()] or None - plan = Plan(catalog, args.tier, families, only_models=only, backend=args.backend) + changed = [m.strip() for m in args.changed_models.split(",") if m.strip()] + plan = Plan(catalog, args.tier, families, only_models=only, backend=args.backend, + changed_models=changed) ctx = Context(flm_bin=args.flm_bin, label=args.label, tier=args.tier, plan=plan, out_dir=args.output_dir, request_timeout=args.request_timeout, port=args.port, verbose=args.verbose, @@ -119,19 +130,46 @@ def main(argv=None) -> int: print(f"{args.label}: {args.tier} tier, {len(catalog)} catalog models, " f"suites: {', '.join(names)}" + (f", touched families: {', '.join(families)}" if families else "") - + (f", backend: {args.backend}" if args.backend else "")) + + (f", backend: {args.backend}" if args.backend else "") + + (f", changed models: {', '.join(changed)}" if args.backend and changed else "")) try: if catalog_error: print(catalog_error) ctx.record(Result("setup", "catalog", FAIL, detail=catalog_error)) else: + # A model larger than the runner's memory would only fail to load. + if args.max_model_gb > 0: + for model_id in plan.planned(names): + size_gb = plan.model(model_id).size / 1e9 + if size_gb > args.max_model_gb: + ctx.record(Result("setup", "size", SKIP, model_id, + f"{size_gb:.1f} GB, larger than this runner holds " + f"(--max-model-gb {args.max_model_gb:g})")) + plan.unavailable.add(model_id) # Pulled first, a download counts against no request's timeout: flm # serve downloads a missing model inside the request that names it, # and every request behind that one waits too. # A GPU backend's models are pulled even when present: the pull # checks their files against the pinned hashes and replaces any - # that do not match, which `flm list` does not. + # that do not match, which `flm list` does not. That takes minutes + # per model, so one a pull on this runner verified before, whose + # pins and files have not changed since, is not pulled again. missing = plan.missing(names, everything=bool(args.backend)) + cache = ModelCache.open(args.flm_bin) if args.backend else None + verified = {} + if cache is not None: + for model_id in missing: + when = cache.verified(model_id) if plan.model(model_id).installed else None + if when: + verified[model_id] = when + if verified: + print(f"\n== setup: verified on this runner before and unchanged since, so not " + f"pulled: {', '.join(verified)}", flush=True) + for model_id, when in verified.items(): + ctx.record(Result("setup", "pull", PASS, model_id, + f"not pulled: verified {when}, and neither its pins nor " + f"its files changed since", data={"cached": True})) + missing = [m for m in missing if m not in verified] if missing: budget = pull_budget(args) limit = f"{budget / 60:g}-min budget" if budget else "no time limit" @@ -141,6 +179,12 @@ def main(argv=None) -> int: started_pull = time.monotonic() ctx.pull(missing, PULL_TIMEOUT, budget) print(f"== setup done in {time.monotonic() - started_pull:.0f}s", flush=True) + if cache is not None: + for model_id in missing: + why = "" if model_id in plan.unavailable else cache.stamp(model_id) + if why: + print(f" [setup] {model_id}: not remembered as verified ({why})", + flush=True) for name in names: ctx.run_suite(SUITES[name]) finally: diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4cf97cbf7..12db1f035 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -18,10 +18,13 @@ name: Build and Test # # The Windows XRT package also holds the oga-hipep backend, which runs its # models on the GPU through hip-ep. Test Windows HIPEP runs the chat, api and -# shutdown suites on qwen3.5-oga-hipep:9b from the same zip, and Test Windows -# XRT tests the NPU models. HIPEP runs when every suite does, or when a change -# reaches that model (the backend, its catalog entries, the qwen3.5 frontend, -# the chat, api and shutdown tests); a change only to the backend or its +# shutdown suites on the oga-hipep models from the same zip, on each kind of +# machine in .github/hipep-machines.json: an STX Halo of its own, beside the +# NPU tests, and whichever shared NPU runner labelled oga-hipep (STX, Krackan +# or STX Halo) is free. Test Windows XRT tests the NPU models. +# HIPEP runs when every suite does, or when a change reaches an oga-hipep +# model (the backend, its catalog entries, the frontend of a family that has +# one, the chat, api and shutdown tests); a change only to the backend or its # entries runs no NPU test. # # A pull request runs only the suites its changes can affect (an ASR-only @@ -30,9 +33,10 @@ name: Build and Test # Tiers (.github/script/npu_test/plan.py): pull requests and pushes run the pr # tier, one model per engine family plus every model of a family the change # touches; the nightly run tests every model. A change that only touches docs -# builds and tests nothing. The portable test jobs get 120 minutes on the pr -# tier and 360 on the full one, which runs every model and every d3 size and -# may first download tens of GB of weights. +# builds and tests nothing. The portable test jobs and Test Windows HIPEP get +# 120 minutes on the pr tier and 360 on the full one, which runs every model +# and every d3 size and may first download tens of GB of weights; HIPEP also +# gets 360 where it tests the models whose catalog entries the change edited. # # A dispatch can name the models to test (models, e.g. qwen3.5-oga-hipep:9b): # every test job then runs only those, so a package without them runs @@ -85,9 +89,9 @@ on: description: How many distributions each .deb variant was built for. value: ${{ jobs.build-lin-xrt-deb.outputs.deb_distros }} -# A superseded pull request commit only holds the NPU runners up, so a new -# push cancels its run. Pushes to main, nightly runs and tags each get their -# own group and always finish. +# A superseded pull request commit only holds the NPU and HIPEP runners up, so +# a new push cancels its run. Pushes to main, nightly runs and tags each get +# their own group and always finish. concurrency: group: build-and-test-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: true @@ -105,6 +109,7 @@ jobs: hrx-suites: ${{ steps.detect.outputs.hrx-suites }} source: ${{ steps.detect.outputs.source }} hipep: ${{ steps.detect.outputs.hipep }} + hipep-machines: ${{ steps.hipep.outputs.machines }} steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -118,6 +123,20 @@ jobs: with: tier: ${{ inputs.tier }} + # One Test Windows HIPEP job per kind of GPU machine, each with the tier + # it runs and the changed models it tests. The repository variable + # FLM_HIPEP_MACHINES names other kinds, or "none", without a commit, + # e.g. while a kind's only runner is down. + - name: Pick HIPEP machines + id: hipep + env: + ONLY: ${{ vars.FLM_HIPEP_MACHINES }} + TIER: ${{ steps.detect.outputs.tier }} + CHANGED: ${{ steps.detect.outputs.hipep-models }} + run: | + python3 .github/script/ci/hipep_machines.py --tier "${TIER}" --changed-models "${CHANGED}" \ + --only "${ONLY}" --github-output + # Every npu_test run imports every suite, but a change to the shutdown # suite or console.py runs Test Windows HIPEP alone. A Windows-only # import there would break the Linux NPU jobs unseen until a change that @@ -139,7 +158,7 @@ jobs: && (needs.changes.outputs.suites != 'none' || needs.changes.outputs.hrx-suites != '' || needs.changes.outputs.source == 'true' - || needs.changes.outputs.hipep == 'true' + || (needs.changes.outputs.hipep == 'true' && needs.changes.outputs.hipep-machines != '[]') || vars.FLM_INSTALL_TESTS == 'true') runs-on: ubuntu-latest environment: npu-approval @@ -606,15 +625,31 @@ jobs: asr-full: ${{ needs.changes.outputs.asr-full }} asr-expected-key: windows - # The XRT zip's oga-hipep models, on the GPU: its runner needs an AMD GPU - # that hip-ep runs on. Besides chat, they run the OpenAI contract tests (api), - # tool calls included, and shutdown: Ctrl+C and /bye must still end flm once - # it has loaded hip-ep. Every tier tests qwen3.5-oga-hipep:9b alone: the other - # oga-hipep packages are 4 to 72 GB each, and a dispatch that names them - # (models) tests them instead. npu_test pulls the models first, so a runner - # without them downloads them (7.5 GB for the 9b) outside the requests' timeouts. + # The XRT zip's oga-hipep models, on the GPU, one job per kind of machine in + # .github/hipep-machines.json: the STX Halo, whose labels (oga-hipep, + # stx-halo) no NPU job asks for, so it runs beside the NPU test jobs, and + # the shared NPU runners also labelled oga-hipep, those whose memory holds + # the pr tier's models (STX, Krackan and STX Halo, whichever is free), one + # of which it holds for the job as an NPU test job does. Besides chat, the + # models run the OpenAI contract tests (api), tool calls included, and + # shutdown: Ctrl+C and /bye must still end flm once it has loaded hip-ep. + # The pr tier tests one model per family (PR_GPU in npu_test/plan.py: the + # newest Qwen and Gemma; Qwen2.5, Gemma 3, gemma4-12b and gpt-oss only when + # the change touches their family); the full tier tests every one, where + # the kind's max-tier allows (the shared runners test the pr tier's models + # nightly). + # The models whose catalog entries the change added or edited run too, on + # the kinds whose changed-models allows it (the STX Halo, not the shared + # runners), with the full tier's time: their new revisions download within + # the job, and a change may add or re-pin many at once. Each kind skips + # the models larger than its memory holds (max-model-gb), and a dispatch + # that names models (models) tests only those. npu_test pulls the models + # first, outside the requests' timeouts, and they stay on the runner: only + # its first run, or a new revision of a model, downloads (about 33 GB for + # the pr tier's, 410 GB for all), and a model verified before whose pins and + # files are unchanged is not verified again. test-win-hipep: - name: Test Windows HIPEP + name: Test Windows HIPEP (${{ matrix.id }}) needs: [changes, build-win-xrt, npu-gate] # npu-gate is skipped for trusted authors, so spell out what the # implicit success() check used to require. @@ -623,9 +658,16 @@ jobs: && (needs.npu-gate.result == 'success' || needs.npu-gate.result == 'skipped') && needs.changes.result == 'success' && needs.build-win-xrt.result == 'success' - && (needs.changes.outputs.hipep == 'true') }} - runs-on: [self-hosted, windows, x64, npu, sweep] - timeout-minutes: 120 + && (needs.changes.outputs.hipep == 'true') + && (needs.changes.outputs.hipep-machines != '[]') }} + strategy: + # Each kind of machine reports on its own; one failing must not cancel + # the others. + fail-fast: false + matrix: + include: ${{ fromJSON(needs.changes.outputs.hipep-machines) }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: ${{ (matrix.tier == 'full' || matrix.changed-models != '') && 360 || 120 }} steps: - *checkout-tests - name: Download package @@ -639,9 +681,17 @@ jobs: with: kind: portable path: package + # An APU's dedicated GPU memory (96 GB on the STX Halo) is outside the + # memory Windows reports, and Win32_VideoController's AdapterRAM stops + # at 4 GB; the display driver's registry entry has its size. - name: Show the GPU shell: powershell - run: Get-CimInstance Win32_VideoController | ForEach-Object { "$($_.Name), driver $($_.DriverVersion)" } + run: | + Get-CimInstance Win32_VideoController | ForEach-Object { "$($_.Name), driver $($_.DriverVersion)" } + "Memory: {0:N0} GB, not counting memory dedicated to a GPU" -f ((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB) + Get-ItemProperty 'HKLM:\SYSTEM\CurrentControlSet\Control\Class\{4d36e968-e325-11ce-bfc1-08002be10318}\0*' -ErrorAction SilentlyContinue | + Where-Object { $_.'HardwareInformation.qwMemorySize' } | + ForEach-Object { "{0}: {1:N0} GB dedicated" -f $_.DriverDesc, ($_.'HardwareInformation.qwMemorySize' / 1GB) } # flm installs hip-ep into %LOCALAPPDATA%\hip-ep once and reuses it after, # so only an empty one makes it download and install again. The runner # empties its temp folder after every job. A short name: the deepest @@ -657,11 +707,14 @@ jobs: uses: ./.github/actions/npu-test with: flm-bin: ${{ steps.package.outputs.flm-bin }} - label: windows-hipep - tier: ${{ needs.changes.outputs.tier }} + label: windows-hipep-${{ matrix.id }} + tier: ${{ matrix.tier }} suites: chat,api,shutdown - models: ${{ inputs.models || 'qwen3.5-oga-hipep:9b' }} + families: ${{ needs.changes.outputs.families }} + models: ${{ inputs.models }} backend: oga-hipep + changed-models: ${{ matrix.changed-models }} + max-model-gb: ${{ matrix.max-model-gb }} test-win-hrx: name: Test Windows HRX