Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/actionlint.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# Labels of the self-hosted NPU runners, for actionlint's runs-on check.
# Labels of the self-hosted runners, for actionlint's runs-on check: the NPU
# runners, and the GPU machines in .github/hipep-machines.json.
self-hosted-runner:
labels:
- flm
- npu
- sweep
- oga-hipep
- stx-halo
5 changes: 4 additions & 1 deletion .github/actions/detect-changes/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,11 @@ outputs:
description: '"true" when the chat suite runs, the only suite the models-from-source and model-extension tests run.'
value: ${{ steps.tier.outputs.source }}
hipep:
description: '"true" when every suite runs or the change reaches the model Test Windows HIPEP tests.'
description: '"true" when every suite runs or the change reaches an oga-hipep model, which Test Windows HIPEP tests one per family.'
value: ${{ steps.tier.outputs.hipep }}
hipep-models:
description: Comma-separated oga-hipep models whose catalog entries the change added or edited, which Test Windows HIPEP tests too.
value: ${{ steps.changes.outputs.hipep_models }}
tier:
description: install, pr or full.
value: ${{ steps.tier.outputs.tier }}
Expand Down
27 changes: 22 additions & 5 deletions .github/actions/npu-test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,17 @@ inputs:
required: false
default: ""
backend:
description: A GPU backend (oga-hipep) whose models to test instead of the NPU models.
description: A GPU backend (oga-hipep) whose models to test instead of the NPU models, one per family (every one on the full tier).
required: false
default: ""
changed-models:
description: With backend, comma-separated models whose catalog entries the change touched, which the pr tier tests too.
required: false
default: ""
max-model-gb:
description: Skip the models larger than this many GB (catalog size), which the runner's memory cannot hold; 0 for no limit.
required: false
default: "0"
model-extension:
description: Optional model-extension manifest passed to FLM.
required: false
Expand Down Expand Up @@ -88,7 +96,10 @@ runs:

# Models persist on the runner between jobs, so only the first job on a
# runner pays for a download. The harness pulls missing models before the
# suites, within a time budget on the pr tier (--pull-budget).
# suites, within a time budget on the pr tier (--pull-budget). A GPU
# backend run pulls every model it tests, which verifies its files, except
# one a pull on this runner verified before whose catalog pins and files
# are unchanged (npu_test/model_cache.py).
- name: Run tests (Linux)
id: linux
if: runner.os == 'Linux'
Expand All @@ -102,14 +113,17 @@ runs:
FAMILIES: ${{ inputs.families }}
MODELS: ${{ inputs.models }}
BACKEND: ${{ inputs.backend }}
CHANGED_MODELS: ${{ inputs.changed-models }}
MAX_MODEL_GB: ${{ inputs.max-model-gb || '0' }}
MODEL_EXTENSION: ${{ inputs.model-extension }}
ASR_FULL: ${{ inputs.asr-full }}
ASR_KEY: ${{ inputs.asr-expected-key }}
run: |
export FLM_MODEL_PATH="${FLM_MODEL_PATH:-/scratch}"
args=(--flm-bin "${FLM_BIN}" --label "${LABEL}" --tier "${TIER}" --suites "${SUITES}"
--families "${FAMILIES}" --models "${MODELS}" --backend "${BACKEND}"
--asr-expected-key "${ASR_KEY}"
--changed-models "${CHANGED_MODELS}"
--max-model-gb "${MAX_MODEL_GB}" --asr-expected-key "${ASR_KEY}"
--output-dir "${GITHUB_WORKSPACE}/npu-test-results/${LABEL}")
[ "${ASR_FULL}" = "true" ] && args+=(--asr-full)
[ -n "${MODEL_EXTENSION}" ] && args+=(--model-extension "${MODEL_EXTENSION}")
Expand All @@ -130,6 +144,8 @@ runs:
FAMILIES: ${{ inputs.families }}
MODELS: ${{ inputs.models }}
BACKEND: ${{ inputs.backend }}
CHANGED_MODELS: ${{ inputs.changed-models }}
MAX_MODEL_GB: ${{ inputs.max-model-gb || '0' }}
MODEL_EXTENSION: ${{ inputs.model-extension }}
ASR_FULL: ${{ inputs.asr-full }}
ASR_KEY: ${{ inputs.asr-expected-key }}
Expand All @@ -146,8 +162,9 @@ runs:
$outDir = Join-Path $env:GITHUB_WORKSPACE "npu-test-results\$env:LABEL"
$cmdArgs = @('-m', 'npu_test', "--flm-bin=$env:FLM_BIN", "--label=$env:LABEL",
"--tier=$env:TIER", "--suites=$env:SUITES", "--families=$env:FAMILIES",
"--models=$env:MODELS", "--backend=$env:BACKEND", "--asr-expected-key=$env:ASR_KEY",
"--output-dir=$outDir")
"--models=$env:MODELS", "--backend=$env:BACKEND",
"--changed-models=$env:CHANGED_MODELS", "--max-model-gb=$env:MAX_MODEL_GB",
"--asr-expected-key=$env:ASR_KEY", "--output-dir=$outDir")
if ($env:ASR_FULL -eq 'true') { $cmdArgs += '--asr-full' }
if ($env:MODEL_EXTENSION) {
$cmdArgs += "--model-extension=$env:MODEL_EXTENSION"
Expand Down
18 changes: 18 additions & 0 deletions .github/hipep-machines.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"machines": [
{
"id": "stx-halo",
"runs-on": ["self-hosted", "windows", "x64", "oga-hipep", "stx-halo"],
"enabled": true,
"max-model-gb": 0
},
{
"id": "npu-runners",
"runs-on": ["self-hosted", "windows", "x64", "npu", "sweep", "oga-hipep"],
"enabled": true,
"max-model-gb": 32,
"max-tier": "pr",
"changed-models": false
}
]
}
43 changes: 30 additions & 13 deletions .github/script/ci/changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@
test of the shutdown suite, which no NPU job runs; a change to a
model_list.json field outside its model entries, such as
model_path, gives "all"
hipep true when the change can reach qwen3.5-oga-hipep:9b, which Test
Windows HIPEP runs on the GPU: when every suite runs, or a changed
file is oga-hipep code, a GPU catalog entry, the qwen3.5 frontend,
or a test of the chat, api and shutdown suites that job runs
hipep true when the change can reach an oga-hipep model, which Test
Windows HIPEP runs on the GPU (one per family): when every suite
runs, or a changed file is oga-hipep code, a GPU catalog entry,
the frontend of a family with a GPU entry, or a test of the chat,
api and shutdown suites that job runs
hipep_models
comma-separated oga-hipep models whose catalog entries were added
or edited, which Test Windows HIPEP tests too
summary one line for the job log

When in doubt it tests more: an all-zero or missing base (a new branch), or a
Expand Down Expand Up @@ -142,17 +146,16 @@
# catalog entries whose effective backend is oga-hipep: their "backend" key,
# or without one, a name ending in GPU_SUFFIX (model_list::backend_for_family).
# No NPU model runs its code (bar the header above) or those entries, so they
# name no family and run no NPU job: Test Windows HIPEP tests them, on
# qwen3.5-oga-hipep:9b.
# name no family and run no NPU job: Test Windows HIPEP tests them, on every
# GPU entry.
GPU_ONLY = re.compile(r"^(src/(common|include)/oga/|src/test/oga_hipep/)")
GPU_SUFFIX = "-oga-hipep"
# An entry's "backend" key (flm::backend::kBackendKey), when present, overrides
# the suffix rule; GPU_BACKEND is the oga-hipep backend's id.
GPU_BACKEND_KEY = "backend"
GPU_BACKEND = "oga-hipep"
# The family of qwen3.5-oga-hipep:9b: a change to its frontend reaches that
# model too, one to its NPU engine does not.
GPU_FAMILIES = {"qwen3.5"}
# A family's frontend also drives its GPU entries, and its NPU engine does not:
# a change to the frontend of a family with one (gpu_families) reaches them.
GPU_FRONTEND = re.compile(r"^src/(include|common)/AutoModel/")
# The suites Test Windows HIPEP runs.
GPU_SUITES = {"chat", "api", "shutdown"}
Expand Down Expand Up @@ -274,8 +277,10 @@ class Decision:
asr_full: bool = False
# None: every suite. Otherwise the suites every changed file is scoped to.
suites: Optional[set] = field(default_factory=set)
# A changed file reaches qwen3.5-oga-hipep:9b (every suite running does too).
# A changed file reaches an oga-hipep model (every suite running does too).
hipep: bool = False
# The oga-hipep entries added or edited in the head catalog.
gpu_models: set = field(default_factory=set)
reasons: list = field(default_factory=list)

def all_suites(self, path: str) -> None:
Expand Down Expand Up @@ -320,6 +325,7 @@ def outputs(self) -> dict[str, str]:
"asr_full": str(self.asr_full).lower(),
"suites": self.suites_value(),
"hipep": str(self.runs_hipep()).lower(),
"hipep_models": ",".join(sorted(self.gpu_models)),
"summary": self.summary()}


Expand Down Expand Up @@ -350,6 +356,14 @@ def runs_on_gpu(model_id: str, entry: Optional[dict] = None) -> bool:
return model_id.split(":", 1)[0].endswith(GPU_SUFFIX)


def gpu_families(catalog: dict[str, dict]) -> Optional[set]:
"""The families with a GPU entry, whose frontends drive the oga-hipep
backend too; None without a catalog, when any frontend might be one."""
if not catalog:
return None
return {family_of(e) for mid, e in catalog.items() if runs_on_gpu(mid, e)} - {""}


def catalog_diff(base: dict[str, dict], head: dict[str, dict]) -> tuple[set, list]:
"""Families with an NPU entry added, removed or edited between two catalogs,
and the GPU entries that were. An entry is a GPU one only if it is on
Expand Down Expand Up @@ -408,7 +422,8 @@ def model_info_diff(base_text: Optional[str], head_text: Optional[str],
def add_catalog_change(d: Decision, path: str, fams: set, gpu: list,
catalog: dict[str, dict], neither: str) -> None:
"""A catalog file's changed entries: their NPU families in full, and Test
Windows HIPEP for GPU entries. A change to neither is not scoped."""
Windows HIPEP for GPU entries, which also tests those the head catalog
still runs on the GPU. A change to neither is not scoped."""
d.families |= fams
if fams:
d.add_suites(family_suites(fams, catalog))
Expand All @@ -417,6 +432,7 @@ def add_catalog_change(d: Decision, path: str, fams: set, gpu: list,
what = ", ".join(sorted(fams)) or ("no NPU family" if gpu else neither)
if gpu:
d.hipep = True
d.gpu_models |= {mid for mid in gpu if mid in catalog and runs_on_gpu(mid, catalog[mid])}
what += (f"; {len(gpu)} oga-hipep entr{'y' if len(gpu) == 1 else 'ies'}, e.g. {gpu[0]},"
" which only Test Windows HIPEP runs")
d.reasons.append(f"{path}: entries changed in {what}")
Expand Down Expand Up @@ -488,6 +504,7 @@ def decide(files: Optional[list[str]], git: Optional[Git] = None,
d.reasons.append(f"{len(code)} of {len(files)} changed file(s) are code, e.g. {code[0]}")

head_catalog = load_catalog(head_catalog_text)
gpu_fams = gpu_families(head_catalog)
xclbin_families: dict[str, set] = {}
for entry in head_catalog.values():
if entry.get("name"):
Expand All @@ -508,8 +525,8 @@ def decide(files: Optional[list[str]], git: Optional[Git] = None,
if new:
d.families |= new
d.reasons.append(f"{path}: {hit[0]} -> {', '.join(sorted(new))}")
if GPU_FAMILIES & set(hit[1]) and GPU_FRONTEND.match(path):
d.reach_hipep(f"{path}: {hit[0]} also runs qwen3.5-oga-hipep:9b,"
if GPU_FRONTEND.match(path) and (gpu_fams is None or gpu_fams & set(hit[1])):
d.reach_hipep(f"{path}: {hit[0]} also runs oga-hipep models,"
" which Test Windows HIPEP tests")

# Which suites this file can affect.
Expand Down
155 changes: 155 additions & 0 deletions .github/script/ci/hipep_machines.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Lists the machines Test Windows HIPEP runs on, as its job matrix.

hipep_machines.py [--tier pr] [--changed-models a,b] [--only stx-halo,npu-runners]
[--github-output]

.github/hipep-machines.json has one entry per kind of machine, each tested
by a job of its own:

id short name (stx-halo, npu-runners), in the job's name and
in its report's label
runs-on the labels its runners register with. Several runners may
share them; a run takes whichever is free. With the NPU
test runners' own (npu, sweep) the job shares those
runners, holding one for its run as an NPU test job does;
a label only some of them carry (oga-hipep) picks those
enabled false leaves it out of every run: a kind with no runner
yet, or with all of them down (a job no runner takes waits
a day before it fails, and holds up the run's verdict)
max-model-gb the largest oga-hipep model, by its catalog size, that the
machine's memory holds; larger ones are skipped there. 0
(the default) for no limit
max-tier the highest tier its job runs: install, pr or full (the
default). A run of a higher tier runs this one there, so
runners that cannot hold every model, or that the NPU
nightly needs, test the pr tier's models nightly
changed-models false leaves the models whose catalog entries the change
added or edited to the kinds that test them (true, the
default): a change to many entries downloads each new
revision within the job, which would hold a shared runner
for hours

--tier is the run's tier; each machine in the matrix carries the tier its job
runs. --changed-models, the change's (changes.py's hipep-models), goes to the
machines that test them, and an empty list to the rest. --only, from the
FLM_HIPEP_MACHINES repository variable, runs the named kinds instead of the
enabled ones ("none" for none), to take one out or put it back without a
commit. Prints the matrix, a JSON list that is empty when no kind runs;
--github-output also writes it to $GITHUB_OUTPUT as "machines".

Standard library only.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
from pathlib import Path
from typing import Optional

REPO_ROOT = Path(__file__).resolve().parents[3]
CONFIG = ".github/hipep-machines.json"
ID = re.compile(r"^[a-z0-9][a-z0-9-]*$")
# npu_test's tiers, lowest first.
TIERS = ("install", "pr", "full")


def load(text: str) -> list[dict]:
"""The config's machines, checked; ValueError names the first bad one."""
machines = json.loads(text).get("machines")
if not isinstance(machines, list):
raise ValueError('no "machines" list')
out, seen = [], set()
for entry in machines:
if not isinstance(entry, dict):
raise ValueError(f"not an object: {entry!r}")
mid = entry.get("id")
if not isinstance(mid, str) or not ID.match(mid) or mid in seen:
raise ValueError(f"id {mid!r}: missing, repeated, or not lower-case letters, digits and -")
seen.add(mid)
labels = entry.get("runs-on")
if (not isinstance(labels, list) or not all(isinstance(l, str) and l for l in labels)
or "self-hosted" not in labels):
raise ValueError(f"{mid}: runs-on must be a list of labels, self-hosted among them")
enabled = entry.get("enabled", True)
limit = entry.get("max-model-gb", 0)
max_tier = entry.get("max-tier", "full")
changed = entry.get("changed-models", True)
if not isinstance(enabled, bool):
raise ValueError(f"{mid}: enabled must be true or false")
if isinstance(limit, bool) or not isinstance(limit, (int, float)) or limit < 0:
raise ValueError(f"{mid}: max-model-gb must be a number of GB, 0 for no limit")
if max_tier not in TIERS:
raise ValueError(f"{mid}: max-tier must be one of {', '.join(TIERS)}")
if not isinstance(changed, bool):
raise ValueError(f"{mid}: changed-models must be true or false")
out.append({"id": mid, "runs-on": labels, "enabled": enabled, "max-model-gb": limit,
"max-tier": max_tier, "changed-models": changed})
return out


def pick(machines: list[dict], only: str) -> tuple[list[dict], list[str]]:
"""The machines to run on, and the names in only that match none."""
names = [n.strip() for n in only.split(",") if n.strip()]
if not names:
return [m for m in machines if m["enabled"]], []
if names == ["none"]:
return [], []
known = {m["id"] for m in machines}
return [m for m in machines if m["id"] in names], [n for n in names if n not in known]


def tier_on(machine: dict, tier: str) -> str:
"""The tier a machine's job runs: the run's, or its max-tier if lower."""
return min(tier, machine["max-tier"], key=TIERS.index)


def matrix(machines: list[dict], tier: str, changed: str = "") -> str:
"""The job matrix's include list, on one line for $GITHUB_OUTPUT."""
return json.dumps([{"id": m["id"], "runs-on": m["runs-on"], "max-model-gb": m["max-model-gb"],
"tier": tier_on(m, tier),
"changed-models": changed if m["changed-models"] else ""}
for m in machines],
separators=(",", ":"))


def main(argv: Optional[list[str]] = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--config", type=Path, default=REPO_ROOT / CONFIG)
parser.add_argument("--tier", choices=TIERS, default="pr", help="The run's tier (default: pr).")
parser.add_argument("--changed-models", default="",
help="Comma-separated models whose catalog entries the change touched.")
parser.add_argument("--only", default="",
help='Comma-separated kinds to run instead of the enabled ones; "none" for none.')
parser.add_argument("--github-output", action="store_true",
help="Also append the matrix to $GITHUB_OUTPUT.")
args = parser.parse_args(argv)

try:
machines = load(args.config.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
print(f"::error file={CONFIG}::{exc}")
return 2
chosen, unknown = pick(machines, args.only)
for name in unknown:
print(f"::warning::FLM_HIPEP_MACHINES names {name}, which {CONFIG} does not list")
value = matrix(chosen, args.tier, args.changed_models.strip())
names = [f"{m['id']} ({tier_on(m, args.tier)} tier)" for m in chosen]
print(f"HIPEP machines: {', '.join(names) or 'none'}")
print(f"machines={value}")
if args.github_output:
target = os.environ.get("GITHUB_OUTPUT")
if not target:
print("--github-output given but $GITHUB_OUTPUT is not set", file=sys.stderr)
return 2
with open(target, "a", encoding="utf-8") as out:
out.write(f"machines={value}\n")
return 0


if __name__ == "__main__":
sys.exit(main())
Loading
Loading