From 9c18c5c163196bbffc898a08f7a9e13388bd9c74 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:07:04 -0700 Subject: [PATCH 01/43] Make NeMo preflight validate real adapter runtime --- src/avo_harness/worker.py | 128 ++++++++++++++++++++++++++++++-------- 1 file changed, 101 insertions(+), 27 deletions(-) diff --git a/src/avo_harness/worker.py b/src/avo_harness/worker.py index 6d0afb9..36c3889 100644 --- a/src/avo_harness/worker.py +++ b/src/avo_harness/worker.py @@ -99,24 +99,25 @@ def _adapter_python_env(adapter_python: str | None): ADAPTER_PYTHON. WorkerConfig.env is already overrideable per role/strategy, so treating that one key as runtime-owned lets Avo mix incompatible harness environments without introducing another configuration surface. - """ - if not adapter_python: - yield - return + ADAPTER_PYTHON lives in process-global os.environ, so every NeMo call must + participate in the same lock. Otherwise a worker without an override can run + while another worker has temporarily set ADAPTER_PYTHON and silently discover + the wrong adapter environment. + """ - # os.environ is process-global. Serialize only calls that override the - # interpreter so parallel workers cannot discover each other's adapters. with _ADAPTER_ENV_LOCK: previous = os.environ.get("ADAPTER_PYTHON") - os.environ["ADAPTER_PYTHON"] = adapter_python + if adapter_python: + os.environ["ADAPTER_PYTHON"] = adapter_python try: yield finally: - if previous is None: - os.environ.pop("ADAPTER_PYTHON", None) - else: - os.environ["ADAPTER_PYTHON"] = previous + if adapter_python: + if previous is None: + os.environ.pop("ADAPTER_PYTHON", None) + else: + os.environ["ADAPTER_PYTHON"] = previous def _nemo_failure(exc: Exception, adapter_id: str) -> str: @@ -210,22 +211,45 @@ def run(self, prompt: str, workspace: Path) -> WorkerResult: result.metadata["adapter_python"] = adapter_python return result except Exception as exc: - metadata: dict[str, Any] = { - "backend": "nemo", - "adapter_id": self.config.adapter_id, - } + return self._failure_result(exc, started, adapter_python) + + def validate(self, workspace: Path) -> WorkerResult: + """Resolve and diagnose the configured adapter without calling a model.""" + + started = time.monotonic() + adapter_python = self.config.env.get("ADAPTER_PYTHON") + try: + with _adapter_python_env(adapter_python): + result = asyncio.run(self._validate(workspace)) + result.duration_seconds = time.monotonic() - started if adapter_python: - metadata["adapter_python"] = adapter_python - return WorkerResult( - success=False, - error=_nemo_failure(exc, self.config.adapter_id), - metadata=metadata, - duration_seconds=time.monotonic() - started, - ) + result.metadata["adapter_python"] = adapter_python + return result + except Exception as exc: + return self._failure_result(exc, started, adapter_python) + + def _failure_result( + self, + exc: Exception, + started: float, + adapter_python: str | None, + ) -> WorkerResult: + metadata: dict[str, Any] = { + "backend": "nemo", + "adapter_id": self.config.adapter_id, + } + if adapter_python: + metadata["adapter_python"] = adapter_python + return WorkerResult( + success=False, + error=_nemo_failure(exc, self.config.adapter_id), + metadata=metadata, + duration_seconds=time.monotonic() - started, + ) - async def _run(self, prompt: str, workspace: Path) -> WorkerResult: + def _fabric_config(self, workspace: Path): try: - from nemo_fabric import Fabric, FabricConfig + from nemo_fabric import FabricConfig except ImportError as exc: raise RuntimeError( "NeMo Fabric is not installed. Install this project with the 'nemo' extra." @@ -287,10 +311,60 @@ async def _run(self, prompt: str, workspace: Path) -> WorkerResult: payload["telemetry"] = self.config.telemetry if hasattr(FabricConfig, "from_mapping"): - config = FabricConfig.from_mapping(payload) - else: - config = FabricConfig(**payload) + return FabricConfig.from_mapping(payload) + return FabricConfig(**payload) + + async def _validate(self, workspace: Path) -> WorkerResult: + try: + from nemo_fabric import Fabric + except ImportError as exc: + raise RuntimeError( + "NeMo Fabric is not installed. Install this project with the 'nemo' extra." + ) from exc + + config = self._fabric_config(workspace) + fabric = Fabric() + plan = fabric.plan(config) + resolved_adapter = getattr(getattr(plan, "adapter", None), "adapter_id", None) + if resolved_adapter != self.config.adapter_id: + raise RuntimeError( + f"NeMo Fabric planned adapter {resolved_adapter!r}, expected {self.config.adapter_id!r}" + ) + + # NeMo's doctor goes beyond descriptor discovery: it validates declared + # adapter/harness requirements and environment assumptions without + # starting a runtime or contacting the configured model. + report = await fabric.doctor(config) + doctor_status = str(getattr(report, "status", "unknown")) + if doctor_status != "pass": + failures = [] + for check in getattr(report, "checks", []) or []: + if str(getattr(check, "status", "")) != "fail": + continue + name = str(getattr(check, "name", "runtime")) + message = str(getattr(check, "message", "failed")) + failures.append(f"{name}: {message}") + detail = "; ".join(failures) or f"overall status {doctor_status}" + raise RuntimeError(f"NeMo Fabric doctor failed: {detail}") + + return WorkerResult( + success=True, + metadata={ + "backend": "nemo", + "adapter_id": self.config.adapter_id, + "doctor_status": doctor_status, + }, + ) + + async def _run(self, prompt: str, workspace: Path) -> WorkerResult: + try: + from nemo_fabric import Fabric + except ImportError as exc: + raise RuntimeError( + "NeMo Fabric is not installed. Install this project with the 'nemo' extra." + ) from exc + config = self._fabric_config(workspace) result = await Fabric().run(config, input=prompt) output_obj = getattr(result, "output", None) response = getattr(output_obj, "response", "") if output_obj is not None else "" From fcb22ded32b1c6c6fa360a3608f8b8e7dfbbff5a Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:07:27 -0700 Subject: [PATCH 02/43] Make doctor run NeMo adapter diagnostics --- src/avo_harness/cli.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/avo_harness/cli.py b/src/avo_harness/cli.py index 0be4486..05d8b30 100644 --- a/src/avo_harness/cli.py +++ b/src/avo_harness/cli.py @@ -12,6 +12,7 @@ from .gitops import GitRepo from .orchestrator import Orchestrator from .store import Store +from .worker import NeMoWorker def _load(path: str) -> AVOConfig: @@ -28,16 +29,26 @@ def cmd_init(args: argparse.Namespace) -> int: return 0 -def _check_worker(label: str, worker: WorkerConfig, problems: list[str]) -> None: +def _check_worker( + label: str, + worker: WorkerConfig, + workspace: Path, + problems: list[str], +) -> None: if worker.backend == "command": executable = worker.command[0] if "{" not in executable and shutil.which(executable) is None: problems.append(f"{label} executable not found: {executable}") - else: - try: - import nemo_fabric # noqa: F401 - except ImportError: - problems.append(f"{label}: nemo_fabric is not installed; install avo-harness[nemo]") + return + + # Importing nemo_fabric alone is not a usable-runtime check. The runtime can + # import while having zero discoverable adapter descriptors, which is exactly + # the failure mode that produces "available adapters: []" at execution time. + # NeMoWorker.validate resolves the configured adapter and runs Fabric.doctor + # without contacting the model. + result = NeMoWorker(worker).validate(workspace) + if not result.success: + problems.append(f"{label}: {result.error}") def cmd_doctor(args: argparse.Namespace) -> int: @@ -64,13 +75,13 @@ def cmd_doctor(args: argparse.Namespace) -> int: for name, worker in config.team.resolved_workers(config.worker).items() ) for label, worker in workers: - _check_worker(label, worker, problems) + _check_worker(label, worker, config.repo_path, problems) if problems: for item in dict.fromkeys(problems): print(f"FAIL: {item}") return 1 - print("OK: configuration and runtime prerequisites look usable") + print("OK: configuration, adapters, harnesses, and runtime prerequisites look usable") return 0 From e99343255f71ca74eb4a3af52c0162bee5fb12e1 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:07:47 -0700 Subject: [PATCH 03/43] Test real NeMo preflight and adapter env isolation --- tests/test_worker_nemo.py | 92 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/tests/test_worker_nemo.py b/tests/test_worker_nemo.py index 8ca59da..991f2a7 100644 --- a/tests/test_worker_nemo.py +++ b/tests/test_worker_nemo.py @@ -3,11 +3,13 @@ import asyncio import os import sys +import threading +import time from pathlib import Path from types import SimpleNamespace from avo_harness.config import WorkerConfig -from avo_harness.worker import NeMoWorker, _nemo_failure +from avo_harness.worker import NeMoWorker, _adapter_python_env, _nemo_failure class FakeFabricConfig: @@ -20,6 +22,16 @@ class FakeFabric: last_config = None adapter_python_seen = None + def plan(self, config): + type(self).last_config = config + return SimpleNamespace( + adapter=SimpleNamespace(adapter_id=config["harness"]["adapter_id"]) + ) + + async def doctor(self, config): + type(self).last_config = config + return SimpleNamespace(status="pass", checks=[]) + async def run(self, config, *, input): type(self).last_config = config type(self).adapter_python_seen = os.environ.get("ADAPTER_PYTHON") @@ -33,13 +45,27 @@ async def run(self, config, *, input): ) -def install_fake_fabric(monkeypatch) -> None: +class FailingDoctorFabric(FakeFabric): + async def doctor(self, config): + return SimpleNamespace( + status="fail", + checks=[ + SimpleNamespace( + status="fail", + name="adapter.requirements", + message="Hermes Agent is not installed", + ) + ], + ) + + +def install_fake_fabric(monkeypatch, fabric_class=FakeFabric) -> None: FakeFabric.last_config = None FakeFabric.adapter_python_seen = None monkeypatch.setitem( sys.modules, "nemo_fabric", - SimpleNamespace(Fabric=FakeFabric, FabricConfig=FakeFabricConfig), + SimpleNamespace(Fabric=fabric_class, FabricConfig=FakeFabricConfig), ) @@ -77,6 +103,29 @@ def test_hermes_keeps_max_turns(monkeypatch, tmp_path: Path) -> None: } +def test_validate_resolves_adapter_and_runs_doctor(monkeypatch, tmp_path: Path) -> None: + install_fake_fabric(monkeypatch) + worker = NeMoWorker(nemo_config("nvidia.fabric.hermes")) + + result = worker.validate(tmp_path) + + assert result.success is True + assert result.metadata["adapter_id"] == "nvidia.fabric.hermes" + assert result.metadata["doctor_status"] == "pass" + assert FakeFabric.last_config["harness"]["adapter_id"] == "nvidia.fabric.hermes" + + +def test_validate_reports_doctor_failures(monkeypatch, tmp_path: Path) -> None: + install_fake_fabric(monkeypatch, FailingDoctorFabric) + worker = NeMoWorker(nemo_config("nvidia.fabric.hermes")) + + result = worker.validate(tmp_path) + + assert result.success is False + assert "adapter.requirements" in result.error + assert "Hermes Agent is not installed" in result.error + + def test_adapter_python_is_scoped_to_one_worker(monkeypatch, tmp_path: Path) -> None: install_fake_fabric(monkeypatch) monkeypatch.delenv("ADAPTER_PYTHON", raising=False) @@ -97,6 +146,43 @@ def test_adapter_python_is_scoped_to_one_worker(monkeypatch, tmp_path: Path) -> assert result.metadata["adapter_python"] == adapter_python +def test_adapter_python_override_cannot_leak_to_unscoped_worker(monkeypatch) -> None: + monkeypatch.delenv("ADAPTER_PYTHON", raising=False) + override_entered = threading.Event() + unscoped_attempted = threading.Event() + release_override = threading.Event() + observed: list[str | None] = [] + + def scoped_worker() -> None: + with _adapter_python_env("/tmp/hermes-python"): + override_entered.set() + assert release_override.wait(timeout=2) + + def unscoped_worker() -> None: + assert override_entered.wait(timeout=2) + unscoped_attempted.set() + with _adapter_python_env(None): + observed.append(os.environ.get("ADAPTER_PYTHON")) + + scoped = threading.Thread(target=scoped_worker) + unscoped = threading.Thread(target=unscoped_worker) + scoped.start() + unscoped.start() + + assert unscoped_attempted.wait(timeout=2) + # Give the unscoped worker a chance to enter the context. Before the fix it + # did so immediately and observed the scoped worker's temporary interpreter. + time.sleep(0.05) + release_override.set() + scoped.join(timeout=2) + unscoped.join(timeout=2) + + assert not scoped.is_alive() + assert not unscoped.is_alive() + assert observed == [None] + assert os.environ.get("ADAPTER_PYTHON") is None + + def test_unknown_adapter_error_explains_install_and_isolation() -> None: message = _nemo_failure( RuntimeError("unknown adapter nvidia.fabric.hermes"), From ba1b1fefd4ed465d70807f19773b7b3148213832 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:08:00 -0700 Subject: [PATCH 04/43] Add reproducible Hermes source installer --- scripts/install_hermes.py | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 scripts/install_hermes.py diff --git a/scripts/install_hermes.py b/scripts/install_hermes.py new file mode 100644 index 0000000..0c02577 --- /dev/null +++ b/scripts/install_hermes.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Install the pinned Hermes Agent source into the current Python environment. + +Hermes Agent 0.20+ is intentionally not distributed on PyPI. Keep this helper as +Avo's single installation path so local development, CI, and Docker all use the +same source ref and interpreter. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +HERMES_REPOSITORY = "https://github.com/NousResearch/hermes-agent.git" +DEFAULT_HERMES_REF = "v2026.8.19" + + +def run(*args: str) -> None: + subprocess.run(args, check=True) + + +def require_supported_python() -> None: + if (3, 11) <= sys.version_info[:2] < (3, 14): + return + raise SystemExit( + "Hermes Agent requires Python 3.11 through 3.13; " + f"this interpreter is Python {sys.version_info.major}.{sys.version_info.minor}." + ) + + +def checkout(destination: Path, ref: str) -> None: + if destination.exists(): + if not (destination / ".git").is_dir(): + raise SystemExit( + f"refusing to replace non-git path used for Hermes Agent: {destination}" + ) + run("git", "-C", str(destination), "fetch", "--depth", "1", "origin", ref) + run("git", "-C", str(destination), "checkout", "--detach", "FETCH_HEAD") + return + + destination.parent.mkdir(parents=True, exist_ok=True) + run( + "git", + "clone", + "--depth", + "1", + "--branch", + ref, + HERMES_REPOSITORY, + str(destination), + ) + + +def main() -> None: + require_supported_python() + root = Path(__file__).resolve().parents[1] + destination = ( + Path(sys.argv[1]).expanduser().resolve() + if len(sys.argv) > 1 + else root / ".deps" / "hermes-agent" + ) + ref = os.environ.get("HERMES_AGENT_REF", DEFAULT_HERMES_REF).strip() + if not ref: + raise SystemExit("HERMES_AGENT_REF cannot be empty") + + checkout(destination, ref) + run(sys.executable, "-m", "pip", "install", "-e", str(destination)) + + # Fail immediately if the source checkout did not install the expected + # distribution into this exact interpreter. + run( + sys.executable, + "-c", + ( + "from importlib.metadata import version; " + "v=version('hermes-agent'); " + "print(f'Hermes Agent installed: {v}')" + ), + ) + + +if __name__ == "__main__": + main() From 72c7bc05013ce06510ea8666520d8d58604c8350 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:08:05 -0700 Subject: [PATCH 05/43] Ignore locally installed source dependencies --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 781a674..8339540 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv/ +.deps/ __pycache__/ *.py[cod] .pytest_cache/ From b1ec396ab22d0803629afb848e7343567e1af637 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:08:23 -0700 Subject: [PATCH 06/43] Use shared Hermes installer in production image --- Dockerfile | 54 +++++++++++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index d95abaa..0e27b01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,40 +13,36 @@ RUN apt-get update \ WORKDIR /app COPY pyproject.toml README.md ./ COPY src ./src -# The production image ships the default Hermes stack only. NeMo Fabric supports -# separate adapter environments through ADAPTER_PYTHON, which avoids dependency -# collisions between harnesses (for example Hermes' OpenAI pin vs Deep Agents). -# Hermes Agent 0.20+ is source-distributed and deliberately refuses wheel/sdist -# builds, so keep a pinned checkout in the image and install it editable. -RUN git clone --depth 1 --branch "${HERMES_AGENT_REF}" \ - https://github.com/NousResearch/hermes-agent.git /opt/hermes-agent \ - && python -m pip install --no-cache-dir -e /opt/hermes-agent \ +COPY scripts ./scripts +# The production image ships the default Hermes stack only. Keep the source +# checkout/install recipe shared with local development and CI so an environment +# cannot look healthy merely because the NeMo runtime itself imports. +RUN PIP_NO_CACHE_DIR=1 HERMES_AGENT_REF="${HERMES_AGENT_REF}" \ + python scripts/install_hermes.py /opt/hermes-agent \ && python -m pip install --no-cache-dir '.[web,nemo]' \ && python -m pip check -# Importing nemo_fabric alone does not prove the configured descriptor exists. -# Planning resolves the default adapter and normalized config without contacting -# a model, so an UnknownAdapter regression fails the image build. +# Exercise Avo's own no-model preflight. This resolves the configured descriptor +# and runs Fabric.doctor, which checks adapter/harness runtime requirements. RUN python - <<'PY' -from nemo_fabric import Fabric, FabricConfig +from pathlib import Path + +from avo_harness.config import WorkerConfig +from avo_harness.worker import NeMoWorker adapter_id = "nvidia.fabric.hermes" -payload = { - "metadata": {"name": "avo-image-smoke-hermes"}, - "harness": {"adapter_id": adapter_id, "settings": {}}, - "runtime": {"max_turns": 1, "timeout_seconds": 1}, - "environment": {"provider": "local", "workspace": "/tmp", "env": {}}, - "models": { - "default": { - "provider": "openai", - "model": "smoke", - "base_url": "http://127.0.0.1:1/v1", - } - }, -} -config = FabricConfig.from_mapping(payload) if hasattr(FabricConfig, "from_mapping") else FabricConfig(**payload) -plan = Fabric().plan(config) -assert plan.adapter.adapter_id == adapter_id, plan -print(f"NeMo Fabric adapter available: {adapter_id}") +worker = WorkerConfig( + backend="nemo", + adapter_id=adapter_id, + model="smoke", + provider="openai", + base_url="http://127.0.0.1:1/v1", + max_turns=1, + timeout_seconds=1, +) +result = NeMoWorker(worker).validate(Path("/tmp")) +if not result.success: + raise RuntimeError(result.error) +print(f"Avo NeMo preflight passed: {adapter_id}") PY COPY --from=ui-build /build/ui/dist /app/static From c521364eccfebaf10b641f823b0f873643ac9393 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:08:45 -0700 Subject: [PATCH 07/43] Test Avo from a clean non-editable install --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a1182c..d62ae4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,31 @@ jobs: - name: Test run: python -m pytest -q + install-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Create isolated environment + run: python -m venv "$RUNNER_TEMP/avo-install" + - name: Install pinned Hermes Agent source + run: "$RUNNER_TEMP/avo-install/bin/python" scripts/install_hermes.py "$RUNNER_TEMP/hermes-agent" + - name: Install Avo as a package + run: | + cd "$GITHUB_WORKSPACE" + "$RUNNER_TEMP/avo-install/bin/python" -m pip install '.[web,nemo]' + "$RUNNER_TEMP/avo-install/bin/python" -m pip check + - name: Verify installed CLI and default adapter preflight + run: | + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/avo-install/bin/avo-harness" --help >/dev/null + "$RUNNER_TEMP/avo-install/bin/avo-harness" init --repo "$GITHUB_WORKSPACE" --output "$RUNNER_TEMP/avo.json" + "$RUNNER_TEMP/avo-install/bin/avo-harness" doctor -c "$RUNNER_TEMP/avo.json" + ui: runs-on: ubuntu-latest defaults: @@ -40,10 +65,12 @@ jobs: image: runs-on: ubuntu-latest - needs: [test, ui] + needs: [test, install-smoke, ui] steps: - uses: actions/checkout@v4 - name: Build unified web/API image run: docker build --tag avo-code-ci . - - name: Verify NeMo Fabric runtime - run: docker run --rm --entrypoint python avo-code-ci -c "import nemo_fabric; print('NeMo Fabric available')" + - name: Verify packaged runtime + run: | + docker run --rm --entrypoint python avo-code-ci -m pip check + docker run --rm --entrypoint avo-harness avo-code-ci --help >/dev/null From 02489fb34c581870a7d5457757cc70377535818f Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:09:13 -0700 Subject: [PATCH 08/43] Document the install path CI actually verifies --- README.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6dcba65..5c5f2e1 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ export AVO_APPLE_TEAM_ID=YOUR_APPLE_TEAM_ID ### Unified Docker image -The root `Dockerfile` builds the Expo web frontend and copies it into the Python runtime image, so frontend and backend deploy together: +The root `Dockerfile` builds the Expo web frontend and copies it into the Python runtime image, so frontend and backend deploy together. It also installs the pinned Hermes Agent source plus the matching NeMo Fabric Hermes adapter, then runs Avo's adapter/harness preflight during the image build: ```bash docker build -t avo-code . @@ -123,10 +123,30 @@ The default container command expects `/data/avo.json` and serves on port `8765` ### Local development -Backend: +The generated default configuration uses NeMo Fabric's Hermes adapter. A usable local installation therefore needs **Python 3.11–3.13**, Hermes Agent 0.20+ installed from source, and the `nemo` extra. Installing only `.[web]` is not enough. + +The supported local setup is: + +```bash +python3.12 -m venv .venv +source .venv/bin/activate + +# Installs the same pinned Hermes source ref used by Docker and CI. +python scripts/install_hermes.py +python -m pip install -e '.[web,nemo,dev]' +python -m pip check + +avo-harness init --repo /path/to/repository --output avo.json +avo-harness doctor -c avo.json +``` + +`avo-harness doctor` does not merely import `nemo_fabric`. It resolves every configured NeMo adapter and runs NeMo Fabric's diagnostics without calling the model. An environment with `available adapters: []`, a missing Hermes harness, an incompatible adapter configuration, or a broken `ADAPTER_PYTHON` environment fails here instead of at the first coding request. + +`python scripts/install_hermes.py` installs into the Python interpreter used to invoke it and checks out the source under `.deps/hermes-agent` by default. Set `HERMES_AGENT_REF` only when intentionally testing a different Hermes release. For a separate harness virtualenv, install Hermes and the matching adapter there and set that worker's `env.ADAPTER_PYTHON` to the virtualenv's Python executable. + +Start the backend after `doctor` passes: ```bash -python -m pip install -e '.[web]' AVO_PUBLIC_ORIGIN=http://localhost:8765 \ AVO_AUTH_DISABLED=1 \ avo-harness web -c avo.json --host 0.0.0.0 --port 8765 From edcd97b869e7c97f1aba35fc12f019ae36f851f1 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:09:56 -0700 Subject: [PATCH 09/43] Fix clean-install workflow YAML --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d62ae4f..9197feb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,8 @@ jobs: - name: Create isolated environment run: python -m venv "$RUNNER_TEMP/avo-install" - name: Install pinned Hermes Agent source - run: "$RUNNER_TEMP/avo-install/bin/python" scripts/install_hermes.py "$RUNNER_TEMP/hermes-agent" + run: | + "$RUNNER_TEMP/avo-install/bin/python" scripts/install_hermes.py "$RUNNER_TEMP/hermes-agent" - name: Install Avo as a package run: | cd "$GITHUB_WORKSPACE" From c39cba46bcc6c5b74b56f94a54a6437797434745 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:11:32 -0700 Subject: [PATCH 10/43] Treat NeMo warnings as nonfatal and anchor paths --- src/avo_harness/worker.py | 49 ++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/avo_harness/worker.py b/src/avo_harness/worker.py index 36c3889..db9defa 100644 --- a/src/avo_harness/worker.py +++ b/src/avo_harness/worker.py @@ -324,7 +324,7 @@ async def _validate(self, workspace: Path) -> WorkerResult: config = self._fabric_config(workspace) fabric = Fabric() - plan = fabric.plan(config) + plan = fabric.plan(config, base_dir=workspace) resolved_adapter = getattr(getattr(plan, "adapter", None), "adapter_id", None) if resolved_adapter != self.config.adapter_id: raise RuntimeError( @@ -333,28 +333,35 @@ async def _validate(self, workspace: Path) -> WorkerResult: # NeMo's doctor goes beyond descriptor discovery: it validates declared # adapter/harness requirements and environment assumptions without - # starting a runtime or contacting the configured model. - report = await fabric.doctor(config) + # starting a runtime or contacting the configured model. Warnings are + # advisory; only a failed diagnostic makes the worker unusable. + report = await fabric.doctor(config, base_dir=workspace) doctor_status = str(getattr(report, "status", "unknown")) - if doctor_status != "pass": - failures = [] - for check in getattr(report, "checks", []) or []: - if str(getattr(check, "status", "")) != "fail": - continue - name = str(getattr(check, "name", "runtime")) - message = str(getattr(check, "message", "failed")) - failures.append(f"{name}: {message}") - detail = "; ".join(failures) or f"overall status {doctor_status}" + failures = [] + warnings = [] + for check in getattr(report, "checks", []) or []: + status = str(getattr(check, "status", "")) + name = str(getattr(check, "name", "runtime")) + message = str(getattr(check, "message", "")) + detail = f"{name}: {message}" if message else name + if status == "fail": + failures.append(detail) + elif status == "warn": + warnings.append(detail) + if doctor_status == "fail" or failures: + detail = "; ".join(failures) or "overall status fail" raise RuntimeError(f"NeMo Fabric doctor failed: {detail}") + if doctor_status not in {"pass", "warn"}: + raise RuntimeError(f"NeMo Fabric doctor returned unknown status {doctor_status!r}") - return WorkerResult( - success=True, - metadata={ - "backend": "nemo", - "adapter_id": self.config.adapter_id, - "doctor_status": doctor_status, - }, - ) + metadata: dict[str, Any] = { + "backend": "nemo", + "adapter_id": self.config.adapter_id, + "doctor_status": doctor_status, + } + if warnings: + metadata["doctor_warnings"] = warnings + return WorkerResult(success=True, metadata=metadata) async def _run(self, prompt: str, workspace: Path) -> WorkerResult: try: @@ -365,7 +372,7 @@ async def _run(self, prompt: str, workspace: Path) -> WorkerResult: ) from exc config = self._fabric_config(workspace) - result = await Fabric().run(config, input=prompt) + result = await Fabric().run(config, base_dir=workspace, input=prompt) output_obj = getattr(result, "output", None) response = getattr(output_obj, "response", "") if output_obj is not None else "" error_obj = getattr(result, "error", None) From bce1f63cf16044a1aa88d6677deb25ee22347398 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:12:55 -0700 Subject: [PATCH 11/43] Cover Fabric base_dir and warning diagnostics --- tests/test_worker_nemo.py | 47 +++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/tests/test_worker_nemo.py b/tests/test_worker_nemo.py index 991f2a7..72989fa 100644 --- a/tests/test_worker_nemo.py +++ b/tests/test_worker_nemo.py @@ -20,20 +20,24 @@ def from_mapping(cls, payload): class FakeFabric: last_config = None + last_base_dir = None adapter_python_seen = None - def plan(self, config): + def plan(self, config, *, base_dir=None): type(self).last_config = config + type(self).last_base_dir = base_dir return SimpleNamespace( adapter=SimpleNamespace(adapter_id=config["harness"]["adapter_id"]) ) - async def doctor(self, config): + async def doctor(self, config, *, base_dir=None): type(self).last_config = config + type(self).last_base_dir = base_dir return SimpleNamespace(status="pass", checks=[]) - async def run(self, config, *, input): + async def run(self, config, *, base_dir=None, input): type(self).last_config = config + type(self).last_base_dir = base_dir type(self).adapter_python_seen = os.environ.get("ADAPTER_PYTHON") return SimpleNamespace( output=SimpleNamespace(response=f"done: {input}"), @@ -46,7 +50,9 @@ async def run(self, config, *, input): class FailingDoctorFabric(FakeFabric): - async def doctor(self, config): + async def doctor(self, config, *, base_dir=None): + type(self).last_config = config + type(self).last_base_dir = base_dir return SimpleNamespace( status="fail", checks=[ @@ -59,8 +65,25 @@ async def doctor(self, config): ) +class WarningDoctorFabric(FakeFabric): + async def doctor(self, config, *, base_dir=None): + type(self).last_config = config + type(self).last_base_dir = base_dir + return SimpleNamespace( + status="warn", + checks=[ + SimpleNamespace( + status="warn", + name="model.connectivity", + message="model endpoint was not contacted during preflight", + ) + ], + ) + + def install_fake_fabric(monkeypatch, fabric_class=FakeFabric) -> None: FakeFabric.last_config = None + FakeFabric.last_base_dir = None FakeFabric.adapter_python_seen = None monkeypatch.setitem( sys.modules, @@ -88,6 +111,7 @@ def test_codex_omits_unsupported_max_turns(monkeypatch, tmp_path: Path) -> None: assert result.success is True assert FakeFabric.last_config["runtime"] == {"timeout_seconds": 91} + assert FakeFabric.last_base_dir == tmp_path def test_hermes_keeps_max_turns(monkeypatch, tmp_path: Path) -> None: @@ -101,6 +125,7 @@ def test_hermes_keeps_max_turns(monkeypatch, tmp_path: Path) -> None: "timeout_seconds": 91, "max_turns": 37, } + assert FakeFabric.last_base_dir == tmp_path def test_validate_resolves_adapter_and_runs_doctor(monkeypatch, tmp_path: Path) -> None: @@ -113,6 +138,7 @@ def test_validate_resolves_adapter_and_runs_doctor(monkeypatch, tmp_path: Path) assert result.metadata["adapter_id"] == "nvidia.fabric.hermes" assert result.metadata["doctor_status"] == "pass" assert FakeFabric.last_config["harness"]["adapter_id"] == "nvidia.fabric.hermes" + assert FakeFabric.last_base_dir == tmp_path def test_validate_reports_doctor_failures(monkeypatch, tmp_path: Path) -> None: @@ -126,6 +152,19 @@ def test_validate_reports_doctor_failures(monkeypatch, tmp_path: Path) -> None: assert "Hermes Agent is not installed" in result.error +def test_validate_preserves_doctor_warnings_without_failing(monkeypatch, tmp_path: Path) -> None: + install_fake_fabric(monkeypatch, WarningDoctorFabric) + worker = NeMoWorker(nemo_config("nvidia.fabric.hermes")) + + result = worker.validate(tmp_path) + + assert result.success is True + assert result.metadata["doctor_status"] == "warn" + assert result.metadata["doctor_warnings"] == [ + "model.connectivity: model endpoint was not contacted during preflight" + ] + + def test_adapter_python_is_scoped_to_one_worker(monkeypatch, tmp_path: Path) -> None: install_fake_fabric(monkeypatch) monkeypatch.delenv("ADAPTER_PYTHON", raising=False) From 5b275aa85ec5a79c82e43145c27bcb16a72fdd7d Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:13:14 -0700 Subject: [PATCH 12/43] Surface nonfatal NeMo doctor warnings --- src/avo_harness/cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/avo_harness/cli.py b/src/avo_harness/cli.py index 05d8b30..e4852f3 100644 --- a/src/avo_harness/cli.py +++ b/src/avo_harness/cli.py @@ -34,6 +34,7 @@ def _check_worker( worker: WorkerConfig, workspace: Path, problems: list[str], + warnings: list[str], ) -> None: if worker.backend == "command": executable = worker.command[0] @@ -49,11 +50,17 @@ def _check_worker( result = NeMoWorker(worker).validate(workspace) if not result.success: problems.append(f"{label}: {result.error}") + return + warnings.extend( + f"{label}: {warning}" + for warning in result.metadata.get("doctor_warnings", []) + ) def cmd_doctor(args: argparse.Namespace) -> int: config = _load(args.config) problems: list[str] = [] + warnings: list[str] = [] if shutil.which("git") is None: problems.append("git is not installed") else: @@ -75,12 +82,14 @@ def cmd_doctor(args: argparse.Namespace) -> int: for name, worker in config.team.resolved_workers(config.worker).items() ) for label, worker in workers: - _check_worker(label, worker, config.repo_path, problems) + _check_worker(label, worker, config.repo_path, problems, warnings) if problems: for item in dict.fromkeys(problems): print(f"FAIL: {item}") return 1 + for item in dict.fromkeys(warnings): + print(f"WARN: {item}") print("OK: configuration, adapters, harnesses, and runtime prerequisites look usable") return 0 From adc21e3d09acecd3648c5f33ae3eb755ce712163 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:14:40 -0700 Subject: [PATCH 13/43] Allow persistent state path during init --- src/avo_harness/cli.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/avo_harness/cli.py b/src/avo_harness/cli.py index e4852f3..98b89e7 100644 --- a/src/avo_harness/cli.py +++ b/src/avo_harness/cli.py @@ -24,7 +24,10 @@ def cmd_init(args: argparse.Namespace) -> int: if path.exists() and not args.force: print(f"refusing to overwrite {path}; use --force", file=sys.stderr) return 2 - path.write_text(json.dumps(example_config(args.repo), indent=2) + "\n", encoding="utf-8") + config = example_config(args.repo) + if args.state_dir is not None: + config["state_dir"] = args.state_dir + path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") print(path) return 0 @@ -174,6 +177,10 @@ def build_parser() -> argparse.ArgumentParser: init = sub.add_parser("init", help="write an example JSON configuration") init.add_argument("--repo", default=".") init.add_argument("--output", default="avo.json") + init.add_argument( + "--state-dir", + help="persistent state directory written into the generated configuration", + ) init.add_argument("--force", action="store_true") init.set_defaults(func=cmd_init) From 4d6ea6dce9d2cff852cd4438f33ac233cb9e2815 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:14:49 -0700 Subject: [PATCH 14/43] Test persistent state configuration during init --- tests/test_cli.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ab81746 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from avo_harness.cli import build_parser + + +def test_init_writes_explicit_state_directory(tmp_path: Path) -> None: + output = tmp_path / "avo.json" + state = tmp_path / "persistent-state" + parser = build_parser() + args = parser.parse_args( + [ + "init", + "--repo", + str(tmp_path), + "--state-dir", + str(state), + "--output", + str(output), + ] + ) + + assert args.func(args) == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["repo"] == str(tmp_path) + assert payload["state_dir"] == str(state) + assert payload["worker"]["adapter_id"] == "nvidia.fabric.hermes" From 0a7c82a94f87425b3c5c41ddd83ab92078a96caf Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:15:20 -0700 Subject: [PATCH 15/43] Document a persistent working Docker deployment --- README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5c5f2e1..348c46c 100644 --- a/README.md +++ b/README.md @@ -107,19 +107,31 @@ export AVO_APPLE_TEAM_ID=YOUR_APPLE_TEAM_ID ### Unified Docker image -The root `Dockerfile` builds the Expo web frontend and copies it into the Python runtime image, so frontend and backend deploy together. It also installs the pinned Hermes Agent source plus the matching NeMo Fabric Hermes adapter, then runs Avo's adapter/harness preflight during the image build: +The root `Dockerfile` builds the Expo web frontend and copies it into the Python runtime image, so frontend and backend deploy together. It also installs the pinned Hermes Agent source plus the matching NeMo Fabric Hermes adapter, then runs Avo's adapter/harness preflight during the image build. + +Build the image, then generate a container-valid configuration once. The repository must be mounted at the same path recorded in `avo.json`, and Avo state must live on a persistent mount so passkeys, conversations, runs, and worktree metadata survive container replacement: ```bash docker build -t avo-code . +mkdir -p /srv/avo + +docker run --rm \ + --entrypoint avo-harness \ + -v /srv/avo:/data \ + -v /path/to/repository:/workspace \ + avo-code \ + init --repo /workspace --state-dir /data/state --output /data/avo.json + docker run --rm \ -p 8765:8765 \ -v /srv/avo:/data \ + -v /path/to/repository:/workspace \ -e AVO_PUBLIC_ORIGIN=https://avo.penginlab.com \ -e AVO_APPLE_TEAM_ID=YOUR_APPLE_TEAM_ID \ avo-code ``` -The default container command expects `/data/avo.json` and serves on port `8765`. +The default container command expects `/data/avo.json` and serves on port `8765`. Mount any additional repository paths referenced by `.avo/access.yaml` as well. Do not point `state_dir` at the container's home directory for a replaceable deployment; use `/data/state` or another persisted path. ### Local development @@ -140,7 +152,7 @@ avo-harness init --repo /path/to/repository --output avo.json avo-harness doctor -c avo.json ``` -`avo-harness doctor` does not merely import `nemo_fabric`. It resolves every configured NeMo adapter and runs NeMo Fabric's diagnostics without calling the model. An environment with `available adapters: []`, a missing Hermes harness, an incompatible adapter configuration, or a broken `ADAPTER_PYTHON` environment fails here instead of at the first coding request. +`avo-harness doctor` does not merely import `nemo_fabric`. It resolves every configured NeMo adapter and runs NeMo Fabric's diagnostics without calling the model. An environment with `available adapters: []`, a missing Hermes harness, an incompatible adapter configuration, or a broken `ADAPTER_PYTHON` environment fails here instead of at the first coding request. Advisory NeMo diagnostics are printed as `WARN`; actual failed checks make `doctor` fail. `python scripts/install_hermes.py` installs into the Python interpreter used to invoke it and checks out the source under `.deps/hermes-agent` by default. Set `HERMES_AGENT_REF` only when intentionally testing a different Hermes release. For a separate harness virtualenv, install Hermes and the matching adapter there and set that worker's `env.ADAPTER_PYTHON` to the virtualenv's Python executable. From 79d9486aae3907c56bd720497e246a29fb5b56de Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:16:18 -0700 Subject: [PATCH 16/43] Smoke test installed API and production container --- .github/workflows/ci.yml | 143 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9197feb..a192c0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,8 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" cache: pip @@ -22,8 +22,8 @@ jobs: install-smoke: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.12" cache: pip @@ -42,8 +42,60 @@ jobs: run: | cd "$RUNNER_TEMP" "$RUNNER_TEMP/avo-install/bin/avo-harness" --help >/dev/null - "$RUNNER_TEMP/avo-install/bin/avo-harness" init --repo "$GITHUB_WORKSPACE" --output "$RUNNER_TEMP/avo.json" + "$RUNNER_TEMP/avo-install/bin/avo-harness" init \ + --repo "$GITHUB_WORKSPACE" \ + --state-dir "$RUNNER_TEMP/avo-state" \ + --output "$RUNNER_TEMP/avo.json" "$RUNNER_TEMP/avo-install/bin/avo-harness" doctor -c "$RUNNER_TEMP/avo.json" + - name: Boot installed API and exercise chat routes + run: | + cd "$RUNNER_TEMP" + AVO_AUTH_DISABLED=1 \ + AVO_PUBLIC_ORIGIN=http://127.0.0.1:8765 \ + "$RUNNER_TEMP/avo-install/bin/avo-harness" web \ + -c "$RUNNER_TEMP/avo.json" \ + --host 127.0.0.1 \ + --port 8765 \ + >"$RUNNER_TEMP/avo-web.log" 2>&1 & + server_pid=$! + cleanup() { + kill "$server_pid" 2>/dev/null || true + } + trap cleanup EXIT + + healthy=0 + for _ in $(seq 1 30); do + if curl -fsS http://127.0.0.1:8765/api/health >"$RUNNER_TEMP/health.json"; then + healthy=1 + break + fi + sleep 1 + done + if [ "$healthy" -ne 1 ]; then + cat "$RUNNER_TEMP/avo-web.log" + exit 1 + fi + + "$RUNNER_TEMP/avo-install/bin/python" - <<'PY' + import json + import os + from pathlib import Path + + health = json.loads(Path(os.environ["RUNNER_TEMP"], "health.json").read_text()) + assert health["ok"] is True + assert Path(health["state_dir"]) == Path(os.environ["RUNNER_TEMP"], "avo-state") + PY + curl -fsS http://127.0.0.1:8765/api/chat/conversations >"$RUNNER_TEMP/conversations.json" + "$RUNNER_TEMP/avo-install/bin/python" - <<'PY' + import json + import os + from pathlib import Path + + conversations = json.loads( + Path(os.environ["RUNNER_TEMP"], "conversations.json").read_text() + ) + assert conversations == [] + PY ui: runs-on: ubuntu-latest @@ -51,16 +103,20 @@ jobs: run: working-directory: ui steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 with: node-version: "22.13" cache: npm cache-dependency-path: ui/package.json - name: Install run: npm install + - name: Resolved direct dependencies + run: npm ls --depth=0 - name: Typecheck run: npm run typecheck + - name: Expo dependency compatibility + run: npx expo install --check - name: Export web run: npx expo export --platform web @@ -68,10 +124,81 @@ jobs: runs-on: ubuntu-latest needs: [test, install-smoke, ui] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build unified web/API image run: docker build --tag avo-code-ci . - name: Verify packaged runtime run: | docker run --rm --entrypoint python avo-code-ci -m pip check docker run --rm --entrypoint avo-harness avo-code-ci --help >/dev/null + - name: Boot production image and exercise UI/API + run: | + smoke="$RUNNER_TEMP/avo-image-smoke" + mkdir -p "$smoke/data" "$smoke/repo" + git -C "$smoke/repo" init + git -C "$smoke/repo" config user.email ci@example.invalid + git -C "$smoke/repo" config user.name "Avo CI" + printf 'smoke\n' >"$smoke/repo/README.md" + git -C "$smoke/repo" add README.md + git -C "$smoke/repo" commit -m init + + docker run --rm \ + --entrypoint avo-harness \ + -v "$smoke/data:/data" \ + -v "$smoke/repo:/workspace" \ + avo-code-ci \ + init --repo /workspace --state-dir /data/state --output /data/avo.json + + docker run -d \ + --name avo-code-smoke \ + -p 127.0.0.1:8765:8765 \ + -v "$smoke/data:/data" \ + -v "$smoke/repo:/workspace" \ + -e AVO_AUTH_DISABLED=1 \ + -e AVO_PUBLIC_ORIGIN=http://127.0.0.1:8765 \ + avo-code-ci + + cleanup() { + docker rm -f avo-code-smoke >/dev/null 2>&1 || true + } + trap cleanup EXIT + + healthy=0 + for _ in $(seq 1 30); do + if curl -fsS http://127.0.0.1:8765/api/health >"$smoke/health.json"; then + healthy=1 + break + fi + sleep 1 + done + if [ "$healthy" -ne 1 ]; then + docker logs avo-code-smoke + exit 1 + fi + + python - <<'PY' + import json + import os + from pathlib import Path + + smoke = Path(os.environ["RUNNER_TEMP"], "avo-image-smoke") + health = json.loads((smoke / "health.json").read_text()) + assert health["ok"] is True + assert health["state_dir"] == "/data/state" + assert health["static_web"] is True + PY + curl -fsS http://127.0.0.1:8765/api/chat/conversations >"$smoke/conversations.json" + python - <<'PY' + import json + import os + from pathlib import Path + + smoke = Path(os.environ["RUNNER_TEMP"], "avo-image-smoke") + assert json.loads((smoke / "conversations.json").read_text()) == [] + PY + curl -fsS http://127.0.0.1:8765/ >"$smoke/index.html" + grep -Eiq '/dev/null + trap - EXIT + test -f "$smoke/data/state/state.sqlite3" From b06b82834f170bec88bef768879121c964e058d0 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:17:37 -0700 Subject: [PATCH 17/43] Use preinstalled NeMo adapter resolution --- src/avo_harness/worker.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/avo_harness/worker.py b/src/avo_harness/worker.py index db9defa..84d8680 100644 --- a/src/avo_harness/worker.py +++ b/src/avo_harness/worker.py @@ -61,13 +61,10 @@ def _normalized_usage(value: Any) -> tuple[int | None, int | None, float | None, if not isinstance(raw, dict): return None, None, None, raw if raw is not None else None - # NeMo adapters/providers do not all use the same field names. Prefer the - # normalized names, then accept common OpenAI/LangChain aliases. input_tokens = _number(raw, "input_tokens", "prompt_tokens", "input") output_tokens = _number(raw, "output_tokens", "completion_tokens", "output") cost = _number(raw, "cost_usd", "total_cost_usd", "cost") - # Some providers wrap usage in a nested token_usage/usage object. if input_tokens is None or output_tokens is None or cost is None: for key in ("token_usage", "usage", "tokens"): nested = raw.get(key) @@ -255,18 +252,12 @@ def _fabric_config(self, workspace: Path): "NeMo Fabric is not installed. Install this project with the 'nemo' extra." ) from exc - # ADAPTER_PYTHON selects the adapter host interpreter and must be visible - # to the Fabric runtime process, not forwarded as a harness environment - # variable. Other configured variables remain harness-visible. harness_env = dict(self.config.env) harness_env.pop("ADAPTER_PYTHON", None) runtime: dict[str, Any] = { "timeout_seconds": self.config.timeout_seconds, } - # max_turns is a normalized optional capability, not a universal one. - # Codex intentionally has no mapping for it; including Avo's default of - # 24 makes Fabric reject an otherwise valid Codex configuration. if self.config.adapter_id not in _MAX_TURNS_UNSUPPORTED_ADAPTERS: runtime["max_turns"] = self.config.max_turns @@ -274,6 +265,11 @@ def _fabric_config(self, workspace: Path): "metadata": {"name": "avo-worker"}, "harness": { "adapter_id": self.config.adapter_id, + # Avo installs adapter packages up front, either in its own + # environment or in ADAPTER_PYTHON. Tell Fabric to use those + # installed descriptors explicitly instead of leaving resolution + # ambiguous and warning at runtime. + "resolution": "preinstalled", "settings": self.config.harness_settings, }, "instructions": { @@ -331,10 +327,6 @@ async def _validate(self, workspace: Path) -> WorkerResult: f"NeMo Fabric planned adapter {resolved_adapter!r}, expected {self.config.adapter_id!r}" ) - # NeMo's doctor goes beyond descriptor discovery: it validates declared - # adapter/harness requirements and environment assumptions without - # starting a runtime or contacting the configured model. Warnings are - # advisory; only a failed diagnostic makes the worker unusable. report = await fabric.doctor(config, base_dir=workspace) doctor_status = str(getattr(report, "status", "unknown")) failures = [] From 721645b620db208b4032a33207242d618c1d17f0 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:21:28 -0700 Subject: [PATCH 18/43] Validate Hermes model credentials before first run --- src/avo_harness/worker.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/avo_harness/worker.py b/src/avo_harness/worker.py index 84d8680..198d330 100644 --- a/src/avo_harness/worker.py +++ b/src/avo_harness/worker.py @@ -29,6 +29,12 @@ def run(self, prompt: str, workspace: Path) -> WorkerResult: ... "nvidia.fabric.langchain.deepagents": "nemo-fabric[deepagents]", "nvidia.fabric.mini-swe-agent": "nemo-fabric[mini-swe-agent]", } +_HERMES_PROVIDER_API_KEY_ENV = { + "anthropic": "ANTHROPIC_API_KEY", + "nvidia": "NVIDIA_API_KEY", + "openai": "OPENAI_API_KEY", + "openrouter": "OPENROUTER_API_KEY", +} def _jsonable(value: Any) -> Any: @@ -131,6 +137,36 @@ def _nemo_failure(exc: Exception, adapter_id: str) -> str: return f"NeMo Fabric worker failed: {detail}" +def _hermes_api_key_env(config: WorkerConfig) -> str | None: + if config.adapter_id != "nvidia.fabric.hermes": + return None + if config.api_key_env: + return config.api_key_env + provider = str(config.provider or "").strip().lower() + api_key_env = _HERMES_PROVIDER_API_KEY_ENV.get(provider) + if api_key_env is None: + raise RuntimeError( + "Hermes requires worker.api_key_env for provider " + f"{provider or ''!r}; Avo cannot infer the credential environment variable." + ) + return api_key_env + + +def _validate_hermes_credentials(config: WorkerConfig) -> str | None: + api_key_env = _hermes_api_key_env(config) + if api_key_env is None: + return None + value = config.env.get(api_key_env) or os.environ.get(api_key_env) + if value: + return api_key_env + raise RuntimeError( + f"Hermes requires {api_key_env} to be nonempty at runtime. Set it in the Avo process " + f"environment or worker.env. The configured provider is {config.provider!r}. " + "For an unauthenticated local OpenAI-compatible endpoint, a dummy value such as " + "'local' is sufficient; do not use a dummy value for a provider that authenticates requests." + ) + + class CommandWorker: def __init__(self, config: WorkerConfig): self.config = config @@ -346,11 +382,14 @@ async def _validate(self, workspace: Path) -> WorkerResult: if doctor_status not in {"pass", "warn"}: raise RuntimeError(f"NeMo Fabric doctor returned unknown status {doctor_status!r}") + api_key_env = _validate_hermes_credentials(self.config) metadata: dict[str, Any] = { "backend": "nemo", "adapter_id": self.config.adapter_id, "doctor_status": doctor_status, } + if api_key_env: + metadata["api_key_env"] = api_key_env if warnings: metadata["doctor_warnings"] = warnings return WorkerResult(success=True, metadata=metadata) From 7531e4c29aa7ea012d85be6f23fd6fc2ad15759f Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:21:55 -0700 Subject: [PATCH 19/43] Test Hermes credential preflight and resolution mode --- tests/test_worker_nemo.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_worker_nemo.py b/tests/test_worker_nemo.py index 72989fa..a86ce58 100644 --- a/tests/test_worker_nemo.py +++ b/tests/test_worker_nemo.py @@ -93,6 +93,7 @@ def install_fake_fabric(monkeypatch, fabric_class=FakeFabric) -> None: def nemo_config(adapter_id: str, **kwargs) -> WorkerConfig: + kwargs.setdefault("env", {"OPENAI_API_KEY": "test"}) return WorkerConfig( backend="nemo", adapter_id=adapter_id, @@ -125,6 +126,7 @@ def test_hermes_keeps_max_turns(monkeypatch, tmp_path: Path) -> None: "timeout_seconds": 91, "max_turns": 37, } + assert FakeFabric.last_config["harness"]["resolution"] == "preinstalled" assert FakeFabric.last_base_dir == tmp_path @@ -137,7 +139,9 @@ def test_validate_resolves_adapter_and_runs_doctor(monkeypatch, tmp_path: Path) assert result.success is True assert result.metadata["adapter_id"] == "nvidia.fabric.hermes" assert result.metadata["doctor_status"] == "pass" + assert result.metadata["api_key_env"] == "OPENAI_API_KEY" assert FakeFabric.last_config["harness"]["adapter_id"] == "nvidia.fabric.hermes" + assert FakeFabric.last_config["harness"]["resolution"] == "preinstalled" assert FakeFabric.last_base_dir == tmp_path @@ -165,6 +169,39 @@ def test_validate_preserves_doctor_warnings_without_failing(monkeypatch, tmp_pat ] +def test_validate_fails_before_first_turn_when_hermes_key_is_missing( + monkeypatch, + tmp_path: Path, +) -> None: + install_fake_fabric(monkeypatch) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + worker = NeMoWorker(nemo_config("nvidia.fabric.hermes", env={})) + + result = worker.validate(tmp_path) + + assert result.success is False + assert "OPENAI_API_KEY" in result.error + assert "dummy value" in result.error + + +def test_validate_accepts_explicit_hermes_api_key_env(monkeypatch, tmp_path: Path) -> None: + install_fake_fabric(monkeypatch) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + worker = NeMoWorker( + nemo_config( + "nvidia.fabric.hermes", + api_key_env="LOCAL_MODEL_TOKEN", + env={"LOCAL_MODEL_TOKEN": "local"}, + ) + ) + + result = worker.validate(tmp_path) + + assert result.success is True + assert result.metadata["api_key_env"] == "LOCAL_MODEL_TOKEN" + assert FakeFabric.last_config["models"]["default"]["api_key_env"] == "LOCAL_MODEL_TOKEN" + + def test_adapter_python_is_scoped_to_one_worker(monkeypatch, tmp_path: Path) -> None: install_fake_fabric(monkeypatch) monkeypatch.delenv("ADAPTER_PYTHON", raising=False) @@ -209,8 +246,6 @@ def unscoped_worker() -> None: unscoped.start() assert unscoped_attempted.wait(timeout=2) - # Give the unscoped worker a chance to enter the context. Before the fix it - # did so immediately and observed the scoped worker's temporary interpreter. time.sleep(0.05) release_override.set() scoped.join(timeout=2) From 6110e914bb326cadecc40746b8870250fc5bac89 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:22:13 -0700 Subject: [PATCH 20/43] Satisfy Hermes credential preflight in image build --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e27b01..d4b04b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,8 +21,10 @@ RUN PIP_NO_CACHE_DIR=1 HERMES_AGENT_REF="${HERMES_AGENT_REF}" \ python scripts/install_hermes.py /opt/hermes-agent \ && python -m pip install --no-cache-dir '.[web,nemo]' \ && python -m pip check -# Exercise Avo's own no-model preflight. This resolves the configured descriptor -# and runs Fabric.doctor, which checks adapter/harness runtime requirements. +# Exercise Avo's own no-model preflight. This resolves the configured descriptor, +# runs Fabric.doctor, and checks the credential environment Hermes will require +# before the first actual turn. The dummy key is valid only for this unreachable +# unauthenticated smoke endpoint. RUN python - <<'PY' from pathlib import Path @@ -36,6 +38,7 @@ worker = WorkerConfig( model="smoke", provider="openai", base_url="http://127.0.0.1:1/v1", + env={"OPENAI_API_KEY": "local"}, max_turns=1, timeout_seconds=1, ) From 00c1e9cd49d0ac4eb09be17a7163599ac07d9ac1 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:22:40 -0700 Subject: [PATCH 21/43] Add local OpenAI-compatible runtime smoke server --- tests/support/openai_smoke_server.py | 149 +++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 tests/support/openai_smoke_server.py diff --git a/tests/support/openai_smoke_server.py b/tests/support/openai_smoke_server.py new file mode 100644 index 0000000..a4d034a --- /dev/null +++ b/tests/support/openai_smoke_server.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Tiny OpenAI-compatible server used only by the clean-install CI smoke test.""" + +from __future__ import annotations + +import json +import sys +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +SMOKE_TEXT = "AVO_HERMES_SMOKE_OK" +MODEL = "local-open-model" + + +class Handler(BaseHTTPRequestHandler): + server_version = "AvoOpenAISmoke/1.0" + + def log_message(self, format: str, *args) -> None: + sys.stderr.write("smoke-server: " + format % args + "\n") + + def _json(self, status: int, payload: object) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + if self.path.rstrip("/") in {"/v1/models", "/models"}: + self._json( + 200, + { + "object": "list", + "data": [ + { + "id": MODEL, + "object": "model", + "created": int(time.time()), + "owned_by": "avo-ci", + } + ], + }, + ) + return + if self.path.startswith("/v1/models/"): + self._json( + 200, + { + "id": MODEL, + "object": "model", + "created": int(time.time()), + "owned_by": "avo-ci", + }, + ) + return + self._json(404, {"error": {"message": f"unknown path {self.path}"}}) + + def do_POST(self) -> None: + if self.path.rstrip("/") not in {"/v1/chat/completions", "/chat/completions"}: + self._json(404, {"error": {"message": f"unknown path {self.path}"}}) + return + + length = int(self.headers.get("Content-Length", "0")) + raw = self.rfile.read(length) if length else b"{}" + try: + request = json.loads(raw) + except json.JSONDecodeError: + self._json(400, {"error": {"message": "invalid JSON"}}) + return + + model = str(request.get("model") or MODEL) + created = int(time.time()) + if request.get("stream"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.end_headers() + chunks = [ + { + "id": "chatcmpl-avo-smoke", + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": SMOKE_TEXT}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-avo-smoke", + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + {"index": 0, "delta": {}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 4, + "total_tokens": 9, + }, + }, + ] + for chunk in chunks: + self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode("utf-8")) + self.wfile.flush() + self.wfile.write(b"data: [DONE]\n\n") + self.wfile.flush() + return + + self._json( + 200, + { + "id": "chatcmpl-avo-smoke", + "object": "chat.completion", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": SMOKE_TEXT, + "refusal": None, + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 4, + "total_tokens": 9, + }, + }, + ) + + +def main() -> None: + port = int(sys.argv[1]) if len(sys.argv) > 1 else 18080 + ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() + + +if __name__ == "__main__": + main() From a394f067e417fd824d81e3e8fbb830f63dde5c9a Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:23:20 -0700 Subject: [PATCH 22/43] Execute a real Hermes turn in clean-install CI --- .github/workflows/ci.yml | 69 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a192c0e..20c5121 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,10 +46,68 @@ jobs: --repo "$GITHUB_WORKSPACE" \ --state-dir "$RUNNER_TEMP/avo-state" \ --output "$RUNNER_TEMP/avo.json" - "$RUNNER_TEMP/avo-install/bin/avo-harness" doctor -c "$RUNNER_TEMP/avo.json" + OPENAI_API_KEY=local \ + "$RUNNER_TEMP/avo-install/bin/avo-harness" doctor -c "$RUNNER_TEMP/avo.json" + - name: Execute real NeMo Hermes turn + run: | + "$RUNNER_TEMP/avo-install/bin/python" \ + "$GITHUB_WORKSPACE/tests/support/openai_smoke_server.py" 18080 \ + >"$RUNNER_TEMP/openai-smoke.log" 2>&1 & + fixture_pid=$! + cleanup() { + kill "$fixture_pid" 2>/dev/null || true + } + trap cleanup EXIT + + ready=0 + for _ in $(seq 1 20); do + if curl -fsS http://127.0.0.1:18080/v1/models >/dev/null; then + ready=1 + break + fi + sleep 0.25 + done + if [ "$ready" -ne 1 ]; then + cat "$RUNNER_TEMP/openai-smoke.log" + exit 1 + fi + + cd "$RUNNER_TEMP" + OPENAI_API_KEY=smoke "$RUNNER_TEMP/avo-install/bin/python" - <<'PY' + import os + from pathlib import Path + + from avo_harness.config import WorkerConfig + from avo_harness.worker import NeMoWorker + + workspace = Path(os.environ["GITHUB_WORKSPACE"]) + worker = NeMoWorker( + WorkerConfig( + backend="nemo", + adapter_id="nvidia.fabric.hermes", + provider="openai", + model="local-open-model", + base_url="http://127.0.0.1:18080/v1", + env={"OPENAI_API_KEY": "smoke"}, + max_turns=1, + timeout_seconds=30, + system_prompt=( + "This is an installation smoke test. Do not use tools. " + "Return the model response directly." + ), + ) + ) + preflight = worker.validate(workspace) + assert preflight.success, preflight.error + result = worker.run("Reply exactly AVO_HERMES_SMOKE_OK", workspace) + assert result.success, result.error + assert "AVO_HERMES_SMOKE_OK" in result.output, result.output + print(result.output) + PY - name: Boot installed API and exercise chat routes run: | cd "$RUNNER_TEMP" + OPENAI_API_KEY=local \ AVO_AUTH_DISABLED=1 \ AVO_PUBLIC_ORIGIN=http://127.0.0.1:8765 \ "$RUNNER_TEMP/avo-install/bin/avo-harness" web \ @@ -149,11 +207,20 @@ jobs: avo-code-ci \ init --repo /workspace --state-dir /data/state --output /data/avo.json + docker run --rm \ + --entrypoint avo-harness \ + -e OPENAI_API_KEY=local \ + -v "$smoke/data:/data" \ + -v "$smoke/repo:/workspace" \ + avo-code-ci \ + doctor -c /data/avo.json + docker run -d \ --name avo-code-smoke \ -p 127.0.0.1:8765:8765 \ -v "$smoke/data:/data" \ -v "$smoke/repo:/workspace" \ + -e OPENAI_API_KEY=local \ -e AVO_AUTH_DISABLED=1 \ -e AVO_PUBLIC_ORIGIN=http://127.0.0.1:8765 \ avo-code-ci From 3c8d64d3d621acaa843c1ea8bf399e69d3088c22 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:24:33 -0700 Subject: [PATCH 23/43] Pin Expo-compatible UI dependencies --- ui/package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/package.json b/ui/package.json index 0c929c7..ad375be 100644 --- a/ui/package.json +++ b/ui/package.json @@ -11,19 +11,19 @@ "doctor": "expo-doctor" }, "dependencies": { - "@expo/vector-icons": "*", + "@expo/vector-icons": "15.1.1", "expo": "^57.0.17", "expo-build-properties": "~57.0.17", "expo-router": "~57.0.20", - "expo-secure-store": "*", - "expo-status-bar": "*", + "expo-secure-store": "57.0.4", + "expo-status-bar": "57.0.1", "react": "19.2.3", "react-dom": "19.2.3", "react-native": "0.86.3", "react-native-passkeys": "^0.4.2", - "react-native-safe-area-context": "*", - "react-native-screens": "*", - "react-native-web": "0.21.0" + "react-native-safe-area-context": "~5.7.0", + "react-native-screens": "~4.26.0", + "react-native-web": "^0.21.2" }, "devDependencies": { "@types/react": "~19.2.2", From 74e3ae68fd63b663a2c76458fcc8745444616647 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:26:13 -0700 Subject: [PATCH 24/43] Validate chat route without assuming an empty default thread --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20c5121..3469190 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,9 @@ jobs: conversations = json.loads( Path(os.environ["RUNNER_TEMP"], "conversations.json").read_text() ) - assert conversations == [] + assert isinstance(conversations, list) + for conversation in conversations: + assert {"id", "title", "created_at", "updated_at", "message_count"} <= set(conversation) PY ui: @@ -261,7 +263,10 @@ jobs: from pathlib import Path smoke = Path(os.environ["RUNNER_TEMP"], "avo-image-smoke") - assert json.loads((smoke / "conversations.json").read_text()) == [] + conversations = json.loads((smoke / "conversations.json").read_text()) + assert isinstance(conversations, list) + for conversation in conversations: + assert {"id", "title", "created_at", "updated_at", "message_count"} <= set(conversation) PY curl -fsS http://127.0.0.1:8765/ >"$smoke/index.html" grep -Eiq ' Date: Thu, 17 Sep 2026 14:27:55 -0700 Subject: [PATCH 25/43] Use Avo interpreter for NeMo adapter hosts by default --- src/avo_harness/worker.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/avo_harness/worker.py b/src/avo_harness/worker.py index 198d330..3c30f29 100644 --- a/src/avo_harness/worker.py +++ b/src/avo_harness/worker.py @@ -4,6 +4,7 @@ import os import shlex import subprocess +import sys import tempfile import threading import time @@ -96,12 +97,14 @@ def _normalized_usage(value: Any) -> tuple[int | None, int | None, float | None, @contextmanager def _adapter_python_env(adapter_python: str | None): - """Temporarily select an isolated NeMo adapter environment for one worker. + """Temporarily select the Python interpreter NeMo uses for adapter hosts. - NeMo Fabric discovers Python adapter descriptors and their harness through - ADAPTER_PYTHON. WorkerConfig.env is already overrideable per role/strategy, - so treating that one key as runtime-owned lets Avo mix incompatible harness - environments without introducing another configuration surface. + NeMo Fabric discovers Python adapter descriptors and launches their harnesses + through ADAPTER_PYTHON. WorkerConfig.env can override that interpreter for a + role that needs an isolated environment. Otherwise Avo must explicitly use + its own sys.executable: relying on PATH can make descriptor discovery happen + in Avo's virtualenv while the persistent adapter host launches from a system + Python that cannot import the adapter package. ADAPTER_PYTHON lives in process-global os.environ, so every NeMo call must participate in the same lock. Otherwise a worker without an override can run @@ -123,6 +126,13 @@ def _adapter_python_env(adapter_python: str | None): os.environ["ADAPTER_PYTHON"] = previous +def _effective_adapter_python(config: WorkerConfig) -> str: + """Return the interpreter Fabric should use to discover and launch adapters.""" + + configured = config.env.get("ADAPTER_PYTHON", "").strip() + return configured or sys.executable + + def _nemo_failure(exc: Exception, adapter_id: str) -> str: detail = str(exc) lowered = detail.lower().replace("_", " ") @@ -235,13 +245,12 @@ def __init__(self, config: WorkerConfig): def run(self, prompt: str, workspace: Path) -> WorkerResult: started = time.monotonic() - adapter_python = self.config.env.get("ADAPTER_PYTHON") + adapter_python = _effective_adapter_python(self.config) try: with _adapter_python_env(adapter_python): result = asyncio.run(self._run(prompt, workspace)) result.duration_seconds = time.monotonic() - started - if adapter_python: - result.metadata["adapter_python"] = adapter_python + result.metadata["adapter_python"] = adapter_python return result except Exception as exc: return self._failure_result(exc, started, adapter_python) @@ -250,13 +259,12 @@ def validate(self, workspace: Path) -> WorkerResult: """Resolve and diagnose the configured adapter without calling a model.""" started = time.monotonic() - adapter_python = self.config.env.get("ADAPTER_PYTHON") + adapter_python = _effective_adapter_python(self.config) try: with _adapter_python_env(adapter_python): result = asyncio.run(self._validate(workspace)) result.duration_seconds = time.monotonic() - started - if adapter_python: - result.metadata["adapter_python"] = adapter_python + result.metadata["adapter_python"] = adapter_python return result except Exception as exc: return self._failure_result(exc, started, adapter_python) @@ -301,10 +309,6 @@ def _fabric_config(self, workspace: Path): "metadata": {"name": "avo-worker"}, "harness": { "adapter_id": self.config.adapter_id, - # Avo installs adapter packages up front, either in its own - # environment or in ADAPTER_PYTHON. Tell Fabric to use those - # installed descriptors explicitly instead of leaving resolution - # ambiguous and warning at runtime. "resolution": "preinstalled", "settings": self.config.harness_settings, }, From 9869760f73331cda240ef5ba3fa55d0ec2487073 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:28:21 -0700 Subject: [PATCH 26/43] Test default NeMo adapter interpreter selection --- tests/test_worker_nemo.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_worker_nemo.py b/tests/test_worker_nemo.py index a86ce58..6711e06 100644 --- a/tests/test_worker_nemo.py +++ b/tests/test_worker_nemo.py @@ -140,6 +140,7 @@ def test_validate_resolves_adapter_and_runs_doctor(monkeypatch, tmp_path: Path) assert result.metadata["adapter_id"] == "nvidia.fabric.hermes" assert result.metadata["doctor_status"] == "pass" assert result.metadata["api_key_env"] == "OPENAI_API_KEY" + assert result.metadata["adapter_python"] == sys.executable assert FakeFabric.last_config["harness"]["adapter_id"] == "nvidia.fabric.hermes" assert FakeFabric.last_config["harness"]["resolution"] == "preinstalled" assert FakeFabric.last_base_dir == tmp_path @@ -202,6 +203,19 @@ def test_validate_accepts_explicit_hermes_api_key_env(monkeypatch, tmp_path: Pat assert FakeFabric.last_config["models"]["default"]["api_key_env"] == "LOCAL_MODEL_TOKEN" +def test_nemo_defaults_adapter_host_to_its_running_interpreter(monkeypatch, tmp_path: Path) -> None: + install_fake_fabric(monkeypatch) + monkeypatch.delenv("ADAPTER_PYTHON", raising=False) + worker = NeMoWorker(nemo_config("nvidia.fabric.hermes")) + + result = worker.run("fix it", tmp_path) + + assert result.success is True + assert FakeFabric.adapter_python_seen == sys.executable + assert result.metadata["adapter_python"] == sys.executable + assert os.environ.get("ADAPTER_PYTHON") is None + + def test_adapter_python_is_scoped_to_one_worker(monkeypatch, tmp_path: Path) -> None: install_fake_fabric(monkeypatch) monkeypatch.delenv("ADAPTER_PYTHON", raising=False) From d271be1e49833c9200e855606f9aa9a162e48798 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:29:35 -0700 Subject: [PATCH 27/43] Document Hermes credentials and reachable model endpoints --- README.md | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 348c46c..95d67f5 100644 --- a/README.md +++ b/README.md @@ -121,11 +121,32 @@ docker run --rm \ -v /path/to/repository:/workspace \ avo-code \ init --repo /workspace --state-dir /data/state --output /data/avo.json +``` + +The generated worker uses an OpenAI-compatible endpoint at `http://127.0.0.1:8000/v1`. Inside Docker, `127.0.0.1` means the Avo container itself. If the model server runs elsewhere, edit `/srv/avo/avo.json` so `worker.base_url` is reachable **from inside the Avo container**, for example a Compose service name, `host.docker.internal` where supported, or another routable host address. + +Hermes also requires the provider's API-key environment variable to be nonempty. For the default `provider: "openai"`, that is `OPENAI_API_KEY`. An unauthenticated local OpenAI-compatible endpoint may use a dummy nonempty value such as `local`; a provider that authenticates requests needs the real credential. + +Validate the final runtime configuration before starting the service: + +```bash +docker run --rm \ + --entrypoint avo-harness \ + -v /srv/avo:/data \ + -v /path/to/repository:/workspace \ + -e OPENAI_API_KEY=local \ + avo-code \ + doctor -c /data/avo.json +``` +Then run Avo with the same repository mounts, model credentials, and network reachability: + +```bash docker run --rm \ -p 8765:8765 \ -v /srv/avo:/data \ -v /path/to/repository:/workspace \ + -e OPENAI_API_KEY=local \ -e AVO_PUBLIC_ORIGIN=https://avo.penginlab.com \ -e AVO_APPLE_TEAM_ID=YOUR_APPLE_TEAM_ID \ avo-code @@ -149,16 +170,23 @@ python -m pip install -e '.[web,nemo,dev]' python -m pip check avo-harness init --repo /path/to/repository --output avo.json + +# For the generated OpenAI-compatible local worker. Use the real key instead +# when the configured endpoint authenticates requests. +export OPENAI_API_KEY=local avo-harness doctor -c avo.json ``` -`avo-harness doctor` does not merely import `nemo_fabric`. It resolves every configured NeMo adapter and runs NeMo Fabric's diagnostics without calling the model. An environment with `available adapters: []`, a missing Hermes harness, an incompatible adapter configuration, or a broken `ADAPTER_PYTHON` environment fails here instead of at the first coding request. Advisory NeMo diagnostics are printed as `WARN`; actual failed checks make `doctor` fail. +`avo-harness doctor` does not merely import `nemo_fabric`. It resolves every configured NeMo adapter, runs NeMo Fabric's diagnostics without calling the model, and checks Hermes' required credential environment. An environment with `available adapters: []`, a missing Hermes harness, an incompatible adapter configuration, a missing provider key, or a broken `ADAPTER_PYTHON` environment fails here instead of at the first coding request. Advisory NeMo diagnostics are printed as `WARN`; actual failed checks make `doctor` fail. + +By default Avo explicitly tells NeMo Fabric to launch adapter hosts with the same Python interpreter that is running Avo. This matters for virtualenv installs: relying on `python` from `PATH` can make discovery succeed in the Avo environment while the real adapter subprocess starts under system Python and fails to import `nemo_fabric_adapters`. If a harness genuinely needs a separate environment, install its harness and matching adapter there and set that worker's `env.ADAPTER_PYTHON` to the separate virtualenv's Python executable. -`python scripts/install_hermes.py` installs into the Python interpreter used to invoke it and checks out the source under `.deps/hermes-agent` by default. Set `HERMES_AGENT_REF` only when intentionally testing a different Hermes release. For a separate harness virtualenv, install Hermes and the matching adapter there and set that worker's `env.ADAPTER_PYTHON` to the virtualenv's Python executable. +`python scripts/install_hermes.py` installs into the Python interpreter used to invoke it and checks out the source under `.deps/hermes-agent` by default. Set `HERMES_AGENT_REF` only when intentionally testing a different Hermes release. Start the backend after `doctor` passes: ```bash +OPENAI_API_KEY=local \ AVO_PUBLIC_ORIGIN=http://localhost:8765 \ AVO_AUTH_DISABLED=1 \ avo-harness web -c avo.json --host 0.0.0.0 --port 8765 From 7b4098ddd7f01219a749f7387dec960c6243914b Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:30:10 -0700 Subject: [PATCH 28/43] Pin the validated Python and NeMo runtime contract --- pyproject.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 131529f..a1deed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,20 +7,20 @@ name = "avo-harness" version = "0.4.0" description = "Unofficial long-horizon agent orchestration inspired by NVIDIA AVO" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.11,<3.14" authors = [{name = "AVO Harness contributors"}] dependencies = ["PyYAML>=6,<7"] [project.optional-dependencies] -# `nemo-fabric` by itself installs only the runtime. The default Avo worker is -# Hermes, so the base NeMo extra must also install Hermes' adapter descriptor. -# Hermes Agent 0.20+ itself is source-installed by the production image (or must -# already be present in the environment for local installs). -nemo = ["nemo-fabric[hermes-agent]>=0.2,<0.3"] -deepagents = ["nemo-fabric[deepagents]>=0.2,<0.3"] -codex = ["nemo-fabric[codex]>=0.2,<0.3"] -claude = ["nemo-fabric[claude]>=0.2,<0.3"] -mini-swe-agent = ["nemo-fabric[mini-swe-agent]>=0.2,<0.3"] +# NeMo Fabric's adapter packages are version-coupled to the runtime. Keep every +# adapter extra on the exact runtime release exercised by clean-install CI rather +# than letting a future 0.2.x publication silently change production behavior. +# Hermes Agent 0.20+ itself is source-installed by scripts/install_hermes.py. +nemo = ["nemo-fabric[hermes-agent]==0.2.0"] +deepagents = ["nemo-fabric[deepagents]==0.2.0"] +codex = ["nemo-fabric[codex]==0.2.0"] +claude = ["nemo-fabric[claude]==0.2.0"] +mini-swe-agent = ["nemo-fabric[mini-swe-agent]==0.2.0"] web = ["fastapi>=0.115,<1", "uvicorn[standard]>=0.34,<1", "webauthn>=3,<4"] dev = ["pytest>=8", "fastapi>=0.115,<1", "httpx>=0.28,<1", "webauthn>=3,<4"] From f0e91777b4dc8a80e3014422a4a6e8e6161b16e2 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:31:36 -0700 Subject: [PATCH 29/43] Fail fast on broken worker runtime before web startup --- src/avo_harness/cli.py | 62 ++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/avo_harness/cli.py b/src/avo_harness/cli.py index 98b89e7..8f04709 100644 --- a/src/avo_harness/cli.py +++ b/src/avo_harness/cli.py @@ -60,10 +60,38 @@ def _check_worker( ) +def _configured_workers(config: AVOConfig) -> list[tuple[str, WorkerConfig]]: + workers: list[tuple[str, WorkerConfig]] = [("worker", config.worker)] + if config.planner.enabled and config.planner.worker is not None: + workers.append(("planner", config.planner.worker)) + if config.supervisor.enabled and config.supervisor.worker is not None: + workers.append(("supervisor", config.supervisor.worker)) + if config.team.enabled: + workers.extend( + (f"team.roles.{name}", worker) + for name, worker in config.team.resolved_workers(config.worker).items() + ) + return workers + + +def _check_configured_workers(config: AVOConfig) -> tuple[list[str], list[str]]: + problems: list[str] = [] + warnings: list[str] = [] + for label, worker in _configured_workers(config): + _check_worker(label, worker, config.repo_path, problems, warnings) + return list(dict.fromkeys(problems)), list(dict.fromkeys(warnings)) + + +def _print_worker_diagnostics(problems: list[str], warnings: list[str]) -> None: + for item in problems: + print(f"FAIL: {item}") + for item in warnings: + print(f"WARN: {item}") + + def cmd_doctor(args: argparse.Namespace) -> int: config = _load(args.config) problems: list[str] = [] - warnings: list[str] = [] if shutil.which("git") is None: problems.append("git is not installed") else: @@ -74,25 +102,12 @@ def cmd_doctor(args: argparse.Namespace) -> int: except Exception as exc: problems.append(str(exc)) - workers: list[tuple[str, WorkerConfig]] = [("worker", config.worker)] - if config.planner.enabled and config.planner.worker is not None: - workers.append(("planner", config.planner.worker)) - if config.supervisor.enabled and config.supervisor.worker is not None: - workers.append(("supervisor", config.supervisor.worker)) - if config.team.enabled: - workers.extend( - (f"team.roles.{name}", worker) - for name, worker in config.team.resolved_workers(config.worker).items() - ) - for label, worker in workers: - _check_worker(label, worker, config.repo_path, problems, warnings) - + worker_problems, warnings = _check_configured_workers(config) + problems.extend(worker_problems) + problems = list(dict.fromkeys(problems)) + _print_worker_diagnostics(problems, warnings) if problems: - for item in dict.fromkeys(problems): - print(f"FAIL: {item}") return 1 - for item in dict.fromkeys(warnings): - print(f"WARN: {item}") print("OK: configuration, adapters, harnesses, and runtime prerequisites look usable") return 0 @@ -148,6 +163,17 @@ def cmd_web(args: argparse.Namespace) -> int: return 2 from .api_chat import create_app + # Fail before binding a port if the configured worker stack cannot actually + # resolve its adapter/harness or lacks Hermes' required credential variable. + # This intentionally does not call the model endpoint. Connectivity remains + # an execution-time concern, but install/runtime mistakes should never hide + # behind a web server that appears healthy until its first chat request. + config = _load(args.config) + problems, warnings = _check_configured_workers(config) + _print_worker_diagnostics(problems, warnings) + if problems: + return 2 + uvicorn.run(create_app(args.config), host=args.host, port=args.port, log_level=args.log_level) return 0 From b2bbdcebf2db3ff7df16a081ac6ed1faa51f7992 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:34:07 -0700 Subject: [PATCH 30/43] Trust only active Avo repositories for Git operations --- src/avo_harness/gitops.py | 80 ++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/src/avo_harness/gitops.py b/src/avo_harness/gitops.py index fdb9b6d..1cd85b4 100644 --- a/src/avo_harness/gitops.py +++ b/src/avo_harness/gitops.py @@ -1,21 +1,65 @@ from __future__ import annotations +import os import shutil import subprocess +import tempfile +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path +from typing import Iterator class GitError(RuntimeError): pass -def _git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: - proc = subprocess.run( - ["git", "-C", str(repo), *args], - text=True, - capture_output=True, +def _git_config_value(value: str) -> str: + """Quote a value for Git's config-file syntax.""" + + escaped = ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\t", "\\t") ) + return f'"{escaped}"' + + +@contextmanager +def _trusted_repo_env(repo: Path) -> Iterator[dict[str, str]]: + """Give one Git subprocess a protected exact-path safe.directory entry. + + Host bind mounts commonly appear to be owned by a different UID inside a + container, which makes modern Git reject them before Avo can even inspect the + repository. Writing `safe.directory=*` would disable that protection globally. + Instead, each Avo Git command receives a short-lived global config containing + only the exact path it was asked to operate on. The user's normal environment + remains untouched after the subprocess exits. + """ + + trusted = repo.resolve() + env = os.environ.copy() + with tempfile.TemporaryDirectory(prefix="avo-git-config-") as directory: + config = Path(directory) / "gitconfig" + config.write_text( + "[safe]\n" + f"\tdirectory = {_git_config_value(str(trusted))}\n", + encoding="utf-8", + ) + env["GIT_CONFIG_GLOBAL"] = str(config) + yield env + + +def _git(repo: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + repo = repo.resolve() + with _trusted_repo_env(repo) as env: + proc = subprocess.run( + ["git", "-C", str(repo), *args], + text=True, + capture_output=True, + env=env, + ) if check and proc.returncode != 0: raise GitError(proc.stderr.strip() or proc.stdout.strip() or "git command failed") return proc @@ -70,24 +114,16 @@ def commit_all(self, worktree: Worktree, message: str) -> str: if not status: return _git(worktree.path, "rev-parse", "HEAD").stdout.strip() _git(worktree.path, "add", "-A") - proc = subprocess.run( - [ - "git", - "-C", - str(worktree.path), - "-c", - "user.name=AVO Harness", - "-c", - "user.email=avo-harness@localhost", - "commit", - "-m", - message, - ], - text=True, - capture_output=True, + _git( + worktree.path, + "-c", + "user.name=AVO Harness", + "-c", + "user.email=avo-harness@localhost", + "commit", + "-m", + message, ) - if proc.returncode != 0: - raise GitError(proc.stderr.strip() or proc.stdout.strip()) return _git(worktree.path, "rev-parse", "HEAD").stdout.strip() def remove_worktree(self, worktree: Worktree) -> None: From fb554ee38fc3a236c3d83000cc1da7ba81d186ff Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:34:18 -0700 Subject: [PATCH 31/43] Test exact-path Git trust configuration --- tests/test_gitops.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/test_gitops.py diff --git a/tests/test_gitops.py b/tests/test_gitops.py new file mode 100644 index 0000000..085941b --- /dev/null +++ b/tests/test_gitops.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from avo_harness.gitops import _git + + +def test_git_uses_ephemeral_exact_safe_directory(monkeypatch, tmp_path: Path) -> None: + repo = tmp_path / 'repo with spaces' + repo.mkdir() + observed: dict[str, str] = {} + + def fake_run(command, *, text, capture_output, env): + config_path = Path(env['GIT_CONFIG_GLOBAL']) + observed['config'] = config_path.read_text(encoding='utf-8') + observed['global_path'] = str(config_path) + observed['command'] = ' '.join(command) + assert config_path.exists() + return subprocess.CompletedProcess(command, 0, stdout='true\n', stderr='') + + monkeypatch.setattr(subprocess, 'run', fake_run) + original_global = os.environ.get('GIT_CONFIG_GLOBAL') + + result = _git(repo, 'rev-parse', '--is-inside-work-tree') + + assert result.stdout.strip() == 'true' + assert f'directory = "{repo.resolve()}"' in observed['config'] + assert 'directory = "*"' not in observed['config'] + assert observed['command'].startswith(f'git -C {repo.resolve()}') + assert not Path(observed['global_path']).exists() + assert os.environ.get('GIT_CONFIG_GLOBAL') == original_global From 2caa35c55db5bbff0d8fb6624672cd93f9c81693 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:37:04 -0700 Subject: [PATCH 32/43] Smoke test every supported NeMo adapter extra --- .github/workflows/adapters.yml | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/adapters.yml diff --git a/.github/workflows/adapters.yml b/.github/workflows/adapters.yml new file mode 100644 index 0000000..1123af1 --- /dev/null +++ b/.github/workflows/adapters.yml @@ -0,0 +1,71 @@ +name: Avo Adapter Matrix + +on: + push: + pull_request: + +jobs: + adapter-discovery: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - extra: codex + adapter: nvidia.fabric.codex + - extra: claude + adapter: nvidia.fabric.claude + - extra: deepagents + adapter: nvidia.fabric.langchain.deepagents + - extra: mini-swe-agent + adapter: nvidia.fabric.mini-swe-agent + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Create isolated environment + run: python -m venv "$RUNNER_TEMP/avo-adapter" + - name: Install Avo and adapter + env: + AVO_ADAPTER_EXTRA: ${{ matrix.extra }} + run: | + "$RUNNER_TEMP/avo-adapter/bin/python" -m pip install ".[${AVO_ADAPTER_EXTRA}]" + "$RUNNER_TEMP/avo-adapter/bin/python" -m pip check + - name: Resolve and plan installed adapter + env: + AVO_ADAPTER_ID: ${{ matrix.adapter }} + run: | + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/avo-adapter/bin/python" - <<'PY' + import os + import sys + from pathlib import Path + + from nemo_fabric import Fabric + + from avo_harness.config import WorkerConfig + from avo_harness.worker import NeMoWorker, _adapter_python_env + + adapter_id = os.environ["AVO_ADAPTER_ID"] + workspace = Path(os.environ["GITHUB_WORKSPACE"]) + config = WorkerConfig( + backend="nemo", + adapter_id=adapter_id, + provider="openai", + model="smoke", + api_key_env="OPENAI_API_KEY", + base_url="http://127.0.0.1:1/v1", + env={"OPENAI_API_KEY": "smoke"}, + timeout_seconds=5, + ) + worker = NeMoWorker(config) + with _adapter_python_env(sys.executable): + fabric_config = worker._fabric_config(workspace) + plan = Fabric().plan(fabric_config, base_dir=workspace) + resolved = getattr(getattr(plan, "adapter", None), "adapter_id", None) + assert resolved == adapter_id, (resolved, adapter_id) + print(f"resolved {adapter_id} with {sys.executable}") + PY From 1f900ca36d314dcd53bdcd3b4f12ec24840ddd5d Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:39:17 -0700 Subject: [PATCH 33/43] Respect Deep Agents max-turn capability --- src/avo_harness/worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/avo_harness/worker.py b/src/avo_harness/worker.py index 3c30f29..d643145 100644 --- a/src/avo_harness/worker.py +++ b/src/avo_harness/worker.py @@ -22,7 +22,10 @@ def run(self, prompt: str, workspace: Path) -> WorkerResult: ... _ADAPTER_ENV_LOCK = threading.Lock() -_MAX_TURNS_UNSUPPORTED_ADAPTERS = {"nvidia.fabric.codex"} +_MAX_TURNS_UNSUPPORTED_ADAPTERS = { + "nvidia.fabric.codex", + "nvidia.fabric.langchain.deepagents", +} _ADAPTER_INSTALL_HINTS = { "nvidia.fabric.hermes": "nemo-fabric[hermes-agent] plus Hermes Agent 0.20+ from source", "nvidia.fabric.codex": "nemo-fabric[codex]", From 331a1925f4c412dc327c222d1428c4358c308c35 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:39:26 -0700 Subject: [PATCH 34/43] Lock Deep Agents runtime capability regression --- tests/test_worker_capabilities.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/test_worker_capabilities.py diff --git a/tests/test_worker_capabilities.py b/tests/test_worker_capabilities.py new file mode 100644 index 0000000..65072ac --- /dev/null +++ b/tests/test_worker_capabilities.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace + +from avo_harness.config import WorkerConfig +from avo_harness.worker import NeMoWorker + + +class FakeFabricConfig: + @classmethod + def from_mapping(cls, payload): + return payload + + +def test_deepagents_omits_unsupported_max_turns(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setitem(sys.modules, "nemo_fabric", SimpleNamespace(FabricConfig=FakeFabricConfig)) + worker = NeMoWorker( + WorkerConfig( + backend="nemo", + adapter_id="nvidia.fabric.langchain.deepagents", + provider="openai", + model="smoke", + max_turns=37, + timeout_seconds=91, + ) + ) + + payload = worker._fabric_config(tmp_path) + + assert payload["runtime"] == {"timeout_seconds": 91} From d7650399f1c6999eeff5b062ba80b78df27385de Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:40:11 -0700 Subject: [PATCH 35/43] Exercise installed chat flow end to end --- .github/workflows/ci.yml | 71 ++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3469190..f3c61a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,10 +104,30 @@ jobs: assert "AVO_HERMES_SMOKE_OK" in result.output, result.output print(result.output) PY - - name: Boot installed API and exercise chat routes + - name: Boot installed API and complete a chat message run: | + "$RUNNER_TEMP/avo-install/bin/python" \ + "$GITHUB_WORKSPACE/tests/support/openai_smoke_server.py" 18080 \ + >"$RUNNER_TEMP/openai-chat-smoke.log" 2>&1 & + fixture_pid=$! + + "$RUNNER_TEMP/avo-install/bin/python" - <<'PY' + import json + import os + from pathlib import Path + + path = Path(os.environ["RUNNER_TEMP"], "avo.json") + config = json.loads(path.read_text()) + config["worker"]["provider"] = "openai" + config["worker"]["model"] = "local-open-model" + config["worker"]["base_url"] = "http://127.0.0.1:18080/v1" + config["worker"]["max_turns"] = 1 + config["worker"]["timeout_seconds"] = 30 + path.write_text(json.dumps(config, indent=2) + "\n") + PY + cd "$RUNNER_TEMP" - OPENAI_API_KEY=local \ + OPENAI_API_KEY=smoke \ AVO_AUTH_DISABLED=1 \ AVO_PUBLIC_ORIGIN=http://127.0.0.1:8765 \ "$RUNNER_TEMP/avo-install/bin/avo-harness" web \ @@ -118,6 +138,7 @@ jobs: server_pid=$! cleanup() { kill "$server_pid" 2>/dev/null || true + kill "$fixture_pid" 2>/dev/null || true } trap cleanup EXIT @@ -131,30 +152,54 @@ jobs: done if [ "$healthy" -ne 1 ]; then cat "$RUNNER_TEMP/avo-web.log" + cat "$RUNNER_TEMP/openai-chat-smoke.log" exit 1 fi "$RUNNER_TEMP/avo-install/bin/python" - <<'PY' import json import os + import time + import urllib.request from pathlib import Path - health = json.loads(Path(os.environ["RUNNER_TEMP"], "health.json").read_text()) + root = Path(os.environ["RUNNER_TEMP"]) + health = json.loads((root / "health.json").read_text()) assert health["ok"] is True - assert Path(health["state_dir"]) == Path(os.environ["RUNNER_TEMP"], "avo-state") - PY - curl -fsS http://127.0.0.1:8765/api/chat/conversations >"$RUNNER_TEMP/conversations.json" - "$RUNNER_TEMP/avo-install/bin/python" - <<'PY' - import json - import os - from pathlib import Path + assert Path(health["state_dir"]) == root / "avo-state" - conversations = json.loads( - Path(os.environ["RUNNER_TEMP"], "conversations.json").read_text() - ) + with urllib.request.urlopen("http://127.0.0.1:8765/api/chat/conversations") as response: + conversations = json.load(response) assert isinstance(conversations, list) for conversation in conversations: assert {"id", "title", "created_at", "updated_at", "message_count"} <= set(conversation) + + request = urllib.request.Request( + "http://127.0.0.1:8765/api/chat/messages", + data=json.dumps( + {"content": "Return a short installation smoke-test answer.", "auto_execute": False} + ).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request) as response: + user_message = json.load(response) + assert user_message["role"] == "user" + + for _ in range(60): + with urllib.request.urlopen("http://127.0.0.1:8765/api/chat/messages?limit=100") as response: + messages = json.load(response) + assistants = [item for item in messages if item.get("role") == "assistant"] + if assistants: + latest = assistants[-1] + if latest.get("status") == "error": + raise AssertionError(latest) + if latest.get("status") == "complete": + assert "AVO_HERMES_SMOKE_OK" in latest.get("content", ""), latest + break + time.sleep(0.5) + else: + raise AssertionError(f"assistant message did not complete: {messages!r}") PY ui: From b4a74d9e78e5db8320255f31298e22133c1bc464 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:40:42 -0700 Subject: [PATCH 36/43] Add runtime-only NeMo Fabric extra for isolated adapters --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a1deed8..18d7d1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = ["PyYAML>=6,<7"] # adapter extra on the exact runtime release exercised by clean-install CI rather # than letting a future 0.2.x publication silently change production behavior. # Hermes Agent 0.20+ itself is source-installed by scripts/install_hermes.py. +fabric = ["nemo-fabric==0.2.0"] nemo = ["nemo-fabric[hermes-agent]==0.2.0"] deepagents = ["nemo-fabric[deepagents]==0.2.0"] codex = ["nemo-fabric[codex]==0.2.0"] From 2d3ff87026230e87e0edeb2dce448e086f3a0ae8 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:41:08 -0700 Subject: [PATCH 37/43] Prove Hermes works from a separate adapter virtualenv --- .github/workflows/adapters.yml | 79 ++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/.github/workflows/adapters.yml b/.github/workflows/adapters.yml index 1123af1..171860a 100644 --- a/.github/workflows/adapters.yml +++ b/.github/workflows/adapters.yml @@ -69,3 +69,82 @@ jobs: assert resolved == adapter_id, (resolved, adapter_id) print(f"resolved {adapter_id} with {sys.executable}") PY + + isolated-hermes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Create separate Avo and Hermes environments + run: | + python -m venv "$RUNNER_TEMP/avo-main" + python -m venv "$RUNNER_TEMP/hermes-adapter" + - name: Install Avo runtime without the Hermes adapter + run: | + "$RUNNER_TEMP/avo-main/bin/python" -m pip install '.[fabric]' + "$RUNNER_TEMP/avo-main/bin/python" -m pip check + - name: Install Hermes only in the adapter environment + run: | + "$RUNNER_TEMP/hermes-adapter/bin/python" -m pip install 'nemo-fabric[hermes-agent]==0.2.0' + "$RUNNER_TEMP/hermes-adapter/bin/python" scripts/install_hermes.py "$RUNNER_TEMP/hermes-agent" + "$RUNNER_TEMP/hermes-adapter/bin/python" -m pip check + - name: Run Hermes through ADAPTER_PYTHON + run: | + "$RUNNER_TEMP/avo-main/bin/python" \ + "$GITHUB_WORKSPACE/tests/support/openai_smoke_server.py" 18080 \ + >"$RUNNER_TEMP/isolated-hermes-smoke.log" 2>&1 & + fixture_pid=$! + cleanup() { + kill "$fixture_pid" 2>/dev/null || true + } + trap cleanup EXIT + + ready=0 + for _ in $(seq 1 20); do + if curl -fsS http://127.0.0.1:18080/v1/models >/dev/null; then + ready=1 + break + fi + sleep 0.25 + done + test "$ready" -eq 1 + + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/avo-main/bin/python" - <<'PY' + import os + from pathlib import Path + + from avo_harness.config import WorkerConfig + from avo_harness.worker import NeMoWorker + + workspace = Path(os.environ["GITHUB_WORKSPACE"]) + adapter_python = str(Path(os.environ["RUNNER_TEMP"], "hermes-adapter", "bin", "python")) + worker = NeMoWorker( + WorkerConfig( + backend="nemo", + adapter_id="nvidia.fabric.hermes", + provider="openai", + model="local-open-model", + base_url="http://127.0.0.1:18080/v1", + env={ + "ADAPTER_PYTHON": adapter_python, + "OPENAI_API_KEY": "smoke", + }, + max_turns=1, + timeout_seconds=30, + system_prompt="Do not use tools. Return the model response directly.", + ) + ) + preflight = worker.validate(workspace) + assert preflight.success, preflight.error + assert preflight.metadata["adapter_python"] == adapter_python + result = worker.run("Reply exactly AVO_HERMES_SMOKE_OK", workspace) + assert result.success, result.error + assert result.metadata["adapter_python"] == adapter_python + assert "AVO_HERMES_SMOKE_OK" in result.output, result.output + print(result.output) + PY From d156e344eee7693539e931d3bcc57fa696dadade Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:42:07 -0700 Subject: [PATCH 38/43] Install Hermes as a normal source-built package --- scripts/install_hermes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/install_hermes.py b/scripts/install_hermes.py index 0c02577..f9a5375 100644 --- a/scripts/install_hermes.py +++ b/scripts/install_hermes.py @@ -66,7 +66,10 @@ def main() -> None: raise SystemExit("HERMES_AGENT_REF cannot be empty") checkout(destination, ref) - run(sys.executable, "-m", "pip", "install", "-e", str(destination)) + # Build/install from the pinned checkout, but do not leave the runtime tied + # to that source directory. A normal install remains usable if the checkout + # is later cleaned up or replaced. + run(sys.executable, "-m", "pip", "install", str(destination)) # Fail immediately if the source checkout did not install the expected # distribution into this exact interpreter. From 7e2887106067f8d480bcb05d9df7808c42baf721 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:44:01 -0700 Subject: [PATCH 39/43] Preserve Hermes source checkout for supported editable install --- scripts/install_hermes.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/install_hermes.py b/scripts/install_hermes.py index f9a5375..7b92544 100644 --- a/scripts/install_hermes.py +++ b/scripts/install_hermes.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Install the pinned Hermes Agent source into the current Python environment. -Hermes Agent 0.20+ is intentionally not distributed on PyPI. Keep this helper as -Avo's single installation path so local development, CI, and Docker all use the -same source ref and interpreter. +Hermes Agent 0.20+ intentionally does not support wheel/sdist builds. Keep this +helper as Avo's single installation path so local development, CI, and Docker all +use the same source ref, interpreter, and durable editable checkout. """ from __future__ import annotations @@ -66,10 +66,11 @@ def main() -> None: raise SystemExit("HERMES_AGENT_REF cannot be empty") checkout(destination, ref) - # Build/install from the pinned checkout, but do not leave the runtime tied - # to that source directory. A normal install remains usable if the checkout - # is later cleaned up or replaced. - run(sys.executable, "-m", "pip", "install", str(destination)) + # Hermes 0.20.x deliberately refuses wheel/sdist builds and supports source + # installation through an editable checkout. This checkout is therefore a + # runtime dependency: Docker keeps it under /opt, and local installs keep it + # under .deps unless the caller provides another durable destination. + run(sys.executable, "-m", "pip", "install", "-e", str(destination)) # Fail immediately if the source checkout did not install the expected # distribution into this exact interpreter. @@ -82,6 +83,7 @@ def main() -> None: "print(f'Hermes Agent installed: {v}')" ), ) + print(f"Hermes Agent source checkout (required at runtime): {destination}") if __name__ == "__main__": From dbdc188a4ce94e6c6f499e9651fdb7d030260fe8 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:44:49 -0700 Subject: [PATCH 40/43] Document durable Hermes and isolated adapter installs --- README.md | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95d67f5..5f85cd2 100644 --- a/README.md +++ b/README.md @@ -179,9 +179,37 @@ avo-harness doctor -c avo.json `avo-harness doctor` does not merely import `nemo_fabric`. It resolves every configured NeMo adapter, runs NeMo Fabric's diagnostics without calling the model, and checks Hermes' required credential environment. An environment with `available adapters: []`, a missing Hermes harness, an incompatible adapter configuration, a missing provider key, or a broken `ADAPTER_PYTHON` environment fails here instead of at the first coding request. Advisory NeMo diagnostics are printed as `WARN`; actual failed checks make `doctor` fail. -By default Avo explicitly tells NeMo Fabric to launch adapter hosts with the same Python interpreter that is running Avo. This matters for virtualenv installs: relying on `python` from `PATH` can make discovery succeed in the Avo environment while the real adapter subprocess starts under system Python and fails to import `nemo_fabric_adapters`. If a harness genuinely needs a separate environment, install its harness and matching adapter there and set that worker's `env.ADAPTER_PYTHON` to the separate virtualenv's Python executable. +By default Avo explicitly tells NeMo Fabric to launch adapter hosts with the same Python interpreter that is running Avo. This matters for virtualenv installs: relying on `python` from `PATH` can make discovery succeed in the Avo environment while the real adapter subprocess starts under system Python and fails to import `nemo_fabric_adapters`. -`python scripts/install_hermes.py` installs into the Python interpreter used to invoke it and checks out the source under `.deps/hermes-agent` by default. Set `HERMES_AGENT_REF` only when intentionally testing a different Hermes release. +Hermes 0.20.x deliberately refuses wheel/sdist builds. `python scripts/install_hermes.py` therefore performs Hermes' supported editable source install into the interpreter used to invoke it. The default checkout is `.deps/hermes-agent`; **that checkout is a runtime dependency and must not be deleted while the environment uses it**. Docker keeps the corresponding checkout under `/opt/hermes-agent`. Set `HERMES_AGENT_REF` only when intentionally testing a different Hermes release. + +If Hermes or another harness needs a separate dependency environment, keep Avo's main environment on the Fabric runtime and put the harness plus matching adapter in a second virtualenv: + +```bash +# Avo environment: Fabric runtime, web app, and Avo itself. +python3.12 -m venv .venv +.venv/bin/python -m pip install -e '.[web,fabric,dev]' + +# Hermes adapter environment: adapter package plus the pinned Hermes source. +python3.12 -m venv .venv-hermes +.venv-hermes/bin/python -m pip install 'nemo-fabric[hermes-agent]==0.2.0' +.venv-hermes/bin/python scripts/install_hermes.py .deps/hermes-agent-isolated +``` + +Then set the relevant worker in `avo.json` to the second interpreter while leaving model credentials alongside it or in Avo's process environment: + +```json +{ + "backend": "nemo", + "adapter_id": "nvidia.fabric.hermes", + "env": { + "ADAPTER_PYTHON": "/absolute/path/to/.venv-hermes/bin/python", + "OPENAI_API_KEY": "local" + } +} +``` + +Avo's CI executes a real Hermes turn using exactly this split-environment arrangement, so this is a tested deployment path rather than a fallback guess. Start the backend after `doctor` passes: From ee3b66b35931c744623e2dcb804943fc17823521 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:46:29 -0700 Subject: [PATCH 41/43] Fix chat acceptance smoke and avoid duplicate PR runs --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3c61a6..c121df0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: AVO Harness on: push: + branches: [main] pull_request: jobs: @@ -183,8 +184,9 @@ jobs: method="POST", ) with urllib.request.urlopen(request) as response: - user_message = json.load(response) - assert user_message["role"] == "user" + accepted = json.load(response) + assert accepted["accepted"] is True, accepted + assert accepted["conversation_id"] == "main", accepted for _ in range(60): with urllib.request.urlopen("http://127.0.0.1:8765/api/chat/messages?limit=100") as response: From 84d3c7868a007c92c670fc4b1992bd4b74d767f2 Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Thu, 17 Sep 2026 14:46:46 -0700 Subject: [PATCH 42/43] Avoid duplicate adapter matrix runs on PR branches --- .github/workflows/adapters.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/adapters.yml b/.github/workflows/adapters.yml index 171860a..f333cfb 100644 --- a/.github/workflows/adapters.yml +++ b/.github/workflows/adapters.yml @@ -2,6 +2,7 @@ name: Avo Adapter Matrix on: push: + branches: [main] pull_request: jobs: From 061e2977db7785d4f65030fa213bd3f865e72bda Mon Sep 17 00:00:00 2001 From: Kaitlyn Concilio Date: Fri, 18 Sep 2026 15:28:39 -0700 Subject: [PATCH 43/43] Harden Hermes install and Git subprocesses --- .github/workflows/adapters.yml | 5 ++- scripts/install_hermes.py | 11 +++++++ src/avo_harness/gitops.py | 10 ++++++ tests/test_gitops.py | 24 ++++++++++++++ tests/test_install_hermes.py | 58 ++++++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 tests/test_install_hermes.py diff --git a/.github/workflows/adapters.yml b/.github/workflows/adapters.yml index f333cfb..2082c75 100644 --- a/.github/workflows/adapters.yml +++ b/.github/workflows/adapters.yml @@ -112,7 +112,10 @@ jobs: fi sleep 0.25 done - test "$ready" -eq 1 + if [ "$ready" -ne 1 ]; then + cat "$RUNNER_TEMP/isolated-hermes-smoke.log" + exit 1 + fi cd "$RUNNER_TEMP" "$RUNNER_TEMP/avo-main/bin/python" - <<'PY' diff --git a/scripts/install_hermes.py b/scripts/install_hermes.py index 7b92544..a5a8c3e 100644 --- a/scripts/install_hermes.py +++ b/scripts/install_hermes.py @@ -36,6 +36,17 @@ def checkout(destination: Path, ref: str) -> None: raise SystemExit( f"refusing to replace non-git path used for Hermes Agent: {destination}" ) + origin = subprocess.run( + ("git", "-C", str(destination), "remote", "get-url", "origin"), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if origin != HERMES_REPOSITORY: + raise SystemExit( + f"refusing to reuse Hermes Agent checkout with unexpected origin: " + f"{destination} tracks {origin!r}, expected {HERMES_REPOSITORY!r}" + ) run("git", "-C", str(destination), "fetch", "--depth", "1", "origin", ref) run("git", "-C", str(destination), "checkout", "--detach", "FETCH_HEAD") return diff --git a/src/avo_harness/gitops.py b/src/avo_harness/gitops.py index 1cd85b4..00c58fb 100644 --- a/src/avo_harness/gitops.py +++ b/src/avo_harness/gitops.py @@ -40,6 +40,16 @@ def _trusted_repo_env(repo: Path) -> Iterator[dict[str, str]]: trusted = repo.resolve() env = os.environ.copy() + for key in ( + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_PREFIX", + ): + env.pop(key, None) with tempfile.TemporaryDirectory(prefix="avo-git-config-") as directory: config = Path(directory) / "gitconfig" config.write_text( diff --git a/tests/test_gitops.py b/tests/test_gitops.py index 085941b..b6b897e 100644 --- a/tests/test_gitops.py +++ b/tests/test_gitops.py @@ -31,3 +31,27 @@ def fake_run(command, *, text, capture_output, env): assert observed['command'].startswith(f'git -C {repo.resolve()}') assert not Path(observed['global_path']).exists() assert os.environ.get('GIT_CONFIG_GLOBAL') == original_global + + +def test_git_strips_inherited_repository_location_variables(monkeypatch, tmp_path: Path) -> None: + repo = tmp_path / 'target' + repo.mkdir() + location_variables = ( + 'GIT_DIR', + 'GIT_WORK_TREE', + 'GIT_COMMON_DIR', + 'GIT_INDEX_FILE', + 'GIT_OBJECT_DIRECTORY', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_PREFIX', + ) + for key in location_variables: + monkeypatch.setenv(key, f'/inherited/{key.lower()}') + + def fake_run(command, *, text, capture_output, env): + assert all(key not in env for key in location_variables) + return subprocess.CompletedProcess(command, 0, stdout='true\n', stderr='') + + monkeypatch.setattr(subprocess, 'run', fake_run) + + assert _git(repo, 'rev-parse', '--is-inside-work-tree').stdout.strip() == 'true' diff --git a/tests/test_install_hermes.py b/tests/test_install_hermes.py new file mode 100644 index 0000000..f10b3b2 --- /dev/null +++ b/tests/test_install_hermes.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import importlib.util +import subprocess +from pathlib import Path + +import pytest + + +def load_installer(): + path = Path(__file__).parents[1] / 'scripts' / 'install_hermes.py' + spec = importlib.util.spec_from_file_location('install_hermes', path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_checkout_rejects_existing_repository_with_wrong_origin(monkeypatch, tmp_path: Path) -> None: + installer = load_installer() + destination = tmp_path / 'hermes-agent' + (destination / '.git').mkdir(parents=True) + commands: list[tuple[str, ...]] = [] + + def fake_run(command, **kwargs): + command = tuple(command) + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout='https://example.invalid/not-hermes.git\n') + + monkeypatch.setattr(subprocess, 'run', fake_run) + + with pytest.raises(SystemExit, match='unexpected origin'): + installer.checkout(destination, installer.DEFAULT_HERMES_REF) + + assert commands == [('git', '-C', str(destination), 'remote', 'get-url', 'origin')] + + +def test_checkout_fetches_existing_repository_with_expected_origin(monkeypatch, tmp_path: Path) -> None: + installer = load_installer() + destination = tmp_path / 'hermes-agent' + (destination / '.git').mkdir(parents=True) + commands: list[tuple[str, ...]] = [] + + def fake_run(command, **kwargs): + command = tuple(command) + commands.append(command) + stdout = installer.HERMES_REPOSITORY + '\n' if command[-3:] == ('remote', 'get-url', 'origin') else '' + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(subprocess, 'run', fake_run) + + installer.checkout(destination, installer.DEFAULT_HERMES_REF) + + assert commands == [ + ('git', '-C', str(destination), 'remote', 'get-url', 'origin'), + ('git', '-C', str(destination), 'fetch', '--depth', '1', 'origin', installer.DEFAULT_HERMES_REF), + ('git', '-C', str(destination), 'checkout', '--detach', 'FETCH_HEAD'), + ]