diff --git a/docker/Dockerfile b/docker/Dockerfile index 9048741d..20ade719 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -34,13 +34,16 @@ ARG CLAUDE_CODE_VERSION=2.1.281 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* \ - && npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} + && npm install -g --engine-strict @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} # Pi Node CLI, pinned — same rationale as the Claude Code pin above (the agent # binary is a dominant non-model driver of results; @latest would freeze # nondeterministically under layer caching). Node 22 + npm are already present. +# --engine-strict (here and in docker/Dockerfile.runtime) turns a CLI pin whose +# `engines.node` floor is above the installed Node into a build failure instead +# of an EBADENGINE warning that installs anyway. ARG PI_VERSION=0.87.1 -RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION} +RUN npm install -g --engine-strict @earendil-works/pi-coding-agent@${PI_VERSION} # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path. # Pinned for the same reason as the CLIs above: agents run `uv` themselves @@ -87,7 +90,10 @@ COPY experiments/default.yaml ./experiments/default.yaml # so an OpenCode env-auth run would authenticate once the CLI is baked; the CLI is # the remaining blocker.) Adding it means a pinned version that travels with the release # tag (as CLAUDE_CODE_VERSION / PI_VERSION do) plus an env_passthrough block; see -# docs/agents/OPENCODE.md "Running in Docker". +# docs/agents/OPENCODE.md "Running in Docker". A Node CLI baked here must also be +# baked, at the same pin, into docker/Dockerfile.runtime, with a pin-parity test +# in tests/test_image_from_dockerfiles.py — otherwise inject-mode tasks find no +# CLI, as happened with Pi before it was added to the kit. # # CODER_EVAL_UV_EXTRAS carries ADDITIONAL opt-in extras on top of those; it # defaults to none. `make docker-image-full` passes `--extra uipath`, which diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime index 9b9eba30..a3d13adb 100644 --- a/docker/Dockerfile.runtime +++ b/docker/Dockerfile.runtime @@ -10,7 +10,7 @@ # # Contract (must match convert_skillsbench_task.py's inject block + the host): # /opt/coder-eval/venv/bin/coder-eval the CLI (its own Python) -# /opt/coder-eval/node/bin/{node,claude} Node + the Claude Code CLI +# /opt/coder-eval/node/bin/{node,claude,pi} Node + the Claude Code and Pi CLIs # /usr/local/bin/coder_eval_entrypoint.sh entrypoint (host pins --entrypoint here) # LABEL org.coder-eval.version= for `docker inspect` / parity # @@ -65,7 +65,14 @@ RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKA uv pip install --python /opt/coder-eval/venv/bin/python --no-deps /src # --- 3. Node LTS + the Claude Code CLI, under the kit dir. ------------------- -ARG NODE_VERSION=22.14.0 +# NODE_VERSION tracks the newest Node 22 LTS, which is what the framework image +# gets from nodesource's setup_22.x — so an injected task runs the CLIs on the +# same Node as a rebased one. Every `npm install -g` below passes +# --engine-strict: a CLI whose `engines.node` floor is above NODE_VERSION then +# fails the build, instead of npm printing EBADENGINE and installing anyway +# (Pi 0.87.1 needs >=22.19.0; on the previous 22.14.0 pin it installed, ran +# `pi --version` fine and would only have misbehaved mid-task). +ARG NODE_VERSION=22.23.4 ARG CLAUDE_CODE_VERSION=2.1.281 # Download to a file first so a bad URL fails the build loudly (a piped # `curl | tar` would mask a 404 and only surface later as a missing npm), and @@ -80,9 +87,20 @@ RUN mkdir -p /opt/coder-eval/node \ && grep " ${NODE_TARBALL}\$" /tmp/node.shasums | sed 's# .*# /tmp/node.tar.xz#' | sha256sum -c - \ && tar -xJf /tmp/node.tar.xz -C /opt/coder-eval/node --strip-components=1 \ && rm /tmp/node.tar.xz /tmp/node.shasums \ - && PATH="/opt/coder-eval/node/bin:${PATH}" npm install -g --prefix /opt/coder-eval/node \ + && PATH="/opt/coder-eval/node/bin:${PATH}" npm install -g --engine-strict --prefix /opt/coder-eval/node \ @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} +# --- 4. The Pi CLI, under the kit dir. -------------------------------------- +# Same package and pin as docker/Dockerfile, so `--driver docker --type pi` runs +# in an injected task image as it does in a rebased one; a parity test +# (tests/test_image_from_dockerfiles.py) keeps the two PI_VERSION pins equal. +# `pi --version` fails the build here if the package's bin did not land on the +# kit's PATH, instead of every pi task failing later with "pi CLI not found". +ARG PI_VERSION=0.87.1 +RUN PATH="/opt/coder-eval/node/bin:${PATH}" npm install -g --engine-strict --prefix /opt/coder-eval/node \ + @earendil-works/pi-coding-agent@${PI_VERSION} \ + && PATH="/opt/coder-eval/node/bin:${PATH}" pi --version + # Sanity: the CLI runs from the kit (no reliance on a system Python). Run it in # the builder, where the kit lives; the final stage is a bare carrier. RUN /opt/coder-eval/venv/bin/coder-eval _run-task-internal --help > /dev/null @@ -99,7 +117,7 @@ COPY --from=builder /opt/coder-eval /opt/coder-eval # Entrypoint: puts the kit's bins on PATH, then runs the orchestrator. Same # filename + location as the framework image, so the host's fixed # `--entrypoint /usr/local/bin/coder_eval_entrypoint.sh` works for injected -# images too — coder-eval/claude/node all resolve from the kit regardless of what +# images too — coder-eval/claude/pi/node all resolve from the kit regardless of what # the task's base image provides. COPY docker/coder_eval_runtime_entrypoint.sh /usr/local/bin/coder_eval_entrypoint.sh RUN chmod +x /usr/local/bin/coder_eval_entrypoint.sh diff --git a/docker/coder_eval_runtime_entrypoint.sh b/docker/coder_eval_runtime_entrypoint.sh index 56f2f57e..1fa4b17d 100644 --- a/docker/coder_eval_runtime_entrypoint.sh +++ b/docker/coder_eval_runtime_entrypoint.sh @@ -5,8 +5,8 @@ # injected image keeps the task's own base and only has the self-contained kit # under /opt/coder-eval. So this entrypoint prepends the kit's Python venv and # Node bin dirs to PATH before launching the orchestrator — making `coder-eval`, -# `claude`, and `node` resolve from the kit regardless of what the task's base -# image provides (or doesn't). +# `claude`, `pi`, and `node` resolve from the kit regardless of what the task's +# base image provides (or doesn't). # # Same filename/location as the framework entrypoint, so the host's fixed # `docker run --entrypoint /usr/local/bin/coder_eval_entrypoint.sh` works for diff --git a/docs/DOCKER_ISOLATION.md b/docs/DOCKER_ISOLATION.md index 8c6e6383..51e10567 100644 --- a/docs/DOCKER_ISOLATION.md +++ b/docs/DOCKER_ISOLATION.md @@ -145,7 +145,7 @@ LABEL org.coder-eval.version="" ``` `make coder-eval-runtime` builds the kit (`docker/Dockerfile.runtime`): a -standalone CPython + Node + the `coder-eval` CLI + Claude Code, all under +standalone CPython + Node + the `coder-eval` CLI + Claude Code + Pi, all under `/opt/coder-eval`, plus the entrypoint at the **same** `/usr/local/bin/...` path the host pins. The kit is **glibc-only** — it runs on debian/ubuntu/fedora/rhel/… but not musl/Alpine. @@ -158,6 +158,9 @@ individual targets when you only need one. > The kit installs the **no-credential** set only (core + codex), like `make docker-image` — > it never installs the `[uipath]` extra, so there is no `make docker-images-full`. An > inject-mode task that needs the LLMGW/`uipath` judge isn't supported by the kit as built. +> Agents under inject mode: `claude-code`, `codex` and `pi` run from the kit; `antigravity` +> and the `litellm` judge route are in the framework image only, so a task that needs them +> must rebase onto `coder-eval-agent`. ```dockerfile # environment/Dockerfile diff --git a/docs/agents/PI.md b/docs/agents/PI.md index eb515197..e9437566 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -216,10 +216,12 @@ success — the error names the unrecognized event types it saw. Intentional cut ## Running in Docker Pi **is supported under `--driver docker`.** The pinned Pi CLI is baked into -`docker/Dockerfile` (`ARG PI_VERSION`, alongside the claude-code CLI), and -`OPENROUTER_API_KEY` is forwarded into the container by the docker driver's -default `env_passthrough` allowlist — so `--driver docker --type pi` runs -end-to-end with no per-task `env_passthrough_extra`: +`docker/Dockerfile` (`ARG PI_VERSION`, alongside the claude-code CLI) and, at the +same pin, into the runtime kit (`docker/Dockerfile.runtime`), so a task that +brings its own base image runs Pi too. `OPENROUTER_API_KEY` is forwarded into +the container by the docker driver's default `env_passthrough` allowlist — so +`--driver docker --type pi` runs end-to-end with no per-task +`env_passthrough_extra`: ```bash export OPENROUTER_API_KEY="sk-or-..." diff --git a/tests/test_image_from_dockerfiles.py b/tests/test_image_from_dockerfiles.py index 4f96aff9..c7ffe8a6 100644 --- a/tests/test_image_from_dockerfiles.py +++ b/tests/test_image_from_dockerfiles.py @@ -418,6 +418,19 @@ def _runtime_dockerfile() -> Path | None: return df if df.is_file() else None +def _framework_dockerfile_text() -> str: + return (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8") + + +def _arg_pin(df_text: str, name: str) -> str | None: + """The value of the first `ARG =` line in a Dockerfile's text, or None.""" + for ln in df_text.splitlines(): + m = re.match(rf"\s*ARG {re.escape(name)}=(\S+)", ln) + if m: + return m.group(1) + return None + + def test_runtime_kit_stamps_version_label() -> None: """The kit Dockerfile must stamp `org.coder-eval.version`. @@ -460,31 +473,48 @@ def test_runtime_kit_path_matches_install_dirs() -> None: ) -def test_claude_code_version_pin_matches_framework() -> None: - """The Claude Code pin must be identical in the framework and runtime-kit - Dockerfiles — else inject-mode tasks run a different agent version than - rebase-mode tasks (a reproducibility/fairness drift, since the agent binary - is a dominant non-model driver of results).""" - root = Path(__file__).resolve().parents[1] +@pytest.mark.parametrize("arg", ["CLAUDE_CODE_VERSION", "PI_VERSION"]) +def test_cli_version_pin_matches_framework(arg: str) -> None: + """Each baked agent CLI's pin must be identical in the framework and + runtime-kit Dockerfiles — else inject-mode tasks run a different agent + version than rebase-mode tasks (a reproducibility/fairness drift, since the + agent binary is a dominant non-model driver of results), or no agent at all + when the kit lacks the CLI.""" runtime_df = _runtime_dockerfile() if runtime_df is None: pytest.skip("docker/Dockerfile.runtime not present") - def _pin(df: Path) -> str | None: - for ln in df.read_text(encoding="utf-8").splitlines(): - m = re.match(r"\s*ARG CLAUDE_CODE_VERSION=(\S+)", ln) - if m: - return m.group(1) - return None - - fw, kit = _pin(root / "docker" / "Dockerfile"), _pin(runtime_df) - assert fw and kit, f"CLAUDE_CODE_VERSION ARG missing (framework={fw!r}, kit={kit!r})" + fw = _arg_pin(_framework_dockerfile_text(), arg) + kit = _arg_pin(runtime_df.read_text(encoding="utf-8"), arg) + assert fw and kit, f"{arg} ARG missing (framework={fw!r}, kit={kit!r})" assert fw == kit, ( - f"CLAUDE_CODE_VERSION drift: docker/Dockerfile pins {fw!r} but docker/Dockerfile.runtime pins {kit!r} — " - "inject and rebase tasks would run different Claude Code versions." + f"{arg} drift: docker/Dockerfile pins {fw!r} but docker/Dockerfile.runtime pins {kit!r} — " + "inject and rebase tasks would run different CLI versions." ) +def test_npm_global_installs_are_engine_strict() -> None: + """Every `npm install -g` in both Dockerfiles must pass --engine-strict. + Without it npm only warns (EBADENGINE) when a CLI's `engines.node` floor is + above the image's Node and installs anyway, so the build — and the kit's + `pi --version` gate — stays green while the agent runs on a Node it does not + support (Pi 0.87.1 needs >=22.19.0; the kit pinned 22.14.0).""" + sources = {"docker/Dockerfile": _framework_dockerfile_text()} + runtime_df = _runtime_dockerfile() + if runtime_df is not None: + sources["docker/Dockerfile.runtime"] = runtime_df.read_text(encoding="utf-8") + for label, text in sources.items(): + installs = [ + ln.strip() for ln in text.splitlines() if "npm install -g" in ln and not ln.lstrip().startswith("#") + ] + assert installs, f"{label}: no `npm install -g` found — did the CLI installs move?" + for install in installs: + assert "--engine-strict" in install, ( + f"{label}: `{install}` lacks --engine-strict, so an unsupported Node would install the CLI " + "with only a warning instead of failing the build." + ) + + def test_uv_pinned_in_agent_image() -> None: """docker/Dockerfile must install a PINNED uv through the versioned installer URL. Agents run `uv` themselves, so its install speed is part of @@ -492,14 +522,9 @@ def test_uv_pinned_in_agent_image() -> None: 0.12.12 on a rebuild, and 0.12.11+ copies cross-filesystem (cache in the image, venv on the bind-mounted workspace) ~40x slower. Fails on an unversioned installer URL or a missing/`latest` pin.""" - df_text = (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8") + df_text = _framework_dockerfile_text() - pin: str | None = None - for ln in df_text.splitlines(): - m = re.match(r"\s*ARG UV_VERSION=(\S+)", ln) - if m: - pin = m.group(1) - break + pin = _arg_pin(df_text, "UV_VERSION") assert pin, "ARG UV_VERSION= missing from docker/Dockerfile — uv is not pinned." assert re.fullmatch(r"\d+\.\d+\.\d+", pin), f"UV_VERSION must be an exact x.y.z version, got {pin!r}." @@ -513,31 +538,24 @@ def test_uv_pinned_in_agent_image() -> None: def test_pi_cli_baked_and_pinned() -> None: - """docker/Dockerfile must ship a PINNED Pi CLI, and its install line must - reference the ARG (not a hardcoded literal) so a bump is one edit and the - pin can't drift from the install. Mirrors the claude-code pinned-CLI guard - above — Pi's presence is a positive, mechanically-checkable invariant - (`--driver docker --type pi` depends on it). Fails on @latest or removal. - Scope: main docker/Dockerfile only — Dockerfile.runtime deliberately does - NOT bake Pi (inject-mode is out of scope), so it is not asserted here.""" - df_text = (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8") - - pin: str | None = None - for ln in df_text.splitlines(): - m = re.match(r"\s*ARG PI_VERSION=(\S+)", ln) - if m: - pin = m.group(1) - break - assert pin, "ARG PI_VERSION= missing from docker/Dockerfile — the Pi CLI is not pinned." - assert pin.lower() != "latest", ( - f"PI_VERSION is pinned to {pin!r} — the Pi CLI must be an exact version, not `latest`, so layer " - "caching cannot freeze it nondeterministically across rebuilds." - ) - - assert re.search( - r"npm install -g @earendil-works/pi-coding-agent@\$\{PI_VERSION\}", - df_text, - ), ( - "docker/Dockerfile must `npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}` — the install " - "must reference the PI_VERSION ARG so the pin and the install cannot drift." - ) + """Both images must ship a PINNED Pi CLI, and the install must reference + the ARG (not a hardcoded literal) so a bump is one edit and the pin can't + drift from the install. Pi's presence is a positive, mechanically-checkable + invariant (`--driver docker --type pi` depends on it, in a rebased and in an + injected task image alike). Fails on @latest, a literal, or removal; the + pin-parity test above keeps the two ARGs equal.""" + sources = {"docker/Dockerfile": _framework_dockerfile_text()} + runtime_df = _runtime_dockerfile() + if runtime_df is not None: + sources["docker/Dockerfile.runtime"] = runtime_df.read_text(encoding="utf-8") + for label, df_text in sources.items(): + pin = _arg_pin(df_text, "PI_VERSION") + assert pin, f"ARG PI_VERSION= missing from {label} — the Pi CLI is not pinned." + assert pin.lower() != "latest", ( + f"{label}: PI_VERSION is pinned to {pin!r} — the Pi CLI must be an exact version, not `latest`, so " + "layer caching cannot freeze it nondeterministically across rebuilds." + ) + assert re.findall(r"@earendil-works/pi-coding-agent@(\S+)", df_text) == ["${PI_VERSION}"], ( + f"{label} must install @earendil-works/pi-coding-agent@${{PI_VERSION}} exactly once — the install " + "must reference the PI_VERSION ARG so the pin and the install cannot drift." + ) diff --git a/uv.lock b/uv.lock index f63fadc6..eed5dc6d 100644 --- a/uv.lock +++ b/uv.lock @@ -911,11 +911,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.32.6" +version = "4.2.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/38/46/126b1831dca12060d4a8296bf9c4fe5c93c4f22197fa239cb0cc82042bba/filelock-3.32.6.tar.gz", hash = "sha256:a3f55a18af3652a94d8f47d6055df434f254ca1d02ef2524850c6d249ca2512c", size = 225172, upload-time = "2026-09-08T22:57:11.528Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/28/3893324bcadf8f89b1616637468d766af3f625008c0735bf704fdfa86617/filelock-4.2.4.tar.gz", hash = "sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d", size = 652444, upload-time = "2026-10-19T01:57:51.001Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/06/4f138f618dbea66803291274f228f01daf29f306fe8b96bc30dab765df75/filelock-3.32.6-py3-none-any.whl", hash = "sha256:3f16ecd0117feae0dfc147e8c62eb5daeccd8bd800378c3ddf416de9b4feb6b1", size = 100189, upload-time = "2026-09-08T22:57:10.182Z" }, + { url = "https://files.pythonhosted.org/packages/e0/1b/5e74289a1878e666bb4b65fe5805a04cafcac2aa5f519ef1ead67ef91c56/filelock-4.2.4-py3-none-any.whl", hash = "sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e", size = 162475, upload-time = "2026-10-19T01:57:49.419Z" }, ] [[package]] @@ -3336,7 +3336,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.14.12" +version = "21.14.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -3345,9 +3345,9 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/2c/1a69b3346931e9a7db9cc00cc0fa847c843a85fa7cf52a598f57261c68df/virtualenv-21.14.12.tar.gz", hash = "sha256:bbcfb78e36c22ad70c6917c4aecb5e63477d3c138e9f88c946f320e31859c76a", size = 5562800, upload-time = "2026-10-21T01:58:51.724Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/aa/bff50daee0488e16d611258a9b6c5a1d0c9e128f8ca424e9f4e1c0444ef2/virtualenv-21.14.13.tar.gz", hash = "sha256:de5307ba4b9a0001d9c8ea02832d38f63ca432880f416c1bface61c4323fd905", size = 5570301, upload-time = "2026-10-22T20:10:16.952Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/92/f3b8af2d36ddec4dc61dc25655dc010e419ff456cb4ca0e720c8112f1047/virtualenv-21.14.12-py3-none-any.whl", hash = "sha256:7820aeaef23f1f59a96458edfc7f9d3deac937155718de15f1c9a64471e15551", size = 5556824, upload-time = "2026-10-21T01:58:48.505Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a9/c10d8a44a29fb94a18b43fdf11f38caca53ae7c9406f11a72b1cb8f624f1/virtualenv-21.14.13-py3-none-any.whl", hash = "sha256:5fbb04138a4861580b78ba9110d529a26bc48fcc93c8ec36bc0202eeed96a788", size = 5557094, upload-time = "2026-10-22T20:10:15.096Z" }, ] [[package]]