From c7c0ab2a98eb02e753719ebf6119e7f4514f6123 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 21:30:29 +0000 Subject: [PATCH 1/9] Start fix for #721 Assisted-by: Claude Code:claude-opus-5-5 From 5444dd686cc6ef763dc1b37e7dcd0910c9cba3c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 22:04:55 +0000 Subject: [PATCH 2/9] Stop UTF-16 requirements.txt being skipped Windows PowerShell 5.1 writes `pip freeze > requirements.txt` as UTF-16 with a byte-order mark, and pip installs from it. Hosted scan read every candidate file as UTF-8 and treated a file it could not decode as missing, so the run exited 0 as a success with nothing pinned, and pip kept installing the unpatched release. A fresh checkout's lock-only scan said "No packages found" for the same file. Hosted runs (disk and in-memory alike) now refuse with candidate_file_unreadable, naming the file and asking for it to be re-saved as UTF-8, whenever a non-UTF-8 candidate file belongs to an ecosystem being redirected. Nothing is written. Lock-only discovery decodes requirements.txt and its -r includes by BOM the way pip does, so the pins are found. Fixes #721 Assisted-by: Claude Code:claude-opus-5-5 --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 + .../tests/in_process_get_hosted_ecosystems.rs | 55 +++++++ .../tests/scan_requirements_lock_only.rs | 35 +++- crates/socket-patch-core/src/hosted/engine.rs | 155 ++++++++++++++++-- .../src/hosted/memory/mod.rs | 1 + .../src/utils/requirements.rs | 64 ++++++++ .../src/vendor/lock_inventory/pypi.rs | 12 +- .../src/vendor/lock_inventory/tests.rs | 50 ++++++ 8 files changed, 360 insertions(+), 14 deletions(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 69fb09df9..5ffc851eb 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -165,6 +165,8 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc The rewriter reads a fixed set of candidate files from the project root: the npm-family locks (`package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `shrinkwrap.yaml`, `yarn.lock`, plus `.yarnrc.yml` for the berry cache-config gate, `bun.lock` / `bun.lockb`, and `vlt-lock.json` with `vlt.json` and `node_modules/.vlt-lock.json` read only), `requirements.txt` / `uv.lock` / `Pipfile.lock` (pipfile-spec 6; see the Pipenv section below) / `poetry.lock` (every Poetry lock generation from 1.0 on — the 0.12 `[metadata.hashes]` layout is refused because that installer ignores URL sources; a Poetry < 1.4 writer additionally gets `redirect_poetry_stale_install_risk`, see `docs/testing/poetry-compatibility.md`) / `pdm.lock` (PDM lock formats `2` and `4.3`–`4.5.1`; the identity-losing `3.1` / `4.0`–`4.2` formats and unknown future formats are refused with `redirect_pdm_refused`, and a lock-format-`2` writer additionally gets `redirect_pdm_legacy_sync_required`, see `docs/testing/pdm-compatibility.md`; when `uv.lock` or `poetry.lock` sits beside it they drive and `pdm.lock` is left alone), `Cargo.toml` / `Cargo.lock` / `.cargo/config.toml` (plus the legacy extensionless `.cargo/config` — cargo reads that spelling in preference when both exist, so the managed `[registries.…]` block is written into whichever one is present; **cargo also reads every workspace-member manifest** — the `[workspace] members` globs minus `exclude` — and every in-root path-dependency manifest, recursively, reached without crossing a symbolic link and never under `.socket/`, and pins the crate in each one that declares it, so those `/Cargo.toml` files can appear in `rewrittenFiles`. A crate is redirected only when every declaration pins and every other `Cargo.lock` package depending on it is a planned member: one a registry or git crate — or a path package outside the root or behind a link — also depends on is refused `redirect_cargo_transitive_dependents` (a pin reaches only the declarations it sits on), a crate no manifest declares keeps `redirect_cargo_toml_dep_not_found` with a transitive-only detail naming `--mode vendored`, a crate every declaration of which requires another version (no requirement accepts the patched version) is refused `redirect_cargo_toml_dep_unrewritable`, and so is a requirement that also matches another locked version of the crate — each a transactional skip, never recorded or attested. With NO `Cargo.lock` there is no resolved graph to ask, so the dependents question is answered from the manifests instead: a crate declared beside any other dependency — anything but a path dependency on a manifest this run also pins, or a `workspace = true` inheritor of a table it scans — or beside a workspace member this run did not read (a `members` glob, or a member outside the project or behind a symbolic link, which member discovery drops) is refused `redirect_cargo_lockless_dependents`, whose detail names the remedies (commit a lockfile, or `--mode vendored`); a project whose only dependency is the patched crate has nothing that could pull it in and still redirects. All-CRLF manifests, locks and configs are rewritten with CRLF kept (mixed endings keep refusing where the grammar does not match), and `remove` / rollback match the recorded fragments across a later CRLF↔LF checkout conversion), `composer.lock`, `nuget.config` / `packages.lock.json`, `Gemfile` / `Gemfile.lock`, `pom.xml` (+ `.mvn/maven.config` / `.mvn/checksums/checksums.sha256` for maven Trusted Checksums merge, and the Gradle build scripts read only to trigger the manual-snippet warning). **npm-family flavor coverage**: package-lock / npm-shrinkwrap, pnpm (root OR any nested `*/pnpm-lock.yaml`), yarn classic, **yarn berry** (the pin yarn writes for a root `resolutions` entry: the root `package.json` — edited only beside a berry `yarn.lock` — gains one `"@npm:": ""` selector per locked range (`redirect_yarn_berry_resolution` edits), and only that `yarn.lock` entry is re-keyed `"@"` with the same `resolution:` + `yarnBerry10c0` checksum (`redirect_yarn_berry_entry`), moved to yarn's key order; never an `npm:` locator, whose fetcher sends npm registry auth to the patch host, nor a tarball locator under an `npm:` key, which hardened mode rejects (YN0078). An older release's `npm:::__archiveUrl=` pin is still recognized and is re-pinned on the next run; rollback rebuilds the key from the selectors and drops them. Refused, nothing written: a user-authored `resolutions` entry for the package `redirect_yarn_berry_resolutions_conflict`, no root manifest `redirect_yarn_berry_manifest_missing`, a builtin `patch:` entry wrapping the same descriptor `redirect_yarn_berry_shared_descriptor`, an artifact URL yarn cannot fetch as a tarball `redirect_yarn_berry_artifact_url_unsupported`; cacheKey `10c0` and `.yarnrc.yml compressionLevel 0` gated by `redirect_yarn_berry_cache_unsupported`), and **bun** (text `bun.lock` lockfileVersion 0, 1 or 2 — 0 is the `--save-text-lockfile` opt-in lock of Bun 1.1.39–1.1.45, 1 the 1.2–1.3 default, 2 the 1.4+ default; all three emit one `packages` grammar, so the registry 4-tuple → URL 3-tuple rewrite is version-independent and the lock's own version line is kept. Any other or missing version, or a `packages` section outside bun's single-line grammar, is refused `redirect_bun_lock_unsupported` — the detail is the shared version gate's text (a newer version: update socket-patch, re-locking would reproduce it; no integer: re-lock with Bun ≥ 1.2), identical to the vendored refusal. A version-0 lock holding `workspace:` packages is refused `redirect_bun_workspace_unsupported` (its 2-tuple workspace grammar cannot keep the hosted tuple through a frozen install); the remedy is to delete `bun.lock` and re-run `bun install` with Bun ≥ 1.2, which writes lockfileVersion 1 (accepted). A plain in-place `bun install` bumps the version only when a workspace depends on another workspace (e.g. root → member — the shape the matrix measured); otherwise Bun 1.2.0 keeps version 0 and Bun 1.2.23+ fail to resolve, so the in-place bump is not the documented remedy. Bun lock version, grammar and workspace compatibility are checked before a vendored takeover, including during dry-run: these refusals preserve the existing lock, artifact and vendor ledger. Version-1 and version-2 workspace locks are rewritten, nested versions included. A granted dep with no rewritable entry warns `redirect_bun_entry_not_found`, a grant without a sha512 `redirect_bun_missing_sha512`; a CRLF lock keeps `\r\n` on the rewritten line, and a hosted URL left by an earlier grant of the same `name@version` is re-pinned in place. **Digest-less re-saves (Bun 1.1.39–1.3.9)**: every text-lock Bun below 1.3.10 re-saves a URL tuple WITHOUT its `sha512` whenever the lock is re-saved for another reason (`bun add`, `bun install` after a package.json or workspace change), leaving the 2-tuple `["name@", {meta}]` — the spec Bun installs from is intact. The CLI treats that spelling as its own wiring: a repeat hosted run counts the dep as redirected (no `redirect_bun_entry_not_found`) and HEALS the line back to the 3-tuple with the current `sha512`, recording the heal as a further `redirect_bun_lock_package` edit whose `original` is the 2-tuple (a stale URL is re-pinned from either spelling); `rollback`, scoped `rollback ` / `remove ` and the vendored takeover accept the digest-less spelling of a recorded `new` line (same key, spec and meta, only the trailing `"sha512-…"` missing) and restore the recorded original over it, so the chain always unwinds to the pristine registry line. Anything else — another uuid/token, another version, a re-laid meta object — is still drift. **Native `bun.lockb`**: when no text `bun.lock` exists, binary format versions 1, 2 and 3 are read and rewritten directly. Socket Patch does not invoke Bun or convert the project to a text lockfile. Exact matching package records are rewritten to hosted tarballs with the granted integrity, preserving dependency resolution IDs, workspace/dependency topology and unrelated package metadata; binary pointers and the package metadata hash are updated. Per-package `redirect_bun_lockb_package` snapshots support scoped rollback, repeat runs, superseding grants and hosted ↔ vendored takeover. A regular binary lock is discoverable even with no Bun runtime or `node_modules`; a dry run previews the same binary edits without writing them. A malformed, unreadable, unsupported or unverified binary structure is `redirect_bun_lockb_invalid` (exit 0, `redirected: 0`), and it refuses the npm rewrite before any takeover or sibling npm-family lock mutation. A symlinked binary write target is `redirect_symlinked_file_unsupported` (exit 1, including dry-run). `bun.lock` wins when both spellings exist. Binary-only projects do not receive `redirect_npm_no_lockfile`. Measured boundaries and the real-Bun matrix: `docs/testing/bun-compatibility.md`), and **vlt** (`vlt-lock.json` without `lockfileVersion`, `0` or `1`; see the vlt hosted-mode contract below). **Rush monorepos**: when `rush.json` is present the rewriter also reads `common/config/rush/pnpm-lock.yaml` and each `common/config/subspaces//pnpm-lock.yaml` (sorted for determinism) under their repo-relative keys and repoints them in place; editing them emits `redirect_rush_repo_state_stale` when `common/config/rush/repo-state.json` exists (the `pnpmShrinkwrapHash` desync is refreshed by `rush update`, which the redirect survives). **maven** is fail-closed via version suffixing: a `mavenSuffixedVersion` + `mavenPomSha256` override pins the Socket-only `-socket.` by rewriting the literal `` (`redirect_maven_dep_version`) or adding a `` entry (`redirect_maven_dep_management_added`), plus optional Trusted Checksums (`redirect_maven_trusted_checksums`, conflicts as `redirect_maven_trusted_checksums_conflict`; when `.mvn/wrapper/maven-wrapper.properties` pins a Maven older than 3.9.4, which ignores those files, the additive warning `redirect_maven_trusted_checksums_unenforced`); a `${property}` version is refused (`redirect_maven_dep_unpinned`), a non-matching literal skipped (`redirect_maven_dep_version_mismatch`), and an override without a suffixed version falls back to same-GAV repository injection (`redirect_maven_same_gav_fallback`, NOT fail-closed). +**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". + **Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery) for each confirmed gem redirect and judges the materialization against the patch record's `afterHash` file map. Judgment rules: records are found **by uuid** among this run's fetched records (v5.0: hosted mode persists no records, so a purl whose `/patches/view` fetch failed this run is not judged; the warning re-fires on every re-scan whose fetch succeeds, until the stale materialization is gone); a materialization with every file at `afterHash` is already patched and never warns (an agent→hosted migration stays quiet by construction), and when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — at least one record file whose bytes were actually read and hash to neither state's expectation — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `redirect.warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, else `vendor/cache`; a relative value is read against the project root. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app config is skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. **Pipenv hosted redirect (`Pipfile.lock`, pipfile-spec 6)**: every category other than `_meta` (`default`, `develop`, and Pipenv 2022+ named categories) that pins the package at the patched version is rewritten to the hosted reference — `{"file" | "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept exactly as Pipenv wrote them (present or absent: whether Pipenv records `index` depends on its release, the Pipfile spelling and the locking environment, so only the entry itself knows) and `version` dropped; `_meta` (the Pipfile content hash) and the Pipfile itself are never touched, so `pipenv install --deploy`/`sync`/`verify` keep passing. The reference KEY depends on the installing Pipenv: releases 7–11 only install `path` references, 2018 and later `file` ones (0–6 write pipfile-spec < 6 and are refused). The release is probed once per command with `pipenv --version`, resolved on ABSOLUTE `PATH` entries only (a relative entry would run a `pipenv` planted in the scanned repository; `.bat`/`.cmd` shims are found through `PATHEXT` on Windows), only when a pypi patch actually targets an entry of the lock, and `SOCKET_PIPENV_MAJOR=` pins the answer without spawning anything. An unknown installer selects `file` and warns `redirect_pipenv_installer_unknown` only when the lock was rewritten. **Refusal scope**: a pin/source CONFLICT (another version pinned, a foreign `file`/`path` source, a VCS/editable dependency) refuses the whole dependency atomically across categories as `redirect_pipenv_refused` AND vetoes the sibling Python rewriters (requirements.txt / uv.lock / pyproject) for that patch — the project's Pipenv install could not pick the patch up, so a half-redirected checkout is refused; anything else (no entry for the package, an old pipfile-spec, an unparseable lock, a digest-less patch) is `redirect_pipenv_skipped` and leaves the siblings alone (a stale Pipfile.lock in a uv/Poetry/requirements project must not block them). The veto applies to a LIVE lock only: a `Pipfile.lock` with no `Pipfile` beside it is abandoned, so its conflict refuses that file but never the siblings. Hash enforcement at install time is split by era — the `#sha256=` URL fragment is what Pipenv 2023+ verifies, the `hashes` list what 2018–2022 verify, Pipenv 11 either — so both are load-bearing. **Pipenv stale-install guard**: Pipenv never reinstalls a release that is already present (`pipenv install`, `install --deploy` and `sync` all exit 0 and keep the installed bytes — measured on 11.10.4, 2018.11.26 and 2026.8.0, hosted and vendored), so after the rewrite the run probes the Python crawler's site-packages (VIRTUAL_ENV, `./.venv`, `./venv`, Pipenv's out-of-tree `WORKON_HOME` venv; `--global`/`--global-prefix` honoured) for each confirmed Pipfile.lock redirect with the same rules as the gem guard (records by uuid from this run's fetch, PATCHED = `verify_patch_record` Ok, STALE needs positive evidence, read-only, skipped on `--dry-run`, stale purls excluded from the same-run `--vex` `assume_applied` set) and the Python stale-install guard (`redirect_pypi_stale_install`, see above) names the site-packages dir and the Pipenv-specific verified remedy: `pipenv run pip uninstall -y && pipenv sync` (or `pipenv --rm && pipenv sync`) — NOT `pipenv uninstall`, which rewrites the Pipfile and re-locks the patch away. The vendored backend emits the twin `pypi_pipenv_stale_install` (`skipped` warning event). **Rollback** (v5.0, upstream restore): each hosted entry gets its registry shape back — `"version": "=="`, the entry's own `index` carried back unchanged (refused unless it — and the Pipfile's explicit `index`, if any — names a PyPI source in `_meta.sources`), and every release file's sha256 from PyPI's JSON API (`SOCKET_PYPI_JSON_API`), sorted by filename as Pipenv records them; an entry that pins another version beside the hosted reference is refused with the `git checkout` remedy (see "Hosted unwind coverage"). A Pipfile names no project, so a same-run `--vex` on a Pipenv project needs `--vex-product` (or a git remote) to detect a product purl. **Discovery**: `Pipfile.lock` is part of the lockfile inventory (every category's `==` pins, with the lock's digest set as `Sha256AnyOf` integrity so a lock-only checkout can be vendored by fetching the pure wheel through PyPI's JSON API — only when `_meta.sources` name the public index; a private-index lock stays discovery-only and never reaches pypi.org), and Socket's own hosted / vendored references stay discoverable as the package they replace, so a re-scan of an already-redirected or already-vendored lock-only checkout re-confirms it (`--vex` attests, vendored reports `already_vendored`) instead of finding nothing. diff --git a/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs b/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs index db2aab79f..efeaa64c5 100644 --- a/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs +++ b/crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs @@ -315,6 +315,61 @@ async fn pypi_requirements_hosted_rewrites_pep440_equivalent_pin() { } } +/// #721: Windows PowerShell 5.1 writes `pip freeze > requirements.txt` as +/// UTF-16 with a BOM, and pip installs from it. The hosted grant must not +/// treat that file as absent and exit 0 with the project unpatched: it is +/// refused by name (`candidate_file_unreadable`, exit 1), nothing written. +#[tokio::test] +#[serial] +async fn pypi_requirements_hosted_refuses_a_utf16_file() { + const UUID: &str = "a1a1a1a1-a1a1-4a1a-8a1a-a1a1a1a1a1a3"; + const PURL: &str = "pkg:pypi/requests@2.31.0"; + const SHA256: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let url = format!( + "http://patch.test/patch/pypi/requests/2.31.0/{TOKEN}/{UUID}/requests-2.31.0-py3-none-any.whl" + ); + + let text = "flask==2.0.1\r\nrequests==2.31.0\r\n"; + let le: Vec = [0xFF, 0xFE] + .into_iter() + .chain(text.encode_utf16().flat_map(u16::to_le_bytes)) + .collect(); + let be: Vec = [0xFE, 0xFF] + .into_iter() + .chain(text.encode_utf16().flat_map(u16::to_be_bytes)) + .collect(); + for (what, bytes) in [("utf-16le", le), ("utf-16be", be)] { + let server = MockServer::start().await; + mock_view(&server, UUID, PURL).await; + mock_reference( + &server, + UUID, + PURL, + &url, + serde_json::json!({ "sha256": SHA256 }), + serde_json::Value::Null, + ) + .await; + + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("requirements.txt"), &bytes).unwrap(); + + let code = + socket_patch_cli::commands::get::run(get_hosted_args(UUID, tmp.path(), server.uri())) + .await; + assert_eq!( + code, 1, + "{what}: a requirements.txt hosted mode cannot read must refuse, not exit 0 unpatched" + ); + assert_eq!( + std::fs::read(tmp.path().join("requirements.txt")).unwrap(), + bytes, + "{what}: the refused file must stay byte-identical" + ); + assert_no_manifest_no_blobs(tmp.path()); + } +} + // --------------------------------------------------------------------------- // maven — pom.xml fail-closed suffixed-version pin (rewrite_maven_pom) // --------------------------------------------------------------------------- diff --git a/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs b/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs index 9ab939607..3ed496c3c 100644 --- a/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs +++ b/crates/socket-patch-cli/tests/scan_requirements_lock_only.rs @@ -5,7 +5,9 @@ //! release: //! //! * #523: whitespace around `==` and the legacy `name (==X)` form; -//! * #412: pins reached through in-root `-r` includes. +//! * #412: pins reached through in-root `-r` includes; +//! * #721: a UTF-16 file with a BOM (Windows PowerShell 5.1's +//! `pip freeze >` output), which pip decodes. //! //! Driven through the built binary against a mock patch API; the //! assertion is what discovery sends to the batch endpoint and the @@ -87,6 +89,11 @@ async fn batch_purls(mock: &MockServer) -> Vec { } async fn assert_lock_only_discovers(files: &[(&str, &str)], expected: &[&str]) { + let files: Vec<(&str, &[u8])> = files.iter().map(|(r, c)| (*r, c.as_bytes())).collect(); + assert_lock_only_discovers_bytes(&files, expected).await; +} + +async fn assert_lock_only_discovers_bytes(files: &[(&str, &[u8])], expected: &[&str]) { for mode in [&[][..], &["--vendor"][..]] { let mock = MockServer::start().await; mount_empty_batch(&mock).await; @@ -148,3 +155,29 @@ async fn lock_only_scan_discovers_included_pins() { ) .await; } + +/// #721: pip decodes a requirements file by its BOM, so a UTF-16 file +/// (what Windows PowerShell 5.1's `pip freeze >` writes) is discovered, +/// in either byte order, instead of reading as "No packages found". +#[tokio::test] +async fn lock_only_scan_discovers_utf16_pins() { + let text = "sp-fixture-idna==3.7\r\nsp-fixture-six==1.16.0\r\n"; + let le: Vec = [0xFF, 0xFE] + .into_iter() + .chain(text.encode_utf16().flat_map(u16::to_le_bytes)) + .collect(); + let be: Vec = [0xFE, 0xFF] + .into_iter() + .chain(text.encode_utf16().flat_map(u16::to_be_bytes)) + .collect(); + for bytes in [le, be] { + assert_lock_only_discovers_bytes( + &[("requirements.txt", &bytes)], + &[ + "pkg:pypi/sp-fixture-idna@3.7", + "pkg:pypi/sp-fixture-six@1.16.0", + ], + ) + .await; + } +} diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index aebeac1ab..945320dba 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -68,7 +68,9 @@ pub const SYMLINK_REFUSAL: &str = "redirect_symlinked_file_unsupported"; /// Refusal code for a candidate file that exists but whose content the /// in-memory host did not provide (oversize, an LFS pointer, -/// presence-only); disk would read and rewrite it. +/// presence-only), and, on disk and in memory alike, for one that is not +/// UTF-8 text (#721): no rewriter can edit it, and reading it as absent +/// would leave its pins unpatched behind an exit-0 run. pub const UNREADABLE_REFUSAL: &str = "candidate_file_unreadable"; /// Rush's repo-state file, whose `pnpmShrinkwrapHash` a lock edit @@ -154,6 +156,18 @@ fn unreadable_refusal(rel: &str) -> Refusal { } } +fn undecodable_refusal(rel: &str) -> Refusal { + Refusal { + code: UNREADABLE_REFUSAL.to_string(), + message: format!( + "{rel} is not UTF-8 text (for example UTF-16, which Windows PowerShell 5.1 \ + writes for `pip freeze > requirements.txt`), so it cannot be rewritten \ + alongside the other lockfiles; re-save it as UTF-8 and re-run; nothing was \ + written" + ), + } +} + /// Reference grants → candidates. A selection without a usable grant is /// recorded in `skipped` (`not_found`, the reference status, `bad_purl`, /// `no_url`). @@ -302,6 +316,11 @@ pub struct CandidateFiles { /// project whose candidates could rewrite (or whose rewrite depends on) /// one is refused, since the rewriters would treat it as absent. pub unreadable_reads: Vec, + /// Candidate files that exist but are not UTF-8 text (a UTF-16 + /// requirements.txt pip reads, #721), on disk and in memory alike. They + /// are left out of `files`; a project whose candidates could rewrite + /// one is refused rather than read as if the file were absent. + pub undecodable_reads: Vec, /// Set when bundler is configured (`BUNDLE_GEMFILE`) to load a manifest /// the gem rewriter cannot edit: every gem manifest and lock was left /// out of `files`, and the rewrite reports this instead of a redirect. @@ -321,7 +340,14 @@ impl CandidateFiles { // (non-blocking open + fstat regular-file check), so a FIFO // under a candidate name is skipped like a missing file instead // of wedging the run in open(2). - ProjectView::Disk(_) | ProjectView::Snapshot(_) => view.read_text(rel).await.ok(), + ProjectView::Disk(_) | ProjectView::Snapshot(_) => match view.read_text(rel).await { + Ok(text) => Some(text), + Err(e) if e.kind() == std::io::ErrorKind::InvalidData => { + self.undecodable_reads.push(rel.to_string()); + None + } + Err(_) => None, + }, ProjectView::Memory(project) => { if project.is_symlink(rel) { self.symlinked_reads.push(rel.to_string()); @@ -331,13 +357,17 @@ impl CandidateFiles { self.unreadable_reads.push(rel.to_string()); return false; } - // Disk reads any UTF-8 regular file; a non-UTF-8 one is - // absent to it as well. + // Disk reads any UTF-8 regular file and records a non-UTF-8 + // one as undecodable; so does memory. match project.get(rel) { Some(MemoryEntry::Text(text)) => Some(text.to_string()), - Some(MemoryEntry::Binary(bytes)) => { - std::str::from_utf8(bytes).ok().map(str::to_string) - } + Some(MemoryEntry::Binary(bytes)) => match std::str::from_utf8(bytes) { + Ok(text) => Some(text.to_string()), + Err(_) => { + self.undecodable_reads.push(rel.to_string()); + None + } + }, _ => None, } } @@ -509,6 +539,8 @@ pub async fn read_candidate_files( out.symlinked_reads.dedup(); out.unreadable_reads.sort(); out.unreadable_reads.dedup(); + out.undecodable_reads.sort(); + out.undecodable_reads.dedup(); out } @@ -557,6 +589,7 @@ async fn keep_bundler_loaded_gem_files(view: &ProjectView<'_>, out: &mut Candida out.files.retain(|rel, _| !dropped(rel)); out.symlinked_reads.retain(|rel| !dropped(rel)); out.unreadable_reads.retain(|rel| !dropped(rel)); + out.undecodable_reads.retain(|rel| !dropped(rel)); out.gem_manifest_unsupported = loaded.unsupported_detail().map(|detail| RewriteWarning { code: "redirect_gem_bundle_gemfile_unsupported".into(), detail, @@ -677,6 +710,7 @@ pub struct Rewritten { pub files: BTreeMap, pub symlinked_reads: Vec, pub unreadable_reads: Vec, + pub undecodable_reads: Vec, /// The rewriters' override slice (the candidates' deps). pub overrides: Vec, pub rewrite: RewriteResult, @@ -826,6 +860,7 @@ pub async fn rewrite( rush_lock_keys, symlinked_reads, unreadable_reads, + undecodable_reads, gem_manifest_unsupported, } = read; // The rewriters' override slice — materialized ONCE, after the last @@ -980,6 +1015,7 @@ pub async fn rewrite( files, symlinked_reads, unreadable_reads, + undecodable_reads, overrides, rewrite, rewritten, @@ -1512,6 +1548,9 @@ fn file_ecosystem(rel: &str) -> Option<&'static str> { /// bytes but never the link. Applies to every ecosystem's files and to dry /// runs, so a dry run predicts the refusal. /// +/// On disk and in memory: a candidate file that is not UTF-8 text, when a +/// candidate of its ecosystem could rewrite it (#721). +/// /// In memory, additionally: a candidate file read through a link (its bytes /// are unknown) or present without content, when a candidate of its /// ecosystem could rewrite it. @@ -1532,13 +1571,20 @@ pub fn guard( if let Some(linked) = written().find(|k| view.is_symlink(k)) { return Some(symlink_refusal(linked)); } - let ProjectView::Memory(project) = view else { - return None; - }; let candidate_ecosystems: BTreeSet<&str> = candidates .iter() .map(|c| c.dep.ecosystem.as_str()) .collect(); + if let Some(rel) = done + .undecodable_reads + .iter() + .find(|rel| file_ecosystem(rel).is_some_and(|eco| candidate_ecosystems.contains(eco))) + { + return Some(undecodable_refusal(rel)); + } + let ProjectView::Memory(project) = view else { + return None; + }; if let Some(linked) = done .symlinked_reads .iter() @@ -1696,6 +1742,95 @@ mod tests { assert!(read.unreadable_reads.is_empty()); } + /// #721: a candidate file that is not UTF-8 (a UTF-16 requirements.txt, + /// which pip reads) is refused by name, on disk and in memory alike, + /// when a candidate of its ecosystem could rewrite it, instead of being + /// treated as absent (exit 0, nothing pinned, no diagnostic). + #[tokio::test] + async fn an_undecodable_candidate_file_refuses_its_ecosystem() { + let purl = "pkg:pypi/six@1.16.0"; + let uuid = "u-721"; + let mut refs = HashMap::new(); + refs.insert( + uuid.to_string(), + reference(serde_json::json!({ + "status": "granted", + "url": format!("https://patch.example/patch/pypi/six/1.16.0/tok/{uuid}/six-1.16.0-py2.py3-none-any.whl"), + "purl": purl, + "artifacts": [{"kind": "tarball", "url": null, "integrity": {"sha256": "ab"}}], + "registryOverride": null + })), + ); + let selected = vec![(purl.to_string(), uuid.to_string())]; + let mut skipped = Vec::new(); + let candidates = build_candidates(&selected, &refs, &mut skipped); + assert_eq!(candidates.len(), 1, "{skipped:?}"); + let utf16: Vec = [0xFF, 0xFE] + .into_iter() + .chain( + "idna==3.7\r\nsix==1.16.0\r\n" + .encode_utf16() + .flat_map(u16::to_le_bytes), + ) + .collect(); + let outer = OuterAllowRemote::default; + let options = || RewriteOptions { + dry_run: false, + targets_pipenv_lock: false, + pipenv_major: None, + pipenv_unknown_detail: String::new(), + trust_lockfile_config: true, + npm_allow_remote_config: true, + npm_outer: &outer, + blocking: false, + }; + + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("requirements.txt"), &utf16).unwrap(); + let mut memory = MemoryProject::new(); + memory.insert( + "requirements.txt", + MemoryEntry::Binary(utf16.clone().into()), + ); + for view in [ProjectView::Disk(tmp.path()), ProjectView::Memory(&memory)] { + let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; + assert_eq!(read.undecodable_reads, vec!["requirements.txt"]); + let done = rewrite( + &view, + read, + &candidates, + BTreeMap::new(), + &BTreeSet::new(), + &[], + options(), + ) + .await; + let refusal = guard(&view, &done, &candidates).expect("refused"); + assert_eq!(refusal.code, UNREADABLE_REFUSAL); + assert!( + refusal.message.contains("requirements.txt") && refusal.message.contains("UTF-8"), + "{}", + refusal.message + ); + + // Another ecosystem's run is not blocked by it. + let (cargo_selected, cargo_refs) = cargo_reference("u-2"); + let cargo = build_candidates(&cargo_selected, &cargo_refs, &mut Vec::new()); + let read = read_candidate_files(&view, &BTreeSet::new(), &cargo).await; + let done = rewrite( + &view, + read, + &cargo, + BTreeMap::new(), + &BTreeSet::new(), + &[], + options(), + ) + .await; + assert!(guard(&view, &done, &cargo).is_none()); + } + } + /// A hosted URL left in a berry project's `package.json` `resolutions` /// while `yarn.lock` still resolves the registry entry confirms nothing: /// only the lock pin installs (#404). diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index e11aa036d..e599e5a78 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -1322,6 +1322,7 @@ mod tests { files: BTreeMap::new(), symlinked_reads: Vec::new(), unreadable_reads: Vec::new(), + undecodable_reads: Vec::new(), overrides: Vec::new(), rewrite, rewritten: files.iter().map(|(rel, _)| (*rel).to_string()).collect(), diff --git a/crates/socket-patch-core/src/utils/requirements.rs b/crates/socket-patch-core/src/utils/requirements.rs index b13f634d3..d22edd761 100644 --- a/crates/socket-patch-core/src/utils/requirements.rs +++ b/crates/socket-patch-core/src/utils/requirements.rs @@ -14,6 +14,42 @@ //! `--hash=sha256:ab#cd` are data. Exactly one leading BOM is encoding, not //! data (pip decodes with utf-8-sig; uv strips it too). +/// Decode a requirements file the way pip's `auto_decode` does: a UTF-16 +/// or UTF-32 byte-order mark selects that encoding and is dropped; anything +/// else is UTF-8, its one leading BOM kept for [`logical_lines`] to drop. +/// Windows PowerShell 5.1 writes `pip freeze > requirements.txt` as UTF-16 +/// LE with a BOM, and pip installs from it (#721). pip tries the UTF-16 +/// marks first, so a UTF-32 LE mark (`FF FE 00 00`) reads as UTF-16 LE, as +/// it does for pip. `None` when the bytes are not valid in that encoding. +/// (pip's last resort, the locale's encoding for a mark-less non-UTF-8 +/// file, is machine-dependent and not modelled.) +pub(crate) fn decode(bytes: &[u8]) -> Option { + fn utf16(body: &[u8], unit: fn([u8; 2]) -> u16) -> Option { + if !body.len().is_multiple_of(2) { + return None; + } + char::decode_utf16(body.chunks_exact(2).map(|c| unit([c[0], c[1]]))) + .collect::>() + .ok() + } + if let Some(body) = bytes.strip_prefix(&[0xFF, 0xFE]) { + return utf16(body, u16::from_le_bytes); + } + if let Some(body) = bytes.strip_prefix(&[0xFE, 0xFF]) { + return utf16(body, u16::from_be_bytes); + } + if let Some(body) = bytes.strip_prefix(&[0x00, 0x00, 0xFE, 0xFF]) { + if !body.len().is_multiple_of(4) { + return None; + } + return body + .chunks_exact(4) + .map(|c| char::from_u32(u32::from_be_bytes([c[0], c[1], c[2], c[3]]))) + .collect(); + } + String::from_utf8(bytes.to_vec()).ok() +} + /// One logical requirements line. pub(crate) struct LogicalLine { /// 0-based index of the first physical line. @@ -236,6 +272,34 @@ pub(crate) fn url_sha256_fragment(location: &str) -> Option { mod tests { use super::*; + /// #721: pip's `auto_decode` BOM table, in pip's order. + #[test] + fn decode_follows_pips_byte_order_marks() { + let text = "six==1.16.0\r\n"; + let le: Vec = text.encode_utf16().flat_map(u16::to_le_bytes).collect(); + let be: Vec = text.encode_utf16().flat_map(u16::to_be_bytes).collect(); + let be32: Vec = text + .chars() + .flat_map(|c| (c as u32).to_be_bytes()) + .collect(); + let with = |bom: &[u8], body: &[u8]| [bom, body].concat(); + assert_eq!(decode(text.as_bytes()).as_deref(), Some(text)); + // The UTF-8 mark is left for `logical_lines`. + let bom8 = with(&[0xEF, 0xBB, 0xBF], text.as_bytes()); + assert_eq!(decode(&bom8).as_deref(), Some("\u{feff}six==1.16.0\r\n")); + assert_eq!(decode(&with(&[0xFF, 0xFE], &le)).as_deref(), Some(text)); + assert_eq!(decode(&with(&[0xFE, 0xFF], &be)).as_deref(), Some(text)); + assert_eq!( + decode(&with(&[0x00, 0x00, 0xFE, 0xFF], &be32)).as_deref(), + Some(text) + ); + // Not valid in the encoding the mark selects (or mark-less and not + // UTF-8): unreadable, never guessed. + assert_eq!(decode(&with(&[0xFF, 0xFE], &le[1..])), None); + assert_eq!(decode(&with(&[0xFF, 0xFE], &[0x00, 0xD8])), None); + assert_eq!(decode(&[b's', 0xC3, 0x28]), None); + } + #[test] fn requires_hashes_reads_pip_hash_checking_mode() { for hashed in [ diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs index 1647ce0df..a9e6d6fc6 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/pypi.rs @@ -704,11 +704,17 @@ async fn inventory_requirements_txt(view: &ProjectView<'_>) -> Option) -> Option> { use crate::vendor::pypi_requirements::{is_in_root_rel, requirements_includes}; const ROOT: &str = "requirements.txt"; - let root = view.read_text(ROOT).await.ok()?; + let read = |rel: String| async move { + let bytes = view.read_bytes(&rel).await.ok()?; + crate::utils::requirements::decode(&bytes) + }; + let root = read(ROOT.to_string()).await?; let mut visited = std::collections::HashSet::from([ROOT.to_string()]); let mut stack: Vec = requirements_includes(ROOT, &root); stack.reverse(); @@ -717,7 +723,7 @@ async fn requirements_tree(view: &ProjectView<'_>) -> Option> { if !is_in_root_rel(&rel) || !visited.insert(rel.clone()) { continue; } - let Ok(text) = view.read_text(&rel).await else { + let Some(text) = read(rel.clone()).await else { continue; }; let mut includes = requirements_includes(&rel, &text); diff --git a/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs b/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs index e6392c30a..da917805e 100644 --- a/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs +++ b/crates/socket-patch-core/src/vendor/lock_inventory/tests.rs @@ -3072,6 +3072,56 @@ async fn requirements_in_root_includes_are_inventoried() { assert_eq!(sorted_pairs(&in_memory), sorted_pairs(&entries)); } +/// #721: pip decodes a requirements file by its BOM, so a UTF-16 root +/// file and a UTF-16 include (what Windows PowerShell 5.1's `pip freeze >` +/// writes) are inventoried like their UTF-8 text, on disk and in memory. +#[tokio::test] +async fn requirements_utf16_files_are_inventoried() { + fn utf16(text: &str, le: bool) -> Vec { + let mut out = if le { + vec![0xFF, 0xFE] + } else { + vec![0xFE, 0xFF] + }; + for unit in text.encode_utf16() { + out.extend(if le { + unit.to_le_bytes() + } else { + unit.to_be_bytes() + }); + } + out + } + for le in [true, false] { + let root_bytes = utf16("-r requirements/base.txt\r\nidna==3.7\r\n", le); + let base_bytes = utf16("six==1.16.0\r\n", !le); + let tmp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(tmp.path().join("requirements")).unwrap(); + std::fs::write(tmp.path().join("requirements.txt"), &root_bytes).unwrap(); + std::fs::write(tmp.path().join("requirements/base.txt"), &base_bytes).unwrap(); + let entries = inventory_pypi_locks(tmp.path()).await.unwrap(); + assert_eq!( + sorted_pairs(&entries), + vec![ + ("idna".to_string(), "3.7".to_string()), + ("six".to_string(), "1.16.0".to_string()), + ], + "le={le}: {entries:?}" + ); + + let mut project = MemoryProject::new(); + project.insert("requirements.txt", MemoryEntry::Binary(root_bytes.into())); + project.insert( + "requirements/base.txt", + MemoryEntry::Binary(base_bytes.into()), + ); + let in_memory = super::pypi::inventory_pypi_locks_in(&ProjectView::Memory(&project)) + .await + .unwrap(); + assert_eq!(sorted_pairs(&in_memory), sorted_pairs(&entries)); + } +} + /// pip applies an index option from ANY file of the tree globally, so an /// `--index-url` inside an include keeps the root file's hashed pins /// unverifiable too (the `public_index` rule spans the whole tree). From b3daafc4691d0ff3a16f0ac5b22db00101b4381e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 22:26:59 +0000 Subject: [PATCH 3/9] Refuse UTF-16 files before reverting a takeover A vendored project switching to hosted mode had its vendored wiring reverted before the new non-UTF-8 check ran. A UTF-16 requirements.txt then refused the run with the vendored package already unwired, so it installed unpatched in both modes, and --dry-run predicted success. The check now runs before any revert, wet or dry. Vendored mode also named only "cannot read" for a UTF-16 root requirements.txt and silently skipped a UTF-16 -r include that pip installs from. Both now refuse by name with a re-save-as-UTF-8 hint. Refs #721 Assisted-by: Claude Code:claude-opus-5-5 --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 +- .../src/commands/scan/hosted.rs | 20 +++++++ .../tests/mode_migration_pypi.rs | 58 +++++++++++++++++++ crates/socket-patch-core/src/hosted/engine.rs | 23 +++++--- .../src/vendor/pypi_requirements.rs | 47 +++++++++++++++ 5 files changed, 142 insertions(+), 8 deletions(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 5ffc851eb..e08196163 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -165,7 +165,7 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc The rewriter reads a fixed set of candidate files from the project root: the npm-family locks (`package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `shrinkwrap.yaml`, `yarn.lock`, plus `.yarnrc.yml` for the berry cache-config gate, `bun.lock` / `bun.lockb`, and `vlt-lock.json` with `vlt.json` and `node_modules/.vlt-lock.json` read only), `requirements.txt` / `uv.lock` / `Pipfile.lock` (pipfile-spec 6; see the Pipenv section below) / `poetry.lock` (every Poetry lock generation from 1.0 on — the 0.12 `[metadata.hashes]` layout is refused because that installer ignores URL sources; a Poetry < 1.4 writer additionally gets `redirect_poetry_stale_install_risk`, see `docs/testing/poetry-compatibility.md`) / `pdm.lock` (PDM lock formats `2` and `4.3`–`4.5.1`; the identity-losing `3.1` / `4.0`–`4.2` formats and unknown future formats are refused with `redirect_pdm_refused`, and a lock-format-`2` writer additionally gets `redirect_pdm_legacy_sync_required`, see `docs/testing/pdm-compatibility.md`; when `uv.lock` or `poetry.lock` sits beside it they drive and `pdm.lock` is left alone), `Cargo.toml` / `Cargo.lock` / `.cargo/config.toml` (plus the legacy extensionless `.cargo/config` — cargo reads that spelling in preference when both exist, so the managed `[registries.…]` block is written into whichever one is present; **cargo also reads every workspace-member manifest** — the `[workspace] members` globs minus `exclude` — and every in-root path-dependency manifest, recursively, reached without crossing a symbolic link and never under `.socket/`, and pins the crate in each one that declares it, so those `/Cargo.toml` files can appear in `rewrittenFiles`. A crate is redirected only when every declaration pins and every other `Cargo.lock` package depending on it is a planned member: one a registry or git crate — or a path package outside the root or behind a link — also depends on is refused `redirect_cargo_transitive_dependents` (a pin reaches only the declarations it sits on), a crate no manifest declares keeps `redirect_cargo_toml_dep_not_found` with a transitive-only detail naming `--mode vendored`, a crate every declaration of which requires another version (no requirement accepts the patched version) is refused `redirect_cargo_toml_dep_unrewritable`, and so is a requirement that also matches another locked version of the crate — each a transactional skip, never recorded or attested. With NO `Cargo.lock` there is no resolved graph to ask, so the dependents question is answered from the manifests instead: a crate declared beside any other dependency — anything but a path dependency on a manifest this run also pins, or a `workspace = true` inheritor of a table it scans — or beside a workspace member this run did not read (a `members` glob, or a member outside the project or behind a symbolic link, which member discovery drops) is refused `redirect_cargo_lockless_dependents`, whose detail names the remedies (commit a lockfile, or `--mode vendored`); a project whose only dependency is the patched crate has nothing that could pull it in and still redirects. All-CRLF manifests, locks and configs are rewritten with CRLF kept (mixed endings keep refusing where the grammar does not match), and `remove` / rollback match the recorded fragments across a later CRLF↔LF checkout conversion), `composer.lock`, `nuget.config` / `packages.lock.json`, `Gemfile` / `Gemfile.lock`, `pom.xml` (+ `.mvn/maven.config` / `.mvn/checksums/checksums.sha256` for maven Trusted Checksums merge, and the Gradle build scripts read only to trigger the manual-snippet warning). **npm-family flavor coverage**: package-lock / npm-shrinkwrap, pnpm (root OR any nested `*/pnpm-lock.yaml`), yarn classic, **yarn berry** (the pin yarn writes for a root `resolutions` entry: the root `package.json` — edited only beside a berry `yarn.lock` — gains one `"@npm:": ""` selector per locked range (`redirect_yarn_berry_resolution` edits), and only that `yarn.lock` entry is re-keyed `"@"` with the same `resolution:` + `yarnBerry10c0` checksum (`redirect_yarn_berry_entry`), moved to yarn's key order; never an `npm:` locator, whose fetcher sends npm registry auth to the patch host, nor a tarball locator under an `npm:` key, which hardened mode rejects (YN0078). An older release's `npm:::__archiveUrl=` pin is still recognized and is re-pinned on the next run; rollback rebuilds the key from the selectors and drops them. Refused, nothing written: a user-authored `resolutions` entry for the package `redirect_yarn_berry_resolutions_conflict`, no root manifest `redirect_yarn_berry_manifest_missing`, a builtin `patch:` entry wrapping the same descriptor `redirect_yarn_berry_shared_descriptor`, an artifact URL yarn cannot fetch as a tarball `redirect_yarn_berry_artifact_url_unsupported`; cacheKey `10c0` and `.yarnrc.yml compressionLevel 0` gated by `redirect_yarn_berry_cache_unsupported`), and **bun** (text `bun.lock` lockfileVersion 0, 1 or 2 — 0 is the `--save-text-lockfile` opt-in lock of Bun 1.1.39–1.1.45, 1 the 1.2–1.3 default, 2 the 1.4+ default; all three emit one `packages` grammar, so the registry 4-tuple → URL 3-tuple rewrite is version-independent and the lock's own version line is kept. Any other or missing version, or a `packages` section outside bun's single-line grammar, is refused `redirect_bun_lock_unsupported` — the detail is the shared version gate's text (a newer version: update socket-patch, re-locking would reproduce it; no integer: re-lock with Bun ≥ 1.2), identical to the vendored refusal. A version-0 lock holding `workspace:` packages is refused `redirect_bun_workspace_unsupported` (its 2-tuple workspace grammar cannot keep the hosted tuple through a frozen install); the remedy is to delete `bun.lock` and re-run `bun install` with Bun ≥ 1.2, which writes lockfileVersion 1 (accepted). A plain in-place `bun install` bumps the version only when a workspace depends on another workspace (e.g. root → member — the shape the matrix measured); otherwise Bun 1.2.0 keeps version 0 and Bun 1.2.23+ fail to resolve, so the in-place bump is not the documented remedy. Bun lock version, grammar and workspace compatibility are checked before a vendored takeover, including during dry-run: these refusals preserve the existing lock, artifact and vendor ledger. Version-1 and version-2 workspace locks are rewritten, nested versions included. A granted dep with no rewritable entry warns `redirect_bun_entry_not_found`, a grant without a sha512 `redirect_bun_missing_sha512`; a CRLF lock keeps `\r\n` on the rewritten line, and a hosted URL left by an earlier grant of the same `name@version` is re-pinned in place. **Digest-less re-saves (Bun 1.1.39–1.3.9)**: every text-lock Bun below 1.3.10 re-saves a URL tuple WITHOUT its `sha512` whenever the lock is re-saved for another reason (`bun add`, `bun install` after a package.json or workspace change), leaving the 2-tuple `["name@", {meta}]` — the spec Bun installs from is intact. The CLI treats that spelling as its own wiring: a repeat hosted run counts the dep as redirected (no `redirect_bun_entry_not_found`) and HEALS the line back to the 3-tuple with the current `sha512`, recording the heal as a further `redirect_bun_lock_package` edit whose `original` is the 2-tuple (a stale URL is re-pinned from either spelling); `rollback`, scoped `rollback ` / `remove ` and the vendored takeover accept the digest-less spelling of a recorded `new` line (same key, spec and meta, only the trailing `"sha512-…"` missing) and restore the recorded original over it, so the chain always unwinds to the pristine registry line. Anything else — another uuid/token, another version, a re-laid meta object — is still drift. **Native `bun.lockb`**: when no text `bun.lock` exists, binary format versions 1, 2 and 3 are read and rewritten directly. Socket Patch does not invoke Bun or convert the project to a text lockfile. Exact matching package records are rewritten to hosted tarballs with the granted integrity, preserving dependency resolution IDs, workspace/dependency topology and unrelated package metadata; binary pointers and the package metadata hash are updated. Per-package `redirect_bun_lockb_package` snapshots support scoped rollback, repeat runs, superseding grants and hosted ↔ vendored takeover. A regular binary lock is discoverable even with no Bun runtime or `node_modules`; a dry run previews the same binary edits without writing them. A malformed, unreadable, unsupported or unverified binary structure is `redirect_bun_lockb_invalid` (exit 0, `redirected: 0`), and it refuses the npm rewrite before any takeover or sibling npm-family lock mutation. A symlinked binary write target is `redirect_symlinked_file_unsupported` (exit 1, including dry-run). `bun.lock` wins when both spellings exist. Binary-only projects do not receive `redirect_npm_no_lockfile`. Measured boundaries and the real-Bun matrix: `docs/testing/bun-compatibility.md`), and **vlt** (`vlt-lock.json` without `lockfileVersion`, `0` or `1`; see the vlt hosted-mode contract below). **Rush monorepos**: when `rush.json` is present the rewriter also reads `common/config/rush/pnpm-lock.yaml` and each `common/config/subspaces//pnpm-lock.yaml` (sorted for determinism) under their repo-relative keys and repoints them in place; editing them emits `redirect_rush_repo_state_stale` when `common/config/rush/repo-state.json` exists (the `pnpmShrinkwrapHash` desync is refreshed by `rush update`, which the redirect survives). **maven** is fail-closed via version suffixing: a `mavenSuffixedVersion` + `mavenPomSha256` override pins the Socket-only `-socket.` by rewriting the literal `` (`redirect_maven_dep_version`) or adding a `` entry (`redirect_maven_dep_management_added`), plus optional Trusted Checksums (`redirect_maven_trusted_checksums`, conflicts as `redirect_maven_trusted_checksums_conflict`; when `.mvn/wrapper/maven-wrapper.properties` pins a Maven older than 3.9.4, which ignores those files, the additive warning `redirect_maven_trusted_checksums_unenforced`); a `${property}` version is refused (`redirect_maven_dep_unpinned`), a non-matching literal skipped (`redirect_maven_dep_version_mismatch`), and an override without a suffixed version falls back to same-GAV repository injection (`redirect_maven_same_gav_fallback`, NOT fail-closed). -**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". +**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". **Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery) for each confirmed gem redirect and judges the materialization against the patch record's `afterHash` file map. Judgment rules: records are found **by uuid** among this run's fetched records (v5.0: hosted mode persists no records, so a purl whose `/patches/view` fetch failed this run is not judged; the warning re-fires on every re-scan whose fetch succeeds, until the stale materialization is gone); a materialization with every file at `afterHash` is already patched and never warns (an agent→hosted migration stays quiet by construction), and when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — at least one record file whose bytes were actually read and hash to neither state's expectation — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `redirect.warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, else `vendor/cache`; a relative value is read against the project root. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app config is skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 97e6866ce..5742a8c55 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1743,6 +1743,26 @@ async fn vendored_takeover( .filter(|_| entry.is_some_and(vlt_entry)) }) }; + // NON-UTF-8 PRE-CHECK (#721) — the GUARD's undecodable-file rule + // (`engine::undecodable_guard`), checked BEFORE any revert dispatches + // (and under --dry-run too): a takeover that reverted first and was + // then refused by the guard would leave the reverted purls unpatched + // in both modes. + if takeover.iter().any(|(_, entry)| entry.is_some()) { + let view = socket_patch_core::vendor::lock_inventory::ProjectView::Disk(&common.cwd); + let read = socket_patch_core::hosted::engine::read_candidate_files( + &view, + &std::collections::BTreeSet::new(), + candidates, + ) + .await; + if let Some(refusal) = socket_patch_core::hosted::engine::undecodable_guard( + &read.undecodable_reads, + candidates, + ) { + return Err(refusal); + } + } // SYMLINK PRE-CHECK for the takeover reverts — the same rule as the // SYMLINK GUARD below, applied to each ledger entry's recorded wiring // (the revert backends also stage and rename over the file). Checked diff --git a/crates/socket-patch-cli/tests/mode_migration_pypi.rs b/crates/socket-patch-cli/tests/mode_migration_pypi.rs index b888a96c4..bd0e500ca 100644 --- a/crates/socket-patch-cli/tests/mode_migration_pypi.rs +++ b/crates/socket-patch-cli/tests/mode_migration_pypi.rs @@ -449,6 +449,64 @@ async fn uv_takeover_without_wheel_metadata_fails_loudly() { /// the revert (the artifact and ledger entry are kept). The takeover must /// then refuse — keeping the ledger — rather than drop the entry and leave /// the project half vendored with no record of it. +/// #721: a non-UTF-8 candidate file (here a UTF-16 `pip freeze` export +/// beside a vendored Poetry project) refuses the hosted run BEFORE the +/// takeover reverts anything, wet and `--dry-run` alike: refusing only at +/// the rewrite would leave the reverted poetry.lock unpatched in both modes. +#[tokio::test] +async fn undecodable_candidate_refuses_before_the_takeover_reverts() { + let (_tmp, root) = project(); + std::fs::write( + root.join("pyproject.toml"), + "[tool.poetry]\nname = \"demo\"\nversion = \"0.1.0\"\ndescription = \"\"\nauthors = [\"x \"]\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = \">=3.9\"\nsix = \"1.16.0\"\n", + ) + .unwrap(); + std::fs::write( + root.join("poetry.lock"), + POETRY_LOCK + .replace("WHEEL_SHA", WHEEL_SHA) + .replace("SDIST_SHA", SDIST_SHA), + ) + .unwrap(); + vendor_project(&root, &["poetry.lock", "pyproject.toml"]); + let mut utf16 = vec![0xFF, 0xFE]; + for unit in "six==1.16.0\r\n".encode_utf16() { + utf16.extend(unit.to_le_bytes()); + } + std::fs::write(root.join("requirements.txt"), &utf16).unwrap(); + let lock = std::fs::read_to_string(root.join("poetry.lock")).unwrap(); + let state = root.join(".socket/vendor/state.json"); + + let server = MockServer::start().await; + mount_hosted_api(&server, true).await; + let uri = server.uri(); + for dry_run in [true, false] { + let mut args = hosted_scan_args(&uri); + if dry_run { + args.push("--dry-run"); + } + let (code, env) = run_cli(&root, &args, &[]); + assert_eq!(code, 1, "dry_run={dry_run}: {env:#}"); + let text = env.to_string(); + assert!( + text.contains("candidate_file_unreadable") && text.contains("requirements.txt"), + "dry_run={dry_run}: {env:#}" + ); + assert!( + !text.contains("redirect_takeover_reverted_vendored"), + "dry_run={dry_run}: nothing is reverted: {env:#}" + ); + assert_eq!( + std::fs::read_to_string(root.join("poetry.lock")).unwrap(), + lock, + "dry_run={dry_run}: the vendored lock is untouched" + ); + assert!(std::fs::read_to_string(&state).unwrap().contains(UUID)); + assert!(root.join(format!(".socket/vendor/pypi/{UUID}")).exists()); + assert_eq!(std::fs::read(root.join("requirements.txt")).unwrap(), utf16); + } +} + #[tokio::test] async fn drifted_vendored_line_refuses_takeover() { let (_tmp, root) = project(); diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 945320dba..b3f754b9d 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -1541,6 +1541,19 @@ fn file_ecosystem(rel: &str) -> Option<&'static str> { .then_some("pypi") } +/// The [`guard`]'s non-UTF-8 rule on its own (#721): the first of +/// `undecodable` (a [`CandidateFiles::undecodable_reads`]) whose ecosystem +/// has a candidate refuses the run. The vendored→hosted takeover runs it +/// before reverting anything, so a refusal never strands a reverted purl. +pub fn undecodable_guard(undecodable: &[String], candidates: &[Candidate]) -> Option { + undecodable + .iter() + .find(|rel| { + file_ecosystem(rel).is_some_and(|eco| candidates.iter().any(|c| c.dep.ecosystem == eco)) + }) + .map(|rel| undecodable_refusal(rel)) +} + /// SYMLINK GUARD — fail-closed, whole rewrite, before the ledger and before /// any write (hosted rewrites are transactional). The writer stages next to /// the path and renames over it, which REPLACES a symbolic link with a @@ -1571,17 +1584,13 @@ pub fn guard( if let Some(linked) = written().find(|k| view.is_symlink(k)) { return Some(symlink_refusal(linked)); } + if let Some(refusal) = undecodable_guard(&done.undecodable_reads, candidates) { + return Some(refusal); + } let candidate_ecosystems: BTreeSet<&str> = candidates .iter() .map(|c| c.dep.ecosystem.as_str()) .collect(); - if let Some(rel) = done - .undecodable_reads - .iter() - .find(|rel| file_ecosystem(rel).is_some_and(|eco| candidate_ecosystems.contains(eco))) - { - return Some(undecodable_refusal(rel)); - } let ProjectView::Memory(project) = view else { return None; }; diff --git a/crates/socket-patch-core/src/vendor/pypi_requirements.rs b/crates/socket-patch-core/src/vendor/pypi_requirements.rs index 70e771ec5..7cba17c9a 100644 --- a/crates/socket-patch-core/src/vendor/pypi_requirements.rs +++ b/crates/socket-patch-core/src/vendor/pypi_requirements.rs @@ -637,6 +637,16 @@ async fn collect_requirements_files(root: &Path) -> Result, (&'stat }); Ok(true) } + // pip decodes a UTF-16 file by its BOM (#721), so a pin inside one + // is installed; wiring around it would leave that pin unpatched. + Err(e) if e.kind() == std::io::ErrorKind::InvalidData => Err(( + "pypi_no_requirements", + format!( + "{} is not UTF-8 text (for example UTF-16, which Windows PowerShell 5.1 \ + writes for `pip freeze > requirements.txt`); re-save it as UTF-8 and re-run", + path.display() + ), + )), Err(_) if out.is_empty() => Err(( "pypi_no_requirements", format!("cannot read {}", path.display()), @@ -951,6 +961,43 @@ mod tests { tmp } + /// #721: pip installs from a UTF-16 requirements file (what Windows + /// PowerShell 5.1's `pip freeze >` writes), so vendoring must refuse it + /// by name, as the root file or as an include, never wire around it. + #[tokio::test] + async fn a_utf16_requirements_file_is_refused_by_name() { + let utf16 = |text: &str| -> Vec { + let mut out = vec![0xFF, 0xFE]; + for unit in text.encode_utf16() { + out.extend(unit.to_le_bytes()); + } + out + }; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write( + tmp.path().join("requirements.txt"), + utf16("six==1.16.0\r\n"), + ) + .unwrap(); + let err = wire_requirements(tmp.path(), "six", "1.16.0", REL_WHEEL, SHA) + .await + .unwrap_err(); + assert_eq!(err.0, "pypi_no_requirements"); + assert!( + err.1.contains("requirements.txt is not UTF-8 text"), + "{}", + err.1 + ); + + let tmp = write_root("-r inc.txt\nidna==3.7\n").await; + std::fs::write(tmp.path().join("inc.txt"), utf16("six==1.16.0\r\n")).unwrap(); + let err = wire_requirements(tmp.path(), "six", "1.16.0", REL_WHEEL, SHA) + .await + .unwrap_err(); + assert!(err.1.contains("inc.txt is not UTF-8 text"), "{}", err.1); + assert_eq!(read_root(tmp.path()).await, "-r inc.txt\nidna==3.7\n"); + } + async fn read_root(root: &Path) -> String { tokio::fs::read_to_string(root.join("requirements.txt")) .await From 94b3ad85d288579a19815da6fff9ed655cf767b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 18:24:21 +0000 Subject: [PATCH 4/9] Route Gradle digests through utils::digest main has failed socket-patch-core's lib tests since Gradle support (#646) and the digest helpers (#865) both landed. The guard test production_digests_go_through_the_helpers flags three files #646 added that still hash inline: crawlers/gradle_cache.rs, patch/jvm_jar.rs and patch/sidecars/maven.rs. That breaks test, test-release and coverage on every open PR. Each inline sha1/sha256 call now goes through sha1_hex_of or sha256_hex_of, which compute the same lowercase hex. Behaviour is unchanged. Assisted-by: Claude Code:claude-opus-5-5 (cherry picked from commit 659ac2c24e5c5904e743b4bc98ea4645da2ed6a1) --- crates/socket-patch-core/src/crawlers/gradle_cache.rs | 9 ++++----- crates/socket-patch-core/src/patch/jvm_jar.rs | 7 ++----- crates/socket-patch-core/src/patch/sidecars/maven.rs | 4 +--- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/crates/socket-patch-core/src/crawlers/gradle_cache.rs b/crates/socket-patch-core/src/crawlers/gradle_cache.rs index ef295ee27..afd7c4fba 100644 --- a/crates/socket-patch-core/src/crawlers/gradle_cache.rs +++ b/crates/socket-patch-core/src/crawlers/gradle_cache.rs @@ -70,8 +70,7 @@ pub fn hash_eq(dir_name: &str, sha1_hex: &str) -> bool { /// Whether `bytes` are the pristine download Gradle stored in the hash /// directory `dir_name` (their sha1 names it). pub fn pristine(dir_name: &str, bytes: &[u8]) -> bool { - use sha1::{Digest, Sha1}; - hash_eq(dir_name, &hex::encode(Sha1::digest(bytes))) + hash_eq(dir_name, &crate::utils::digest::sha1_hex_of(bytes)) } /// Whether `path` is a version directory of a `files-2.1` tree @@ -432,8 +431,6 @@ impl DerivedIndex { /// The [`DerivedCopies`] of the jar `jar_leaf` whose pristine bytes /// hash to `pristine_sha1`. pub fn query(&self, jar_leaf: &str, pristine_sha1: &str) -> DerivedCopies { - use sha1::{Digest, Sha1}; - let instrumented = format!("instrumented-{jar_leaf}"); let mut out = DerivedCopies { incomplete: self.incomplete, @@ -460,7 +457,9 @@ impl DerivedIndex { out.stale.push(path.clone()); } else if name == jar_leaf || name == instrumented { match crate::utils::fs::read_regular_to_bytes_sync(path) { - Ok(bytes) if hash_eq(&hex::encode(Sha1::digest(&bytes)), pristine_sha1) => { + Ok(bytes) + if hash_eq(&crate::utils::digest::sha1_hex_of(&bytes), pristine_sha1) => + { out.stale.push(path.clone()) } Ok(_) => out.unknown.push(path.clone()), diff --git a/crates/socket-patch-core/src/patch/jvm_jar.rs b/crates/socket-patch-core/src/patch/jvm_jar.rs index 82d679406..f38a84403 100644 --- a/crates/socket-patch-core/src/patch/jvm_jar.rs +++ b/crates/socket-patch-core/src/patch/jvm_jar.rs @@ -25,8 +25,6 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; -use sha1::Digest as _; - use crate::crawlers::gradle_cache; use crate::hash::git_sha256::compute_git_sha256_from_bytes; use crate::manifest::schema::PatchFileInfo; @@ -353,12 +351,11 @@ fn unpatched_members( } fn sha256_hex(bytes: &[u8]) -> String { - use sha2::Digest as _; - hex::encode(sha2::Sha256::digest(bytes)) + crate::utils::digest::sha256_hex_of(bytes) } fn sha1_hex(bytes: &[u8]) -> String { - hex::encode(sha1::Sha1::digest(bytes)) + crate::utils::digest::sha1_hex_of(bytes) } /// `/jvm-originals/.jar`. diff --git a/crates/socket-patch-core/src/patch/sidecars/maven.rs b/crates/socket-patch-core/src/patch/sidecars/maven.rs index f2f5a2466..8798bfce6 100644 --- a/crates/socket-patch-core/src/patch/sidecars/maven.rs +++ b/crates/socket-patch-core/src/patch/sidecars/maven.rs @@ -17,8 +17,6 @@ use std::path::{Path, PathBuf}; -use sha1::Digest as _; - use super::{ SidecarAdvisory, SidecarAdvisoryCode, SidecarError, SidecarFile, SidecarFileAction, SidecarPayload, SidecarSeverity, @@ -44,7 +42,7 @@ impl Algo { fn digest(self, bytes: &[u8]) -> String { match self { - Algo::Sha1 => hex::encode(sha1::Sha1::digest(bytes)), + Algo::Sha1 => crate::utils::digest::sha1_hex_of(bytes), Algo::Md5 => hex::encode(md5(bytes)), } } From a91478bb460ce98f976afec117bc88a8832af3ab Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 22:24:12 +0000 Subject: [PATCH 5/9] Let Gradle refuse its own non-UTF-8 build files A non-UTF-8 settings.gradle was recorded both as an undecodable candidate (#721) and in gradle_unreadable. The run-wide #721 guard then refused the whole hosted run with candidate_file_unreadable (exit 1), pre-empting the Gradle planner's own per-build refusal (redirect_gradle_build_file_unreadable, exit 0) that main added. Files the Gradle planner already refuses are now dropped from undecodable_reads, so only that build is refused and the run goes ahead. Refs #721 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YED4tY7Yytk79MTPzLnSfA --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 +- crates/socket-patch-core/src/hosted/engine.rs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index fcd124269..815bfa52d 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -165,7 +165,7 @@ For a **9.0 root lock**, the CLI ensures `pnpm-workspace.yaml` carries `trustLoc The rewriter reads a fixed set of candidate files from the project root: the npm-family locks (`package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `shrinkwrap.yaml`, `yarn.lock`, plus `.yarnrc.yml` for the berry cache-config gate, `bun.lock` / `bun.lockb`, and `vlt-lock.json` with `vlt.json` and `node_modules/.vlt-lock.json` read only), `requirements.txt` / `uv.lock` / `Pipfile.lock` (pipfile-spec 6; see the Pipenv section below) / `poetry.lock` (every Poetry lock generation from 1.0 on — the 0.12 `[metadata.hashes]` layout is refused because that installer ignores URL sources; a Poetry < 1.4 writer additionally gets `redirect_poetry_stale_install_risk`, see `docs/testing/poetry-compatibility.md`) / `pdm.lock` (PDM lock formats `2` and `4.3`–`4.5.1`; the identity-losing `3.1` / `4.0`–`4.2` formats and unknown future formats are refused with `redirect_pdm_refused`, and a lock-format-`2` writer additionally gets `redirect_pdm_legacy_sync_required`, see `docs/testing/pdm-compatibility.md`; when `uv.lock` or `poetry.lock` sits beside it they drive and `pdm.lock` is left alone), `Cargo.toml` / `Cargo.lock` / `.cargo/config.toml` (plus the legacy extensionless `.cargo/config` — cargo reads that spelling in preference when both exist, so the managed `[registries.…]` block is written into whichever one is present; **cargo also reads every workspace-member manifest** — the `[workspace] members` globs minus `exclude` — and every in-root path-dependency manifest, recursively, reached without crossing a symbolic link and never under `.socket/`, and pins the crate in each one that declares it, so those `/Cargo.toml` files can appear in `rewrittenFiles`. A crate is redirected only when every declaration pins and every other `Cargo.lock` package depending on it is a planned member: one a registry or git crate — or a path package outside the root or behind a link — also depends on is refused `redirect_cargo_transitive_dependents` (a pin reaches only the declarations it sits on), a crate no manifest declares keeps `redirect_cargo_toml_dep_not_found` with a transitive-only detail naming `--mode vendored`, a crate every declaration of which requires another version (no requirement accepts the patched version) is refused `redirect_cargo_toml_dep_unrewritable`, and so is a requirement that also matches another locked version of the crate — each a transactional skip, never recorded or attested. With NO `Cargo.lock` there is no resolved graph to ask, so the dependents question is answered from the manifests instead: a crate declared beside any other dependency — anything but a path dependency on a manifest this run also pins, or a `workspace = true` inheritor of a table it scans — or beside a workspace member this run did not read (a `members` glob, or a member outside the project or behind a symbolic link, which member discovery drops) is refused `redirect_cargo_lockless_dependents`, whose detail names the remedies (commit a lockfile, or `--mode vendored`); a project whose only dependency is the patched crate has nothing that could pull it in and still redirects. All-CRLF manifests, locks and configs are rewritten with CRLF kept (mixed endings keep refusing where the grammar does not match), and `remove` / rollback match the recorded fragments across a later CRLF↔LF checkout conversion), `composer.lock`, `nuget.config` / `packages.lock.json`, `Gemfile` / `Gemfile.lock`, `pom.xml` (+ `.mvn/maven.config` / `.mvn/checksums/checksums.sha256` for maven Trusted Checksums merge, and, for a Gradle build, every settings, build, `buildSrc`, included-build, applied and plugin-source script, version catalog and lock file the script graph reaches, plus `gradle/verification-metadata.xml`, `gradle/wrapper/gradle-wrapper.properties` and the owned `.socket/gradle/` files). **npm-family flavor coverage**: package-lock / npm-shrinkwrap, pnpm (root OR any nested `*/pnpm-lock.yaml`), yarn classic, **yarn berry** (the pin yarn writes for a root `resolutions` entry: the root `package.json` — edited only beside a berry `yarn.lock` — gains one `"@npm:": ""` selector per locked range (`redirect_yarn_berry_resolution` edits), and only that `yarn.lock` entry is re-keyed `"@"` with the same `resolution:` + `yarnBerry10c0` checksum (`redirect_yarn_berry_entry`), moved to yarn's key order; never an `npm:` locator, whose fetcher sends npm registry auth to the patch host, nor a tarball locator under an `npm:` key, which hardened mode rejects (YN0078). An older release's `npm:::__archiveUrl=` pin is still recognized and is re-pinned on the next run; rollback rebuilds the key from the selectors and drops them. Refused, nothing written: a user-authored `resolutions` entry for the package `redirect_yarn_berry_resolutions_conflict`, no root manifest `redirect_yarn_berry_manifest_missing`, a builtin `patch:` entry wrapping the same descriptor `redirect_yarn_berry_shared_descriptor`, an artifact URL yarn cannot fetch as a tarball `redirect_yarn_berry_artifact_url_unsupported`; cacheKey `10c0` and `.yarnrc.yml compressionLevel 0` gated by `redirect_yarn_berry_cache_unsupported`), and **bun** (text `bun.lock` lockfileVersion 0, 1 or 2 — 0 is the `--save-text-lockfile` opt-in lock of Bun 1.1.39–1.1.45, 1 the 1.2–1.3 default, 2 the 1.4+ default; all three emit one `packages` grammar, so the registry 4-tuple → URL 3-tuple rewrite is version-independent and the lock's own version line is kept. Any other or missing version, or a `packages` section outside bun's single-line grammar, is refused `redirect_bun_lock_unsupported` — the detail is the shared version gate's text (a newer version: update socket-patch, re-locking would reproduce it; no integer: re-lock with Bun ≥ 1.2), identical to the vendored refusal. A version-0 lock holding `workspace:` packages is refused `redirect_bun_workspace_unsupported` (its 2-tuple workspace grammar cannot keep the hosted tuple through a frozen install); the remedy is to delete `bun.lock` and re-run `bun install` with Bun ≥ 1.2, which writes lockfileVersion 1 (accepted). A plain in-place `bun install` bumps the version only when a workspace depends on another workspace (e.g. root → member — the shape the matrix measured); otherwise Bun 1.2.0 keeps version 0 and Bun 1.2.23+ fail to resolve, so the in-place bump is not the documented remedy. Bun lock version, grammar and workspace compatibility are checked before a vendored takeover, including during dry-run: these refusals preserve the existing lock, artifact and vendor ledger. Version-1 and version-2 workspace locks are rewritten, nested versions included. A granted dep with no rewritable entry warns `redirect_bun_entry_not_found`, a grant without a sha512 `redirect_bun_missing_sha512`; a CRLF lock keeps `\r\n` on the rewritten line, and a hosted URL left by an earlier grant of the same `name@version` is re-pinned in place. **Digest-less re-saves (Bun 1.1.39–1.3.9)**: every text-lock Bun below 1.3.10 re-saves a URL tuple WITHOUT its `sha512` whenever the lock is re-saved for another reason (`bun add`, `bun install` after a package.json or workspace change), leaving the 2-tuple `["name@", {meta}]` — the spec Bun installs from is intact. The CLI treats that spelling as its own wiring: a repeat hosted run counts the dep as redirected (no `redirect_bun_entry_not_found`) and HEALS the line back to the 3-tuple with the current `sha512`, recording the heal as a further `redirect_bun_lock_package` edit whose `original` is the 2-tuple (a stale URL is re-pinned from either spelling); `rollback`, scoped `rollback ` / `remove ` and the vendored takeover accept the digest-less spelling of a recorded `new` line (same key, spec and meta, only the trailing `"sha512-…"` missing) and restore the recorded original over it, so the chain always unwinds to the pristine registry line. Anything else — another uuid/token, another version, a re-laid meta object — is still drift. **Native `bun.lockb`**: when no text `bun.lock` exists, binary format versions 1, 2 and 3 are read and rewritten directly. Socket Patch does not invoke Bun or convert the project to a text lockfile. Exact matching package records are rewritten to hosted tarballs with the granted integrity, preserving dependency resolution IDs, workspace/dependency topology and unrelated package metadata; binary pointers and the package metadata hash are updated. Per-package `redirect_bun_lockb_package` snapshots support scoped rollback, repeat runs, superseding grants and hosted ↔ vendored takeover. A regular binary lock is discoverable even with no Bun runtime or `node_modules`; a dry run previews the same binary edits without writing them. A malformed, unreadable, unsupported or unverified binary structure is `redirect_bun_lockb_invalid` (exit 0, `redirected: 0`), and it refuses the npm rewrite before any takeover or sibling npm-family lock mutation. A symlinked binary write target is `redirect_symlinked_file_unsupported` (exit 1, including dry-run). `bun.lock` wins when both spellings exist. Binary-only projects do not receive `redirect_npm_no_lockfile`. Measured boundaries and the real-Bun matrix: `docs/testing/bun-compatibility.md`), and **vlt** (`vlt-lock.json` without `lockfileVersion`, `0` or `1`; see the vlt hosted-mode contract below). **Rush monorepos**: when `rush.json` is present the rewriter also reads `common/config/rush/pnpm-lock.yaml` and each `common/config/subspaces//pnpm-lock.yaml` (sorted for determinism) under their repo-relative keys and repoints them in place; editing them emits `redirect_rush_repo_state_stale` when `common/config/rush/repo-state.json` exists (the `pnpmShrinkwrapHash` desync is refreshed by `rush update`, which the redirect survives). **maven** is fail-closed via version suffixing: a `mavenSuffixedVersion` + `mavenPomSha256` override pins the Socket-only `-socket.` by rewriting the literal `` (`redirect_maven_dep_version`) or adding a `` entry (`redirect_maven_dep_management_added`), plus optional Trusted Checksums (`redirect_maven_trusted_checksums`, conflicts as `redirect_maven_trusted_checksums_conflict`; when `.mvn/wrapper/maven-wrapper.properties` pins a Maven older than 3.9.4, which ignores those files, the additive warning `redirect_maven_trusted_checksums_unenforced`); a `${property}` version is refused (`redirect_maven_dep_unpinned`), a non-matching literal skipped (`redirect_maven_dep_version_mismatch`), and an override without a suffixed version falls back to same-GAV repository injection (`redirect_maven_same_gav_fallback`, NOT fail-closed). **gradle** (v5.0) is automated wiring, no longer a pasted snippet: the owned settings script `.socket/gradle/socket-patch.hosted.settings.gradle` with its index `.socket/gradle/hosted-index.tsv`, one apply line per build's settings file, every lock entry of the GA moved to the suffixed version, and the suffixed component in an existing `gradle/verification-metadata.xml`. A refused dep writes nothing and keeps `redirect_gradle_manual_snippet` as its fallback; same-GAV grants are refused (`redirect_gradle_same_gav_unsupported`). Rules, refusals and codes: [Gradle builds](#gradle-builds-v50). -**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". +**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. A Gradle build file the Gradle planner reaches is the exception: it keeps that planner's own per-build refusal (`redirect_gradle_build_file_unreadable`, exit 0), and the rest of the run goes ahead. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". **Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery, plus — read-only — a `.bundle/config` bundle path refused as a write root because it resolves outside the project, which takes the project-local remedy) for each confirmed gem redirect and judges the materialization against the patch record's `afterHash` file map. Judgment rules: records are found **by uuid** among this run's fetched records (v5.0: hosted mode persists no records, so a purl whose `/patches/view` fetch failed this run is not judged; the warning re-fires on every re-scan whose fetch succeeds, until the stale materialization is gone); a materialization with every file at `afterHash` is already patched and never warns (an agent→hosted migration stays quiet by construction), and when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — at least one record file whose bytes were actually read and hash to neither state's expectation — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `redirect.warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, then `BUNDLE_CACHE_PATH:` in the global config (`bundle config set --global`: `$BUNDLE_CONFIG`, else `$BUNDLE_USER_CONFIG`, else `$BUNDLE_USER_HOME/config`, else `~/.bundle/config`), else `vendor/cache`; a relative value is read against the project root. The same global tier, below the app config and the environment, applies to `BUNDLE_GEMFILE:` and, for agent-mode install-root discovery, to `BUNDLE_PATH:`. A present local or environment `path`, `path.system`, or `disable_shared_gems` setting (including an empty string or false flag) shadows the global path tier, matching the tested Bundler 2.6/4 behavior; Bundler 1.x's legacy global-path shortcut is not modeled. An empty higher-tier `gemfile` setting also shadows the global value but leaves an existing nonempty `BUNDLE_GEMFILE` environment value in effect, or uses default manifest discovery when there is none. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app and global configs are skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index ab019d44e..c786cb815 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -584,6 +584,12 @@ pub async fn read_candidate_files( && crate::patch::redirect::gradle::gradle_build_present(&out.files) { read_gradle_files(view, unreadable, &mut out).await; + // The Gradle planner refuses a build over a file it cannot read as + // text and the scan carries on, so such a file is not a reason to + // refuse the whole run (#721). + let gradle_unreadable = &out.gradle_unreadable; + out.undecodable_reads + .retain(|rel| !gradle_unreadable.contains(rel)); } out.symlinked_reads.sort(); out.symlinked_reads.dedup(); @@ -2463,6 +2469,14 @@ mod tests { "{:?}", read.gradle_unreadable ); + // Only the Gradle build is refused, not the whole run (#721). + assert!( + !read + .undecodable_reads + .contains(&"settings.gradle".to_string()), + "{:?}", + read.undecodable_reads + ); assert!(done.rewrite.refused_gradle_uuids.contains(GRADLE_UUID)); assert!( !done.rewrite.files.contains_key("settings.gradle"), From 92a44fd5a676105505f7ab71b94e7a369490f221 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 14:43:26 +0000 Subject: [PATCH 6/9] Leave Gradle and sbt files to their own refusals The run-wide non-UTF-8 check (#721) pre-empted two refusals main owns: - An unreadable socket-patch.sbt was refused as candidate_file_unreadable instead of redirect_sbt_owned_file_unreadable, failing e2e_sbt_hosted::hosted_unreadable_owned_file_is_refused_untouched. - With no readable Gradle build, a stray non-UTF-8 Gradle file (a lone settings.gradle or a gradle.lockfile) refused a whole Maven run, pom.xml included, although the Gradle planner never runs and never rewrites it (Bugbot). On disk, socket-patch.sbt is now left to the sbt refusal, and with no readable Gradle build, Gradle-owned files no longer refuse the run. Refs #721 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YED4tY7Yytk79MTPzLnSfA --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 +- crates/socket-patch-core/src/hosted/engine.rs | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 867dfb645..3a7a77efa 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -167,7 +167,7 @@ The rewriter reads a fixed set of candidate files from the project root: the npm **Hosted sbt (v5.0, additive)**: an sbt build root (`project/build.properties` naming an `sbt.version`, 0.13.18 or later) is wired through ONE generated root file, `socket-patch.sbt` — no user file is edited. It pins every granted Maven patch build-wide (a `ThisBuild` `dependencyOverrides +=` of the Socket-only `-socket.` version plus a `file:` resolver over `.socket/sbt-hosted/maven2/`, moved ahead of the default repositories on sbt 0.13 / 1.x so an unreachable one never blocks it offline), downloads the pinned pom and jar there on the first sbt load (sha256-checked, gitignored by the file itself), and installs a load-time verifier that fails `update` when any project resolves another version or a pinned artifact whose bytes are not pinned. Edits: `redirect_sbt_pin` (added), `redirect_sbt_pin_updated` (an existing row replaced: same GA and base under a new uuid, or the same uuid with new served values; `original` names the previous uuid and version), `redirect_sbt_pin_rechecked` (an existing row re-verified after the build's dependencies changed: its dependency digest is recorded anew, `original`/`new` are `{deps}`). The load-time verifier also fails `update` when a project declares a pinned GA at a version newer than the pin's base (the build-wide override would otherwise force it back down). A new pin is gated on sbt's own resolution records under `target/` (never the machine-wide cache): run-level stops wire nothing, warn once and exit 0 — `redirect_sbt_no_resolution_evidence` (none; run `sbt update` first; always the in-memory engine's answer), `redirect_sbt_resolution_incomplete` (a declared project left no evidence, or the project definitions cannot be read statically), `redirect_sbt_resolution_stale` (a build source is newer than some project's evidence: each project is dated by its own newest record, so a partial `sbt /update` does not vouch for the others). Per-patch refusals (never confirmed): `redirect_sbt_missing_override` (no `maven2` override or no suffixed version), `redirect_sbt_integrity_missing` (jar or pom sha256 missing), `redirect_sbt_unsafe_value` (a value unsafe in a Scala literal, or an index URL not naming the uuid), `redirect_sbt_version_conflict` (some project resolves another version, or a build source declares the GA newer than the patch's base), `redirect_sbt_override_conflict` (two patches for one GA in a run, or another base already pinned), `redirect_sbt_vendored_conflict` (the GA is pinned by `socket-patch-vendor.sbt`, or that file cannot be parsed — then every Maven patch), `redirect_sbt_owned_file_modified` / `redirect_sbt_owned_file_foreign` (`socket-patch.sbt` edited, or not socket-patch's — every Maven patch), `redirect_sbt_owned_file_unreadable` (a whole-run refusal: `socket-patch.sbt` is on disk but cannot be read as UTF-8 text, so writing it would replace it; nothing is written), `redirect_sbt_unsupported_version`, `redirect_sbt_build_root_unknown` (sbt files but no versioned build root — every Maven patch), `redirect_sbt_overrides_assignment` / `redirect_sbt_resolvers_assignment` (a build source reassigns `dependencyOverrides` / `resolvers` with `:=`, `~=` or `--=`), `redirect_sbt_dependency_lock_present` (a `build.sbt.lock`), `redirect_sbt_scala_runtime_unsupported` (`org.scala-lang`), `redirect_sbt_classifier_unsupported`; a GA no library configuration resolves is skipped silently (`redirect_sbt_meta_build_only` when only the meta-build resolves it). Advisories: `redirect_sbt_version_untested` (sbt 2.1+, still wired), `redirect_sbt_override_build_repos` (`sbt.override.build.repos=true`), `redirect_maven_pom_ignored_sbt_build` (a `pom.xml` beside the sbt build, which sbt never reads; the Maven rewriter still edits it for the Maven build). A re-run keeps an existing row and re-checks it. When the build's dependency digest changed since the pin, evidence resolved after the change (fresh, newer than the generated file) re-verifies it and the row's digest is refreshed (`redirect_sbt_pin_rechecked`); the uuid is NOT confirmed on `redirect_sbt_pin_declared_newer` (a build source now declares the GA newer than the pin's base; the row stays, sbt's load-time verifier fails the build, and the remedy is `socket-patch rollback` or declaring the base again), `redirect_sbt_pin_unverifiable` (the digest changed and the evidence predates the change, or the digest cannot be computed: run `sbt update`, then re-run socket-patch), `redirect_sbt_override_shadowed` (the evidence still resolves the base version) or `redirect_sbt_resolved_elsewhere` (the pinned version resolves from outside the pin repository from a file whose sha256 is not the pinned jar's; a copy holding the pinned bytes, such as the Ivy cache a second checkout reads, is fine — at most 64 pinned artifact files of up to 256 MiB are hashed, anything else counts as elsewhere), and also when a build source now reassigns `dependencyOverrides` / `resolvers` or a `build.sbt.lock` appeared (the same `redirect_sbt_overrides_assignment` / `redirect_sbt_resolvers_assignment` / `redirect_sbt_dependency_lock_present` codes; the row stays and sbt's load-time verifier fails the build). For a pure sbt root (no `pom.xml` / Gradle script beside it), maven confirmation is decided only by the sbt rewriter's report; on a mixed root a uuid the sbt rewriter refused is still confirmed by the Maven rewriter's own `pom.xml` pin (the generated sbt files never prove a pin by substring). **Mill and scala-cli** are guidance only: per Maven patch `redirect_mill_manual_snippet` / `redirect_scala_cli_manual_snippet` carry a paste-able snippet (repository + forced suffixed version), nothing is written or confirmed, and a pure Mill / scala-cli root gets no `redirect_maven_no_pom`; there, a Maven patch the server sent without a `maven2` registry override gets `redirect_maven_missing_override` instead of a snippet (with a `pom.xml` beside the Mill / scala-cli files the pom rewriter reports it). `rollback` / `remove` restore `socket-patch.sbt` offline (the rows removed, the file deleted with its last pin; the gitignored downloads are left). Manifest-less VEX reads every strictly parsed pin as a hosted reference but grants it the lockfile basis only when the local evidence shows every recorded version of the GA is the pinned one and every recorded artifact hashes to a pinned sha256 (else `sbt_resolution_unverified`). -**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. A Gradle build file the Gradle planner reaches is the exception: it keeps that planner's own per-build refusal (`redirect_gradle_build_file_unreadable`, exit 0), and the rest of the run goes ahead. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". +**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. Two exceptions keep their own refusals. A Gradle build file the Gradle planner reaches gets that planner's per-build refusal (`redirect_gradle_build_file_unreadable`, exit 0) and the rest of the run goes ahead, and a stray Gradle file with no readable Gradle build is never rewritten, so it does not refuse the run. An unreadable `socket-patch.sbt` is refused with `redirect_sbt_owned_file_unreadable`. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". **Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery, plus — read-only — a `.bundle/config` bundle path refused as a write root because it resolves outside the project, which takes the project-local remedy) for each confirmed gem redirect and judges the materialization against the patch record's `afterHash` file map. Judgment rules: records are found **by uuid** among this run's fetched records (v5.0: hosted mode persists no records, so a purl whose `/patches/view` fetch failed this run is not judged; the warning re-fires on every re-scan whose fetch succeeds, until the stale materialization is gone); a materialization with every file at `afterHash` is already patched and never warns (an agent→hosted migration stays quiet by construction), and when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — at least one record file whose bytes were actually read and hash to neither state's expectation — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `redirect.warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, then `BUNDLE_CACHE_PATH:` in the global config (`bundle config set --global`: `$BUNDLE_CONFIG`, else `$BUNDLE_USER_CONFIG`, else `$BUNDLE_USER_HOME/config`, else `~/.bundle/config`), else `vendor/cache`; a relative value is read against the project root. The same global tier, below the app config and the environment, applies to `BUNDLE_GEMFILE:` and, for agent-mode install-root discovery, to `BUNDLE_PATH:`. A present local or environment `path`, `path.system`, or `disable_shared_gems` setting (including an empty string or false flag) shadows the global path tier, matching the tested Bundler 2.6/4 behavior; Bundler 1.x's legacy global-path shortcut is not modeled. An empty higher-tier `gemfile` setting also shadows the global value but leaves an existing nonempty `BUNDLE_GEMFILE` environment value in effect, or uses default manifest discovery when there is none. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app and global configs are skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index e1f01fe92..a1cb91df9 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -620,6 +620,19 @@ pub async fn read_candidate_files( let gradle_unreadable = &out.gradle_unreadable; out.undecodable_reads .retain(|rel| !gradle_unreadable.contains(rel)); + } else { + // No readable Gradle build: the Gradle planner never runs, so a + // stray Gradle file it would own (a lone settings script, a lock) + // is never rewritten and must not refuse the rest of the run. + out.undecodable_reads + .retain(|rel| !is_gradle_owned_file(rel)); + } + // `socket-patch.sbt` the sbt planner takes for absent and would create: + // on disk the scan refuses that write with its own + // `redirect_sbt_owned_file_unreadable`, so leave it to that refusal. + if !matches!(view, ProjectView::Memory(_)) { + out.undecodable_reads + .retain(|rel| rel != crate::formats::sbt::owned_file::HOSTED_FILE); } // An sbt build's resolution evidence rides a synthetic key (see // `patch::redirect::sbt::SBT_RESOLUTION_KEY`). @@ -1945,6 +1958,19 @@ fn file_ecosystem(rel: &str) -> Option<&'static str> { .then_some("pypi") } +/// A file only the hosted Gradle planner reads or writes: a settings or +/// build script, a dependency lock, the verification metadata, the wrapper +/// properties, or the planner's own owned files. +fn is_gradle_owned_file(rel: &str) -> bool { + let base = rel.rsplit('/').next().unwrap_or(rel); + base.ends_with(".gradle") + || base.ends_with(".gradle.kts") + || base.ends_with(".lockfile") + || rel == "gradle/verification-metadata.xml" + || rel == "gradle/wrapper/gradle-wrapper.properties" + || rel.starts_with(".socket/gradle/") +} + /// The [`guard`]'s non-UTF-8 rule on its own (#721): the first of /// `undecodable` (a [`CandidateFiles::undecodable_reads`]) whose ecosystem /// has a candidate refuses the run. The vendored→hosted takeover runs it @@ -2868,6 +2894,34 @@ mod tests { assert!(text.contains("include 'core'"), "{text}"); } + /// #721 review: with no readable Gradle build the Gradle planner never + /// runs, so a stray non-UTF-8 Gradle file (a lone settings script, a + /// lock) does not refuse the rest of a Maven run. + #[tokio::test] + async fn a_stray_non_utf8_gradle_file_does_not_refuse_a_maven_run() { + const POM: &str = "com.socketfixturevictim1.10.0\n"; + let latin1: &[u8] = b"rootProject.name = 'Andr\xe9'\n"; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("pom.xml"), POM).unwrap(); + std::fs::write(tmp.path().join("settings.gradle"), latin1).unwrap(); + std::fs::write(tmp.path().join("gradle.lockfile"), latin1).unwrap(); + let mut memory = MemoryProject::new(); + memory.insert_text("pom.xml", POM); + for rel in ["settings.gradle", "gradle.lockfile"] { + memory.insert(rel, MemoryEntry::Binary(latin1.to_vec().into())); + } + let candidates = vec![gradle_candidate()]; + for view in [ProjectView::Disk(tmp.path()), ProjectView::Memory(&memory)] { + let (read, done) = gradle_rewrite_in(&view).await; + assert!( + read.undecodable_reads.is_empty(), + "{:?}", + read.undecodable_reads + ); + assert!(guard(&view, &done, &candidates).is_none()); + } + } + /// A refused Gradle build is never confirmed by a snippet pasted into a /// build script, though it names the suffixed version and the index url. #[tokio::test] From 950b9e6310368cf8fd0a25a5836d09f3882c4e30 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 14:53:39 +0000 Subject: [PATCH 7/9] Refuse an unreadable socket-patch.sbt before any takeover 92a44fd dropped socket-patch.sbt from the run-wide non-UTF-8 check so the sbt refusal could report its own code. That also hid it from the pre-takeover check, so a mixed run (a vendored PyPI purl plus sbt) could revert vendored wiring and only then be refused, leaving those packages unpatched in both modes (Bugbot). The file stays in the early check, which now refuses it with redirect_sbt_owned_file_unreadable, before any revert. Refs #721 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YED4tY7Yytk79MTPzLnSfA --- .../src/commands/scan/hosted.rs | 2 +- crates/socket-patch-core/src/hosted/engine.rs | 47 ++++++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index c9f917248..979e39302 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -2675,7 +2675,7 @@ fn created_settings_over_existing( } /// [`created_settings_over_existing`]'s code for `socket-patch.sbt`. -const SBT_OWNED_FILE_UNREADABLE: &str = "redirect_sbt_owned_file_unreadable"; +use socket_patch_core::hosted::engine::SBT_OWNED_FILE_UNREADABLE; #[cfg(test)] mod tests { diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index a1cb91df9..6c36bd7f7 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -158,7 +158,22 @@ fn unreadable_refusal(rel: &str) -> Refusal { } } +/// The refusal for an unreadable `socket-patch.sbt`, the file the sbt +/// planner owns and would otherwise create over the user's bytes. +pub const SBT_OWNED_FILE_UNREADABLE: &str = "redirect_sbt_owned_file_unreadable"; + fn undecodable_refusal(rel: &str) -> Refusal { + // Keeps the sbt planner's own refusal code, and still refuses here, + // before any vendored->hosted takeover revert. + if rel == crate::formats::sbt::owned_file::HOSTED_FILE { + return Refusal { + code: SBT_OWNED_FILE_UNREADABLE.to_string(), + message: format!( + "{rel} is not UTF-8 text, so the hosted sbt wiring would replace it; \ + re-save it as UTF-8 and re-run; nothing was written" + ), + }; + } Refusal { code: UNREADABLE_REFUSAL.to_string(), message: format!( @@ -627,13 +642,6 @@ pub async fn read_candidate_files( out.undecodable_reads .retain(|rel| !is_gradle_owned_file(rel)); } - // `socket-patch.sbt` the sbt planner takes for absent and would create: - // on disk the scan refuses that write with its own - // `redirect_sbt_owned_file_unreadable`, so leave it to that refusal. - if !matches!(view, ProjectView::Memory(_)) { - out.undecodable_reads - .retain(|rel| rel != crate::formats::sbt::owned_file::HOSTED_FILE); - } // An sbt build's resolution evidence rides a synthetic key (see // `patch::redirect::sbt::SBT_RESOLUTION_KEY`). if candidates.iter().any(|c| c.dep.ecosystem == "maven") @@ -2922,6 +2930,31 @@ mod tests { } } + /// #721 review: an unreadable `socket-patch.sbt` is refused by the + /// run-wide check itself, which also runs before any vendored->hosted + /// takeover revert, and keeps the sbt planner's own refusal code. + #[tokio::test] + async fn an_unreadable_sbt_owned_file_refuses_early_with_the_sbt_code() { + let latin1: &[u8] = b"// Auteur: Andr\xe9\n"; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("socket-patch.sbt"), latin1).unwrap(); + let candidates = vec![gradle_candidate()]; + let read = read_candidate_files( + &ProjectView::Disk(tmp.path()), + &BTreeSet::new(), + &candidates, + ) + .await; + assert_eq!(read.undecodable_reads, vec!["socket-patch.sbt"]); + let refusal = undecodable_guard(&read.undecodable_reads, &candidates).expect("refused"); + assert_eq!(refusal.code, SBT_OWNED_FILE_UNREADABLE); + assert!( + refusal.message.contains("socket-patch.sbt"), + "{}", + refusal.message + ); + } + /// A refused Gradle build is never confirmed by a snippet pasted into a /// build script, though it names the suffixed version and the index url. #[tokio::test] From 072b46ab7bd605bf496480798e25d0542d7f3e5d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 17:22:00 +0000 Subject: [PATCH 8/9] Refuse a non-UTF-8 berry package.json Beside a yarn berry lock the root package.json is a rewrite target (its resolutions) and is read strictly, so a non-UTF-8 one was recorded as undecodable. The registry marks package.json vendored-only, so the run-wide #721 check ignored it and the run took the live manifest for absent instead of refusing (Bugbot). The check now counts the root package.json as npm. Advisory reads never record it, so beside an npm lock it still never refuses. Refs #721 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YED4tY7Yytk79MTPzLnSfA --- crates/socket-patch-core/src/hosted/engine.rs | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 6b0956a88..3b2d3af00 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -1988,7 +1988,11 @@ pub fn undecodable_guard(undecodable: &[String], candidates: &[Candidate]) -> Op undecodable .iter() .find(|rel| { - file_ecosystem(rel).is_some_and(|eco| candidates.iter().any(|c| c.dep.ecosystem == eco)) + // The root manifest is read strictly only as a yarn berry + // rewrite target (its `resolutions`); advisory reads never + // record it, so here it is always an npm rewrite target. + let eco = file_ecosystem(rel).or((rel.as_str() == "package.json").then_some("npm")); + eco.is_some_and(|eco| candidates.iter().any(|c| c.dep.ecosystem == eco)) }) .map(|rel| undecodable_refusal(rel)) } @@ -2280,6 +2284,56 @@ mod tests { } } + /// #721 review: beside a yarn berry lock the root `package.json` is a + /// rewrite target (its `resolutions`), so a non-UTF-8 one refuses the + /// run instead of being taken for absent. Beside an npm lock it is + /// advisory only and never refuses. + #[tokio::test] + async fn a_non_utf8_berry_manifest_refuses_the_npm_run() { + use crate::patch::redirect::Integrity; + let candidates = vec![Candidate { + purl: "pkg:npm/left-pad@1.3.0".into(), + dep: DepOverride { + ecosystem: "npm".into(), + name: "left-pad".into(), + namespace: None, + version: "1.3.0".into(), + token: "tok".into(), + patch_uuid: "uuid".into(), + artifact_url: + "https://patch.socket.dev/patch/npm/left-pad/1.3.0/tok/uuid/left-pad-1.3.0.tgz" + .into(), + registry_override: None, + integrity: Integrity::default(), + }, + }]; + let berry = "__metadata:\n version: 8\n cacheKey: 10c0\n\n\"left-pad@npm:^1.3.0\":\n \ + version: 1.3.0\n resolution: \"left-pad@npm:1.3.0\"\n"; + let latin1: &[u8] = b"{\"name\": \"Andr\xe9\"}\n"; + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("yarn.lock"), berry).unwrap(); + std::fs::write(tmp.path().join("package.json"), latin1).unwrap(); + let view = ProjectView::Disk(tmp.path()); + let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; + assert_eq!(read.undecodable_reads, vec!["package.json"]); + let refusal = undecodable_guard(&read.undecodable_reads, &candidates).expect("refused"); + assert_eq!(refusal.code, UNREADABLE_REFUSAL); + + // Beside an npm lock the manifest is advisory: never refused. + std::fs::remove_file(tmp.path().join("yarn.lock")).unwrap(); + std::fs::write( + tmp.path().join("package-lock.json"), + "{\"lockfileVersion\": 3, \"packages\": {}}\n", + ) + .unwrap(); + let read = read_candidate_files(&view, &BTreeSet::new(), &candidates).await; + assert!( + read.undecodable_reads.is_empty(), + "{:?}", + read.undecodable_reads + ); + } + /// A hosted URL left in a berry project's `package.json` `resolutions` /// while `yarn.lock` still resolves the registry entry confirms nothing: /// only the lock pin installs (#404). From 365475b65c11f929d335547eba0da372e6ea555c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 17:39:04 +0000 Subject: [PATCH 9/9] Refuse a Gradle build whose root scripts are all non-UTF-8 92a44fd exempted every Gradle-owned file from the run-wide #721 check when no readable Gradle build was present. A Gradle-only project whose settings.gradle and build.gradle are all non-UTF-8 then looked like "no Gradle build". Its scripts were dropped, the planner never ran, and the run exited 0 unpatched: the silent skip #721 closes (Bugbot). With no readable build, root build and settings scripts now stay in the check and refuse the run, since they may be the build itself. Stray Gradle files such as locks are still exempt. Refs #721 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01YED4tY7Yytk79MTPzLnSfA --- crates/socket-patch-cli/CLI_CONTRACT.md | 2 +- crates/socket-patch-core/src/hosted/engine.rs | 57 ++++++++++++++----- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 952df0802..e508abde6 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -167,7 +167,7 @@ The rewriter reads a fixed set of candidate files from the project root: the npm **Hosted sbt (v5.0, additive)**: an sbt build root (`project/build.properties` naming an `sbt.version`, 0.13.18 or later) is wired through ONE generated root file, `socket-patch.sbt` — no user file is edited. It pins every granted Maven patch build-wide (a `ThisBuild` `dependencyOverrides +=` of the Socket-only `-socket.` version plus a `file:` resolver over `.socket/sbt-hosted/maven2/`, moved ahead of the default repositories on sbt 0.13 / 1.x so an unreachable one never blocks it offline), downloads the pinned pom and jar there on the first sbt load (sha256-checked, gitignored by the file itself), and installs a load-time verifier that fails `update` when any project resolves another version or a pinned artifact whose bytes are not pinned. Edits: `redirect_sbt_pin` (added), `redirect_sbt_pin_updated` (an existing row replaced: same GA and base under a new uuid, or the same uuid with new served values; `original` names the previous uuid and version), `redirect_sbt_pin_rechecked` (an existing row re-verified after the build's dependencies changed: its dependency digest is recorded anew, `original`/`new` are `{deps}`). The load-time verifier also fails `update` when a project declares a pinned GA at a version newer than the pin's base (the build-wide override would otherwise force it back down). A new pin is gated on sbt's own resolution records under `target/` (never the machine-wide cache): run-level stops wire nothing, warn once and exit 0 — `redirect_sbt_no_resolution_evidence` (none; run `sbt update` first; always the in-memory engine's answer), `redirect_sbt_resolution_incomplete` (a declared project left no evidence, or the project definitions cannot be read statically), `redirect_sbt_resolution_stale` (a build source is newer than some project's evidence: each project is dated by its own newest record, so a partial `sbt /update` does not vouch for the others). Per-patch refusals (never confirmed): `redirect_sbt_missing_override` (no `maven2` override or no suffixed version), `redirect_sbt_integrity_missing` (jar or pom sha256 missing), `redirect_sbt_unsafe_value` (a value unsafe in a Scala literal, or an index URL not naming the uuid), `redirect_sbt_version_conflict` (some project resolves another version, or a build source declares the GA newer than the patch's base), `redirect_sbt_override_conflict` (two patches for one GA in a run, or another base already pinned), `redirect_sbt_vendored_conflict` (the GA is pinned by `socket-patch-vendor.sbt`, or that file cannot be parsed — then every Maven patch), `redirect_sbt_owned_file_modified` / `redirect_sbt_owned_file_foreign` (`socket-patch.sbt` edited, or not socket-patch's — every Maven patch), `redirect_sbt_owned_file_unreadable` (a whole-run refusal: `socket-patch.sbt` is on disk but cannot be read as UTF-8 text, so writing it would replace it; nothing is written), `redirect_sbt_unsupported_version`, `redirect_sbt_build_root_unknown` (sbt files but no versioned build root — every Maven patch), `redirect_sbt_overrides_assignment` / `redirect_sbt_resolvers_assignment` (a build source reassigns `dependencyOverrides` / `resolvers` with `:=`, `~=` or `--=`), `redirect_sbt_dependency_lock_present` (a `build.sbt.lock`), `redirect_sbt_scala_runtime_unsupported` (`org.scala-lang`), `redirect_sbt_classifier_unsupported`; a GA no library configuration resolves is skipped silently (`redirect_sbt_meta_build_only` when only the meta-build resolves it). Advisories: `redirect_sbt_version_untested` (sbt 2.1+, still wired), `redirect_sbt_override_build_repos` (`sbt.override.build.repos=true`), `redirect_maven_pom_ignored_sbt_build` (a `pom.xml` beside the sbt build, which sbt never reads; the Maven rewriter still edits it for the Maven build). A re-run keeps an existing row and re-checks it. When the build's dependency digest changed since the pin, evidence resolved after the change (fresh, newer than the generated file) re-verifies it and the row's digest is refreshed (`redirect_sbt_pin_rechecked`); the uuid is NOT confirmed on `redirect_sbt_pin_declared_newer` (a build source now declares the GA newer than the pin's base; the row stays, sbt's load-time verifier fails the build, and the remedy is `socket-patch rollback` or declaring the base again), `redirect_sbt_pin_unverifiable` (the digest changed and the evidence predates the change, or the digest cannot be computed: run `sbt update`, then re-run socket-patch), `redirect_sbt_override_shadowed` (the evidence still resolves the base version) or `redirect_sbt_resolved_elsewhere` (the pinned version resolves from outside the pin repository from a file whose sha256 is not the pinned jar's; a copy holding the pinned bytes, such as the Ivy cache a second checkout reads, is fine — at most 64 pinned artifact files of up to 256 MiB are hashed, anything else counts as elsewhere), and also when a build source now reassigns `dependencyOverrides` / `resolvers` or a `build.sbt.lock` appeared (the same `redirect_sbt_overrides_assignment` / `redirect_sbt_resolvers_assignment` / `redirect_sbt_dependency_lock_present` codes; the row stays and sbt's load-time verifier fails the build). For a pure sbt root (no `pom.xml` / Gradle script beside it), maven confirmation is decided only by the sbt rewriter's report; on a mixed root a uuid the sbt rewriter refused is still confirmed by the Maven rewriter's own `pom.xml` pin (the generated sbt files never prove a pin by substring). **Mill and scala-cli** are guidance only: per Maven patch `redirect_mill_manual_snippet` / `redirect_scala_cli_manual_snippet` carry a paste-able snippet (repository + forced suffixed version), nothing is written or confirmed, and a pure Mill / scala-cli root gets no `redirect_maven_no_pom`; there, a Maven patch the server sent without a `maven2` registry override gets `redirect_maven_missing_override` instead of a snippet (with a `pom.xml` beside the Mill / scala-cli files the pom rewriter reports it). `rollback` / `remove` restore `socket-patch.sbt` offline (the rows removed, the file deleted with its last pin; the gitignored downloads are left). Manifest-less VEX reads every strictly parsed pin as a hosted reference but grants it the lockfile basis only when the local evidence shows every recorded version of the GA is the pinned one and every recorded artifact hashes to a pinned sha256 (else `sbt_resolution_unverified`). -**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. Two exceptions keep their own refusals. A Gradle build file the Gradle planner reaches gets that planner's per-build refusal (`redirect_gradle_build_file_unreadable`, exit 0) and the rest of the run goes ahead, and a stray Gradle file with no readable Gradle build is never rewritten, so it does not refuse the run. An unreadable `socket-patch.sbt` is refused with `redirect_sbt_owned_file_unreadable`. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". +**Non-UTF-8 candidate files (#721)**: the rewriters edit UTF-8 text only. A candidate file that exists but is not UTF-8 (for example a UTF-16 `requirements.txt`, which is what Windows PowerShell 5.1's `pip freeze >` writes and which pip installs from) is never read as absent. When a candidate of its ecosystem could rewrite it, the run is refused with `candidate_file_unreadable` (exit 1, `--dry-run` included), the message names the file, and nothing is written; the remedy is to re-save the file as UTF-8. Two exceptions keep their own refusals. A Gradle build file the Gradle planner reaches gets that planner's per-build refusal (`redirect_gradle_build_file_unreadable`, exit 0) and the rest of the run goes ahead, and with no readable Gradle build a stray Gradle file (a lock, a nested script) is never rewritten, so it does not refuse the run, while a non-UTF-8 root `settings.gradle(.kts)` or `build.gradle(.kts)` still refuses it (it may be the build itself). An unreadable `socket-patch.sbt` is refused with `redirect_sbt_owned_file_unreadable`. A vendored→hosted takeover checks this before it reverts anything, so a refused run leaves the vendored wiring, ledger entry and artifact byte-identical. Vendored mode likewise refuses a non-UTF-8 `requirements.txt` or `-r` include by name (`pypi_no_requirements`) instead of wiring around it. Lock-only discovery reads `requirements.txt` and its in-root `-r` includes the way pip decodes them (a UTF-16 or UTF-32 byte-order mark selects that encoding), so such a project's pins are still found instead of reporting "No packages found". **Gem stale-install guard (additive warning — the canonical narrative; other mentions point here)**: the gem hosted rewrite is pure Gemfile/lock text, so a gem ALREADY materialized under the project's bundle paths keeps its upstream bytes — the next `bundle install` prints `Using ` and never refetches, on **every** bundler major (live-verified 2026-08-19 on 1.17.3 / 2.7.2 / 4.0.18: bundler 4's CHECKSUMS verify at download time only, and nothing is downloaded; `bundle install --force`/`--redownload` re-install from the stale cached `.gem` instead of re-fetching — bundler 1 silently, bundler 4 with an exit-37 checksum refusal that still leaves the upstream bytes installed; the **verified** remedy is removing the installed dir + cache `.gem` + `specifications` entry, then `bundle install`). After the rewrite, a hosted run therefore probes the installed-gem discovery paths (the same ruby-crawler discovery `apply` uses, honoring `--global`/`--global-prefix` like scan's own discovery, plus — read-only — a `.bundle/config` bundle path refused as a write root because it resolves outside the project, which takes the project-local remedy) for each confirmed gem redirect and judges the materialization against the patch record's `afterHash` file map. Judgment rules: records are found **by uuid** among this run's fetched records (v5.0: hosted mode persists no records, so a purl whose `/patches/view` fetch failed this run is not judged; the warning re-fires on every re-scan whose fetch succeeds, until the stale materialization is gone); a materialization with every file at `afterHash` is already patched and never warns (an agent→hosted migration stays quiet by construction), and when several confirmed variant purls resolve to one installed dir, ANY of them judging it patched keeps it quiet; staleness needs **positive evidence** — at least one record file whose bytes were actually read and hash to neither state's expectation — so missing or unreadable files never produce a warning. Warnings emit `redirect_gem_stale_install` (JSON `redirect.warnings[]` + a code-tagged stderr line) in three flavors: a PROJECT-LOCAL dir gets the verified delete-list remedy (installed dir, cache `.gem`, `specifications` entry — plus the project's committed `/.gem` when present and not proven to be the patched artifact, since bundler installs from its cache dir in preference to fetching); a SHARED gem-env home gets a caveat that the home is shared machine-wide and prefers migrating the project to a local bundle path over deleting shared files; and a committed cache-dir archive whose sha256 differs from the patched artifact's warns standalone even with no installed dir at all (a fresh checkout with a committed stale cache re-materializes the upstream bytes forever). A stale-flagged purl is additionally **excluded from the same run's `--vex` `assume_applied` set** — the envelope must never attest a CVE its own warning says is live; the purl falls back to normal installed-tree verification (a patched install still attests, a stale one is omitted). The cache dir is bundler's `cache_path` setting (`Bundler.app_cache`), resolved in `Bundler::Settings` priority: `BUNDLE_CACHE_PATH:` in the bundler app config (`$BUNDLE_APP_CONFIG/config`, else `.bundle/config`) first, then the `BUNDLE_CACHE_PATH` environment variable, then `BUNDLE_CACHE_PATH:` in the global config (`bundle config set --global`: `$BUNDLE_CONFIG`, else `$BUNDLE_USER_CONFIG`, else `$BUNDLE_USER_HOME/config`, else `~/.bundle/config`), else `vendor/cache`; a relative value is read against the project root. The same global tier, below the app config and the environment, applies to `BUNDLE_GEMFILE:` and, for agent-mode install-root discovery, to `BUNDLE_PATH:`. A present local or environment `path`, `path.system`, or `disable_shared_gems` setting (including an empty string or false flag) shadows the global path tier, matching the tested Bundler 2.6/4 behavior; Bundler 1.x's legacy global-path shortcut is not modeled. An empty higher-tier `gemfile` setting also shadows the global value but leaves an existing nonempty `BUNDLE_GEMFILE` environment value in effect, or uses default manifest discovery when there is none. With `BUNDLE_IGNORE_CONFIG` set (any value) bundler reads no config file, so the app and global configs are skipped here too and only the environment and the default count — the same holds for the `BUNDLE_GEMFILE:` app-config setting. The probe is read-only (nothing is deleted) and skipped on `--dry-run` — deliberately explicit, since nothing was rewritten. Exit code and `status` are unchanged (warning-only, the hosted-refusal posture); a same-run `--vex` may still fail on "nothing to attest" per the embedded-VEX contract. diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 3b2d3af00..dcb693a59 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -641,10 +641,14 @@ pub async fn read_candidate_files( .retain(|rel| !gradle_unreadable.contains(rel)); } else { // No readable Gradle build: the Gradle planner never runs, so a - // stray Gradle file it would own (a lone settings script, a lock) - // is never rewritten and must not refuse the rest of the run. - out.undecodable_reads - .retain(|rel| !is_gradle_owned_file(rel)); + // stray Gradle file it would own (a lock, a nested script) is never + // rewritten and must not refuse the rest of the run. A root build + // or settings script still refuses: it may be the build itself, + // unreadable, which the planner would otherwise skip silently. + out.undecodable_reads.retain(|rel| { + !is_gradle_owned_file(rel) + || crate::patch::redirect::gradle::GRADLE_ROOT_FILES.contains(&rel.as_str()) + }); } // An sbt build's resolution evidence rides a synthetic key (see // `patch::redirect::sbt::SBT_RESOLUTION_KEY`). @@ -1990,8 +1994,14 @@ pub fn undecodable_guard(undecodable: &[String], candidates: &[Candidate]) -> Op .find(|rel| { // The root manifest is read strictly only as a yarn berry // rewrite target (its `resolutions`); advisory reads never - // record it, so here it is always an npm rewrite target. - let eco = file_ecosystem(rel).or((rel.as_str() == "package.json").then_some("npm")); + // record it, so here it is always an npm rewrite target. A root + // Gradle script left here (no readable build beside it) may be + // the build itself, which only maven candidates could patch. + let eco = file_ecosystem(rel) + .or((rel.as_str() == "package.json").then_some("npm")) + .or(crate::patch::redirect::gradle::GRADLE_ROOT_FILES + .contains(&rel.as_str()) + .then_some("maven")); eco.is_some_and(|eco| candidates.iter().any(|c| c.dep.ecosystem == eco)) }) .map(|rel| undecodable_refusal(rel)) @@ -2958,22 +2968,26 @@ mod tests { } /// #721 review: with no readable Gradle build the Gradle planner never - /// runs, so a stray non-UTF-8 Gradle file (a lone settings script, a - /// lock) does not refuse the rest of a Maven run. + /// runs, so a stray non-UTF-8 Gradle lock does not refuse the rest of a + /// Maven run. A non-UTF-8 root build or settings script does: it may be + /// the whole build, unreadable, which would otherwise be skipped + /// silently (a Gradle-only project exiting 0 unpatched). #[tokio::test] - async fn a_stray_non_utf8_gradle_file_does_not_refuse_a_maven_run() { + async fn non_utf8_gradle_files_without_a_readable_build() { const POM: &str = "com.socketfixturevictim1.10.0\n"; let latin1: &[u8] = b"rootProject.name = 'Andr\xe9'\n"; + let candidates = vec![gradle_candidate()]; + + // A stray lock beside a pom.xml: not refused. let tmp = tempfile::tempdir().unwrap(); std::fs::write(tmp.path().join("pom.xml"), POM).unwrap(); - std::fs::write(tmp.path().join("settings.gradle"), latin1).unwrap(); std::fs::write(tmp.path().join("gradle.lockfile"), latin1).unwrap(); let mut memory = MemoryProject::new(); memory.insert_text("pom.xml", POM); - for rel in ["settings.gradle", "gradle.lockfile"] { - memory.insert(rel, MemoryEntry::Binary(latin1.to_vec().into())); - } - let candidates = vec![gradle_candidate()]; + memory.insert( + "gradle.lockfile", + MemoryEntry::Binary(latin1.to_vec().into()), + ); for view in [ProjectView::Disk(tmp.path()), ProjectView::Memory(&memory)] { let (read, done) = gradle_rewrite_in(&view).await; assert!( @@ -2983,6 +2997,21 @@ mod tests { ); assert!(guard(&view, &done, &candidates).is_none()); } + + // A Gradle-only project whose root scripts are all non-UTF-8: + // refused, never skipped. + for root in ["settings.gradle", "build.gradle", "build.gradle.kts"] { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join(root), latin1).unwrap(); + let mut memory = MemoryProject::new(); + memory.insert(root, MemoryEntry::Binary(latin1.to_vec().into())); + for view in [ProjectView::Disk(tmp.path()), ProjectView::Memory(&memory)] { + let (read, done) = gradle_rewrite_in(&view).await; + assert_eq!(read.undecodable_reads, vec![root.to_string()]); + let refusal = guard(&view, &done, &candidates).expect("refused"); + assert_eq!(refusal.code, UNREADABLE_REFUSAL); + } + } } /// #721 review: an unreadable `socket-patch.sbt` is refused by the