Fix UTF-16 requirements.txt silently skipped (#721)

crates/socket-patch-cli/src/commands/scan/hosted.rs

@@ -1930,6 +1930,26 @@ async fn vendored_takeover(
                     .filter(|_| entry.is_some_and(vlt_entry))
             })
     };
+    // NON-UTF-8 PRE-CHECK (#721) — the GUARD's undecodable-file rule
+    // (`engine::undecodable_guard`), checked BEFORE any revert dispatches
+    // (and under --dry-run too): a takeover that reverted first and was
+    // then refused by the guard would leave the reverted purls unpatched
+    // in both modes.
+    if takeover.iter().any(|(_, entry)| entry.is_some()) {
+        let view = socket_patch_core::vendor::lock_inventory::ProjectView::Disk(&common.cwd);
+        let read = socket_patch_core::hosted::engine::read_candidate_files(
+            &view,
+            &std::collections::BTreeSet::new(),
+            candidates,
+        )
+        .await;
+        if let Some(refusal) = socket_patch_core::hosted::engine::undecodable_guard(
+            &read.undecodable_reads,
+            candidates,
+        ) {
+            return Err(refusal);
+        }
+    }
     // SYMLINK PRE-CHECK for the takeover reverts — the same rule as the
     // SYMLINK GUARD below, applied to each ledger entry's recorded wiring
     // (the revert backends also stage and rename over the file). Checked
@@ -2660,7 +2680,7 @@ fn created_settings_over_existing(
 }
 
 /// [`created_settings_over_existing`]'s code for `socket-patch.sbt`.
-const SBT_OWNED_FILE_UNREADABLE: &str = "redirect_sbt_owned_file_unreadable";
+use socket_patch_core::hosted::engine::SBT_OWNED_FILE_UNREADABLE;
 
 #[cfg(test)]
 mod tests {

crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs

@@ -315,6 +315,61 @@ async fn pypi_requirements_hosted_rewrites_pep440_equivalent_pin() {
     }
 }
 
+/// #721: Windows PowerShell 5.1 writes `pip freeze > requirements.txt` as
+/// UTF-16 with a BOM, and pip installs from it. The hosted grant must not
+/// treat that file as absent and exit 0 with the project unpatched: it is
+/// refused by name (`candidate_file_unreadable`, exit 1), nothing written.
+#[tokio::test]
+#[serial]
+async fn pypi_requirements_hosted_refuses_a_utf16_file() {
+    const UUID: &str = "a1a1a1a1-a1a1-4a1a-8a1a-a1a1a1a1a1a3";
+    const PURL: &str = "pkg:pypi/requests@2.31.0";
+    const SHA256: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+    let url = format!(
+        "http://patch.test/patch/pypi/requests/2.31.0/{TOKEN}/{UUID}/requests-2.31.0-py3-none-any.whl"
+    );
+
+    let text = "flask==2.0.1\r\nrequests==2.31.0\r\n";
+    let le: Vec<u8> = [0xFF, 0xFE]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_le_bytes))
+        .collect();
+    let be: Vec<u8> = [0xFE, 0xFF]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_be_bytes))
+        .collect();
+    for (what, bytes) in [("utf-16le", le), ("utf-16be", be)] {
+        let server = MockServer::start().await;
+        mock_view(&server, UUID, PURL).await;
+        mock_reference(
+            &server,
+            UUID,
+            PURL,
+            &url,
+            serde_json::json!({ "sha256": SHA256 }),
+            serde_json::Value::Null,
+        )
+        .await;
+
+        let tmp = tempfile::tempdir().unwrap();
+        std::fs::write(tmp.path().join("requirements.txt"), &bytes).unwrap();
+
+        let code =
+            socket_patch_cli::commands::get::run(get_hosted_args(UUID, tmp.path(), server.uri()))
+                .await;
+        assert_eq!(
+            code, 1,
+            "{what}: a requirements.txt hosted mode cannot read must refuse, not exit 0 unpatched"
+        );
+        assert_eq!(
+            std::fs::read(tmp.path().join("requirements.txt")).unwrap(),
+            bytes,
+            "{what}: the refused file must stay byte-identical"
+        );
+        assert_no_manifest_no_blobs(tmp.path());
+    }
+}
+
 // ---------------------------------------------------------------------------
 // maven — pom.xml fail-closed suffixed-version pin (rewrite_maven_pom)
 // ---------------------------------------------------------------------------

crates/socket-patch-cli/tests/mode_migration_pypi.rs

@@ -717,6 +717,64 @@ async fn uv_takeover_without_wheel_metadata_fails_loudly() {
 /// the revert (the artifact and ledger entry are kept). The takeover must
 /// then refuse — keeping the ledger — rather than drop the entry and leave
 /// the project half vendored with no record of it.
+/// #721: a non-UTF-8 candidate file (here a UTF-16 `pip freeze` export
+/// beside a vendored Poetry project) refuses the hosted run BEFORE the
+/// takeover reverts anything, wet and `--dry-run` alike: refusing only at
+/// the rewrite would leave the reverted poetry.lock unpatched in both modes.
+#[tokio::test]
+async fn undecodable_candidate_refuses_before_the_takeover_reverts() {
+    let (_tmp, root) = project();
+    std::fs::write(
+        root.join("pyproject.toml"),
+        "[tool.poetry]\nname = \"demo\"\nversion = \"0.1.0\"\ndescription = \"\"\nauthors = [\"x <x@x>\"]\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = \">=3.9\"\nsix = \"1.16.0\"\n",
+    )
+    .unwrap();
+    std::fs::write(
+        root.join("poetry.lock"),
+        POETRY_LOCK
+            .replace("WHEEL_SHA", WHEEL_SHA)
+            .replace("SDIST_SHA", SDIST_SHA),
+    )
+    .unwrap();
+    vendor_project(&root, &["poetry.lock", "pyproject.toml"]);
+    let mut utf16 = vec![0xFF, 0xFE];
+    for unit in "six==1.16.0\r\n".encode_utf16() {
+        utf16.extend(unit.to_le_bytes());
+    }
+    std::fs::write(root.join("requirements.txt"), &utf16).unwrap();
+    let lock = std::fs::read_to_string(root.join("poetry.lock")).unwrap();
+    let state = root.join(".socket/vendor/state.json");
+
+    let server = MockServer::start().await;
+    mount_hosted_api(&server, true).await;
+    let uri = server.uri();
+    for dry_run in [true, false] {
+        let mut args = hosted_scan_args(&uri);
+        if dry_run {
+            args.push("--dry-run");
+        }
+        let (code, env) = run_cli(&root, &args, &[]);
+        assert_eq!(code, 1, "dry_run={dry_run}: {env:#}");
+        let text = env.to_string();
+        assert!(
+            text.contains("candidate_file_unreadable") && text.contains("requirements.txt"),
+            "dry_run={dry_run}: {env:#}"
+        );
+        assert!(
+            !text.contains("redirect_takeover_reverted_vendored"),
+            "dry_run={dry_run}: nothing is reverted: {env:#}"
+        );
+        assert_eq!(
+            std::fs::read_to_string(root.join("poetry.lock")).unwrap(),
+            lock,
+            "dry_run={dry_run}: the vendored lock is untouched"
+        );
+        assert!(std::fs::read_to_string(&state).unwrap().contains(UUID));
+        assert!(root.join(format!(".socket/vendor/pypi/{UUID}")).exists());
+        assert_eq!(std::fs::read(root.join("requirements.txt")).unwrap(), utf16);
+    }
+}
+
 #[tokio::test]
 async fn drifted_vendored_line_refuses_takeover() {
     let (_tmp, root) = project();

crates/socket-patch-cli/tests/scan_requirements_lock_only.rs

@@ -6,6 +6,8 @@
 //!
 //! * #523: whitespace around `==` and the legacy `name (==X)` form;
 //! * #412: pins reached through in-root `-r` includes;
+//! * #721: a UTF-16 file with a BOM (Windows PowerShell 5.1's
+//!   `pip freeze >` output), which pip decodes.
 //! * #994: include targets pip unquotes (`-r "dev reqs.txt"`,
 //!   `--requirement="dev.txt"`, `-r dev\ reqs.txt`) or expands
 //!   (`-r ${REQDIR}/dev.txt`).
@@ -103,6 +105,19 @@ async fn assert_lock_only_discovers_with_env(
     files: &[(&str, &str)],
     envs: &[(&str, &str)],
     expected: &[&str],
+) {
+    let files: Vec<(&str, &[u8])> = files.iter().map(|(r, c)| (*r, c.as_bytes())).collect();
+    assert_lock_only_discovers_bytes_with_env(&files, envs, expected).await;
+}
+
+async fn assert_lock_only_discovers_bytes(files: &[(&str, &[u8])], expected: &[&str]) {
+    assert_lock_only_discovers_bytes_with_env(files, &[], expected).await;
+}
+
+async fn assert_lock_only_discovers_bytes_with_env(
+    files: &[(&str, &[u8])],
+    envs: &[(&str, &str)],
+    expected: &[&str],
 ) {
     for mode in [&[][..], &["--vendor"][..]] {
         let mock = MockServer::start().await;
@@ -166,6 +181,32 @@ async fn lock_only_scan_discovers_included_pins() {
     .await;
 }
 
+/// #721: pip decodes a requirements file by its BOM, so a UTF-16 file
+/// (what Windows PowerShell 5.1's `pip freeze >` writes) is discovered,
+/// in either byte order, instead of reading as "No packages found".
+#[tokio::test]
+async fn lock_only_scan_discovers_utf16_pins() {
+    let text = "sp-fixture-idna==3.7\r\nsp-fixture-six==1.16.0\r\n";
+    let le: Vec<u8> = [0xFF, 0xFE]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_le_bytes))
+        .collect();
+    let be: Vec<u8> = [0xFE, 0xFF]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_be_bytes))
+        .collect();
+    for bytes in [le, be] {
+        assert_lock_only_discovers_bytes(
+            &[("requirements.txt", &bytes)],
+            &[
+                "pkg:pypi/sp-fixture-idna@3.7",
+                "pkg:pypi/sp-fixture-six@1.16.0",
+            ],
+        )
+        .await;
+    }
+}
+
 /// #994: pip `shlex`-splits an include line's options, so a quoted or
 /// backslash-escaped target names the file without its quotes, and a
 /// target with a space is one path, not two words.