diff --git a/Cargo.lock b/Cargo.lock index 67a2143..21d88ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ checksum = "2af0513e0a025ce1ebbbdbf00e81ca84c29f26b33a6e1e0dd877945ceef75042" dependencies = [ "bit-set", "regex", - "thiserror", + "thiserror 2.0.20", "tree-sitter", ] @@ -215,7 +215,16 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.1", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", ] [[package]] @@ -347,7 +356,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -444,6 +453,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -493,6 +511,16 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.20.11" @@ -584,6 +612,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "dirs" version = "6.0.0" @@ -793,6 +832,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -894,6 +943,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hf-hub" version = "0.5.0" @@ -910,11 +965,20 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror", + "thiserror 2.0.20", "ureq", "windows-sys 0.61.2", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -1644,7 +1708,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] @@ -1751,7 +1815,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -1943,6 +2007,24 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "secrets_scanner" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d3e9b1732051e9d5b17c071414ffe702231a6af3df74492ea0bb06119cbc5" +dependencies = [ + "aho-corasick", + "hex", + "hmac", + "log", + "memchr", + "regex", + "serde", + "sha2", + "thiserror 1.0.69", + "toml 0.8.23", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2016,6 +2098,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -2050,6 +2141,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2222,13 +2324,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -2309,7 +2431,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror", + "thiserror 2.0.20", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -2363,6 +2485,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" version = "1.1.4+spec-1.1.0" @@ -2371,11 +2505,20 @@ checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] @@ -2387,6 +2530,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.15+spec-1.1.0" @@ -2394,9 +2551,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.4", ] [[package]] @@ -2405,9 +2562,15 @@ version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -2723,6 +2886,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3140,6 +3309,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.4" @@ -3229,6 +3407,7 @@ dependencies = [ name = "zerograph" version = "0.7.0" dependencies = [ + "aho-corasick", "anyhow", "ast-grep-language", "blake3", @@ -3242,6 +3421,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", + "secrets_scanner", "serde", "serde_json", "serde_yaml", @@ -3249,7 +3429,7 @@ dependencies = [ "streaming-iterator", "tempfile", "tokenizers", - "toml", + "toml 1.1.4+spec-1.1.0", "tree-sitter", "tree-sitter-bash", "tree-sitter-bicep", diff --git a/Cargo.toml b/Cargo.toml index 8e61503..e799f80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,6 +82,11 @@ ast-grep-language = { version = "=0.45.3", default-features = false, features = "tree-sitter-ruby", "tree-sitter-elixir", "tree-sitter-scala", ] } postcard = { version = "=1.1.3", features = ["use-std"] } +# Credentials inside files that are read (src/secrets.rs). Without `native`: its CLI, walker, logger and +# thread pool are not needed to scan a string, and the walk here is `ignore`'s. +secrets_scanner = { version = "=0.2.3", default-features = false } +# Already in the tree through `regex`; named here to find which rules a text's keywords call for. +aho-corasick = "=1.1.5" [dev-dependencies] tempfile = "=3.27.0" @@ -102,3 +107,14 @@ windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_Stor [profile.release] lto = "thin" + +# The secret scanner compiles hundreds of regexes, which an unoptimised regex-automata does seven +# times slower: every test that builds a store would pay that, not only the scanner's own. +[profile.dev.package.regex-automata] +opt-level = 3 +[profile.dev.package.regex-syntax] +opt-level = 3 +[profile.dev.package.aho-corasick] +opt-level = 3 +[profile.dev.package.secrets_scanner] +opt-level = 3 diff --git a/docs/configuration.md b/docs/configuration.md index 6e888d7..5777f41 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,6 +34,16 @@ keys and keystores, `.npmrc`, `.pypirc`, `.netrc`, `.git-credentials`, cloud cre `*.tfvars`, `*.tfstate` and `secrets.{yaml,json,toml,…}` — their content would otherwise reach the store, a `--rerank` prompt sent to a model, and the answers. +A credential written inside a file that is read — a token in a source file, a key exported in a +shell script, a private key pasted into a README — is replaced with `[REDACTED_SECRET]` before the +file is extracted, by the bundled rules of +[secrets_scanner](https://crates.io/crates/secrets_scanner) (gitleaks-compatible). The rest of the +file is still read and keeps its line numbers; `update` logs how many were redacted in which file. +An inline `gitleaks:allow` does not exempt one; the rules' own path allowlist does — images, fonts, +lockfiles, `vendor/` and `node_modules/` trees are not scanned. A run compiles only the rules whose keywords the +files it re-reads contain — a few dozen for a one-file update, tens of milliseconds, where all ~1,000 +would cost 1.8 s. + The store writes its own `.gitignore` holding `*`, so `.zerograph/` stays out of a commit in a repository that never listed it. diff --git a/src/lib.rs b/src/lib.rs index 690961d..dbac57f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ mod prime; mod priority; mod query; mod refresh; +mod secrets; mod serve; mod status; mod store; @@ -276,6 +277,9 @@ pub(crate) struct UpdateReport { pub graph_at: Option, } +/// How much file text is held before it is scanned for secrets and extracted. +const REDACT_BATCH_BYTES: usize = 64 << 20; + /// Re-extracts what the diff names, drops what is gone, and writes the store back. Nothing else /// is read: every citation a file holds was extracted when the file was read, whatever prefix it /// names, and which of them a reader may follow is decided on the graph by `settle` — so a family @@ -345,6 +349,27 @@ pub(crate) fn apply_diff(repo: &std::path::Path, store: &store::Store, graph: &m work.sort_by(|a, b| a.rel.cmp(&b.rel)); let reread = co_declared.iter().map(|rel| by_rel[rel]); let mut unread = false; + // The scanner compiles only the rules whose keywords a batch of texts holds, so a one-file + // update pays for a few dozen regexes, not ~1,000. Texts are held a batch at a time: every + // text of the tree at once would be the largest allocation of a build. An empty batch builds + // no scanner, which keeps the no-op update a commit hook fires free of the rules. + let mut batch: Vec<(&walk::Entry, String)> = Vec::new(); + let mut batch_bytes = 0; + let mut extract_batch = |batch: &mut Vec<(&walk::Entry, String)>| { + if batch.is_empty() { return; } + let redactor = secrets::Redactor::for_texts(batch.iter().map(|(_, text)| text.as_str())); + for (e, text) in batch.drain(..) { + let (text, redacted) = redactor.redact(&e.rel, &text); + if redacted > 0 { eprintln!("{}", log_line(&format!("{}: {redacted} secret(s) redacted", e.rel))); } + let extractor = match e.kind { + walk::FileKind::Doc => &ex.doc, + walk::FileKind::Code => &ex.code, + walk::FileKind::Registry => &ex.registry, + walk::FileKind::Text => &ex.text, + }; + graph.apply(extractor.extract(&e.rel, &text)); + } + }; for e in work.into_iter().chain(reread) { let text = match std::fs::read(repo.join(&e.rel)) { // NUL is legal inside a TypeScript string literal; only invalid UTF-8 marks a binary. @@ -356,14 +381,14 @@ pub(crate) fn apply_diff(repo: &std::path::Path, store: &store::Store, graph: &m // that would not open is one this pass has no reading of at all. Err(err) => { eprintln!("{}", log_line(&format!("read {}: {err}", e.rel))); unread = true; continue; } }; - let extractor = match e.kind { - walk::FileKind::Doc => &ex.doc, - walk::FileKind::Code => &ex.code, - walk::FileKind::Registry => &ex.registry, - walk::FileKind::Text => &ex.text, - }; - graph.apply(extractor.extract(&e.rel, &text)); + batch_bytes += text.len(); + batch.push((e, text)); + if batch_bytes >= REDACT_BATCH_BYTES { + extract_batch(&mut batch); + batch_bytes = 0; + } } + extract_batch(&mut batch); // Whether anything was re-extracted or dropped. Nothing else can move a family, so on a // no-op update — the one a commit hook fires — the settle below would read every node and // rebuild the whole edge set to arrive at what is already there. @@ -1435,6 +1460,25 @@ mod tests { run_update(repo, cfg, true).unwrap(); } + #[test] + fn a_secret_written_in_a_document_reaches_no_file_of_the_store() { + let token = format!("ghp_{}", "R7xK2mQ9vT4wL8nB3cF6hJ1pS5dG0aZyEuXo"); + let dir = doc_repo(&format!("# A\n\n**FR-PAY-22 · MUST · cancellation window**\n\nCall with `{token}`.\n")); + let (repo, cfg) = (dir.path(), config::Config::default()); + built(repo, &cfg); + let (graph, _) = store::Store::new(repo).load().unwrap(); + assert!(graph.nodes.contains_key("FR-PAY-22"), "the rest of the document is still read"); + let leaked: Vec<_> = walkdir_files(&repo.join(".zerograph")).into_iter() + .filter(|p| std::fs::read(p).unwrap().windows(token.len()).any(|w| w == token.as_bytes())) + .collect(); + assert!(leaked.is_empty(), "{leaked:?}"); + } + + fn walkdir_files(dir: &std::path::Path) -> Vec { + ignore::WalkBuilder::new(dir).standard_filters(false).build().flatten() + .filter(|e| e.file_type().is_some_and(|t| t.is_file())).map(|e| e.into_path()).collect() + } + /// The report exists to say the one thing a configuration-shaped answer would hide: a store /// keeps answering with the model that wrote it, whatever the file says today. So where the /// store's name comes from, and what is printed when the two differ, are the report. diff --git a/src/secrets.rs b/src/secrets.rs new file mode 100644 index 0000000..f752b6e --- /dev/null +++ b/src/secrets.rs @@ -0,0 +1,195 @@ +//! Credentials written inside a file that is read. `walk` refuses the files that exist to hold +//! them (`.env`, `*.pem`); this catches the key pasted into a source file, a README or a shell +//! script, before its text reaches `graph.json`, the lexical and dense indexes, or a rerank prompt. + +use aho_corasick::AhoCorasick; +use secrets_scanner::{RedactionMode, ScanConfig, Scanner}; +use std::sync::OnceLock; + +pub const MARKER: &str = "[REDACTED_SECRET]"; + +/// The bundled rules, split so a scanner can be built from any subset of them. Compiling all of +/// them costs 1.8 s, and the scanner already skips a rule whose keywords a text does not hold — +/// so a scanner of only the rules a text's keywords name finds exactly what the full one does. +struct Catalog { + /// Everything but the rules: the global allowlist, the minimum version. + base: toml::Table, + rules: Vec, + keywords: AhoCorasick, + rule_of_keyword: Vec, + unkeyworded: Vec, +} + +fn catalog() -> &'static Catalog { + static C: OnceLock = OnceLock::new(); + C.get_or_init(|| { + // The bundled rules only: `Scanner::new` would also read `$SECRETS_SCANNER_RULES` and a + // cache in the user's data directory, so what is redacted would vary by machine. + let mut base: toml::Table = toml::from_str(secrets_scanner::rules::BUNDLED_RULES).expect("the bundled rules parse"); + let rules = match base.remove("rules") { Some(toml::Value::Array(r)) => r, _ => Vec::new() }; + let (mut words, mut rule_of_keyword, mut unkeyworded) = (Vec::new(), Vec::new(), Vec::new()); + for (i, rule) in rules.iter().enumerate() { + let listed: Vec<&str> = rule.get("keywords").and_then(|k| k.as_array()).into_iter().flatten().filter_map(|k| k.as_str()).collect(); + if listed.is_empty() { unkeyworded.push(i); } + for word in listed { words.push(word.to_lowercase()); rule_of_keyword.push(i); } + } + let keywords = AhoCorasick::builder().ascii_case_insensitive(true).build(&words).expect("the rule keywords build an automaton"); + Catalog { base, rules, keywords, rule_of_keyword, unkeyworded } + }) +} + +/// A scanner for one pass's texts, holding only the rules their keywords name. +pub struct Redactor(Option); + +impl Redactor { + pub fn for_texts<'a>(texts: impl IntoIterator) -> Redactor { + let c = catalog(); + let mut wanted = vec![false; c.rules.len()]; + for &i in &c.unkeyworded { wanted[i] = true; } + for text in texts { + for m in c.keywords.find_overlapping_iter(text) { wanted[c.rule_of_keyword[m.pattern().as_usize()]] = true; } + } + let rules: Vec = c.rules.iter().zip(&wanted).filter(|(_, w)| **w).map(|(r, _)| r.clone()).collect(); + if rules.is_empty() { + return Redactor(None); + } + let mut subset = c.base.clone(); + subset.insert("rules".into(), toml::Value::Array(rules)); + let toml = toml::to_string(&subset).expect("a subset of the bundled rules serializes"); + let scanner = Scanner::from_toml(&toml).expect("a subset of the bundled rules compiles"); + Redactor(Some(scanner.with_config(ScanConfig { + redaction_mode: RedactionMode::Full, + // A repository is content someone else wrote: a `gitleaks:allow` beside a key must not + // be what lets it into the graph. + honor_allow_markers: false, + capture_context: false, + ..ScanConfig::default() + }))) + } + + /// `text` with every detected secret replaced by [`MARKER`], and how many were found. A secret + /// spanning lines keeps its newlines after the marker, so every line below it keeps its number. + pub fn redact(&self, rel: &str, text: &str) -> (String, usize) { + let Some(scanner) = &self.0 else { return (text.to_string(), 0) }; + redact_found(text, &scanner.scan_content_detailed(rel, text).findings) + } +} + +fn redact_found(text: &str, found: &[secrets_scanner::Finding]) -> (String, usize) { + let mut spans: Vec<(usize, usize)> = found.iter() + .map(|f| (f.secret_start_offset, f.secret_end_offset)) + .filter(|(s, e)| s < e && *e <= text.len()) + .map(|(s, e)| (floor_boundary(text, s), ceil_boundary(text, e))) + .collect(); + if spans.is_empty() { + return (text.to_string(), 0); + } + spans.sort_unstable(); + // Two rules often match one key: overlapping or touching spans become one marker. + let mut merged: Vec<(usize, usize)> = Vec::with_capacity(spans.len()); + for (start, end) in spans { + match merged.last_mut() { + Some(last) if start <= last.1 => last.1 = last.1.max(end), + _ => merged.push((start, end)), + } + } + let mut out = String::with_capacity(text.len()); + let mut cursor = 0; + let count = merged.len(); + for (start, end) in merged { + out.push_str(&text[cursor..start]); + out.push_str(MARKER); + out.extend(text[start..end].chars().filter(|&c| c == '\n')); + cursor = end; + } + out.push_str(&text[cursor..]); + (out, count) +} + +fn floor_boundary(text: &str, mut i: usize) -> usize { + while !text.is_char_boundary(i) { i -= 1; } + i +} + +fn ceil_boundary(text: &str, mut i: usize) -> usize { + while !text.is_char_boundary(i) { i += 1; } + i +} + +#[cfg(test)] +mod tests { + use super::{Redactor, MARKER}; + + fn redact(rel: &str, text: &str) -> (String, usize) { + Redactor::for_texts([text]).redact(rel, text) + } + + // Assembled at run time so this file holds no string a scanner of this repository flags. + fn github_token() -> String { format!("ghp_{}", "R7xK2mQ9vT4wL8nB3cF6hJ1pS5dG0aZyEuXo") } + fn aws_key() -> String { format!("AKIA{}", "QX4TZ7WMB2KD6RLN") } + + #[test] + fn a_token_in_source_is_replaced_and_the_rest_of_the_line_stays() { + let text = format!("const client = new Octokit({{ auth: \"{}\" }});\nexport default client;\n", github_token()); + let (out, n) = redact("src/github.ts", &text); + assert_eq!(n, 1); + assert!(!out.contains(&github_token()), "{out}"); + assert!(out.contains(MARKER) && out.contains("new Octokit") && out.contains("export default client;"), "{out}"); + } + + #[test] + fn a_key_exported_in_a_shell_script_is_replaced() { + let text = format!("#!/bin/sh\nexport AWS_ACCESS_KEY_ID={}\naws s3 ls\n", aws_key()); + let (out, n) = redact("scripts/deploy.sh", &text); + assert!(n >= 1); + assert!(!out.contains(&aws_key()), "{out}"); + assert!(out.contains("aws s3 ls")); + } + + #[test] + fn a_private_key_block_is_replaced_and_the_lines_below_keep_their_numbers() { + let body = "MIIEowIBAAKCAQEA0Z3VS5JJcds3xfn/ygWyF8PbnGy0AHB7MhgHcTz6sE2I2yPB\naFDrBz9vFqU4yQmpQ6fFoXyZ0YkRh4AXKzk7b1HgGzK7v1mN3rGv5WqL8XKjR2Ys\n"; + let text = format!("# Ops\n\n-----BEGIN RSA {k}-----\n{body}-----END RSA {k}-----\n\nnext line\n", k = "PRIVATE KEY"); + let (out, n) = redact("docs/ops.md", &text); + assert!(n >= 1); + assert!(!out.contains("MIIEowIBAAKCAQEA"), "{out}"); + assert_eq!(out.lines().count(), text.lines().count(), "{out}"); + assert_eq!(out.lines().position(|l| l == "next line"), text.lines().position(|l| l == "next line")); + } + + #[test] + fn an_allow_marker_in_the_file_does_not_let_a_secret_through() { + let text = format!("token = \"{}\" # gitleaks:allow\n", github_token()); + let (out, _) = redact("config.py", &text); + assert!(!out.contains(&github_token()), "{out}"); + } + + #[test] + fn text_without_a_secret_comes_back_unchanged() { + let text = "fn main() {\n let password = read_password();\n println!(\"héllo\");\n}\n"; + assert_eq!(redact("src/main.rs", text), (text.to_string(), 0)); + } + + /// The subset is only an optimisation if it finds what the whole ruleset finds, on real text + /// that names many providers' keywords: this repository's own sources and documents. + #[test] + fn a_scanner_of_the_rules_a_text_names_finds_what_the_whole_ruleset_finds() { + let full = secrets_scanner::Scanner::from_toml(secrets_scanner::rules::BUNDLED_RULES).expect("the whole ruleset passes the strict gate"); + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let spans = |f: Vec| { let mut s: Vec<_> = f.into_iter().map(|f| (f.rule_id, f.secret_start_offset, f.secret_end_offset)).collect(); s.sort(); s }; + let mut paths: Vec<_> = ignore::Walk::new(root).flatten().map(|e| e.into_path()) + .filter(|p| p.extension().is_some_and(|x| x == "rs" || x == "md")).collect(); + paths.sort(); + let (mut compared, mut flagged) = (0, 0); + for path in &paths { + let Ok(text) = std::fs::read_to_string(path) else { continue }; + let rel = path.strip_prefix(root).unwrap().to_string_lossy(); + let expected = spans(full.scan_content_detailed(&rel, &text).findings); + let Redactor(Some(subset)) = Redactor::for_texts([text.as_str()]) else { panic!("no rule at all for {rel}") }; + assert_eq!(spans(subset.scan_content_detailed(&rel, &text).findings), expected, "{rel}"); + compared += 1; + flagged += usize::from(!expected.is_empty()); + } + assert!(compared > 100 && flagged > 0, "compared {compared} files, {flagged} flagged"); + } +} diff --git a/src/walk.rs b/src/walk.rs index 76b8e36..f4b834c 100644 --- a/src/walk.rs +++ b/src/walk.rs @@ -37,7 +37,9 @@ pub struct Entry { pub rel: String, pub kind: FileKind, pub hash: String, pub st /// 6: every language read by one query engine. Some nodes' spans and bodies differ from what 5's /// per-language readers wrote, such as a decorated field's line or the comment kept above a /// declaration, so a store stamped 5 is re-read once. -pub const GRAMMAR: u32 = 6; +/// 7: a credential written inside a file is replaced before the file is read, so a store stamped 6 +/// still holds the ones its unmoved files carry until it is re-read once. +pub const GRAMMAR: u32 = 7; /// Refused as text whatever `text_globs` says: machine output nobody asks a question of, any one of /// which would outweigh the hand-written files in the lexical index. Text entries only — `skip`