From 918855de9c0d31d7ed3862646fdeb52fe59c0bce Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 22:03:28 +1100 Subject: [PATCH 01/15] Give a panic and not a stack trace when a path can not be looked at --- src/engine.ts | 24 +++++++++++++++++++++--- test/fix/788.test.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 test/fix/788.test.js diff --git a/src/engine.ts b/src/engine.ts index 49c83ad9..39ca4fa2 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1135,7 +1135,7 @@ function getFilePaths(conf) { // --exclude-glob is always globs, also for the name of a file. They are used together, and the files they find are taken away. if (excludeGlob.length) { - const excluded = new Set(globs.sync(excludeGlob, globOptions).map(file => path.resolve(file))); + const excluded = new Set(glob(excludeGlob, globOptions).map(file => path.resolve(file))); files = files.filter(file => !excluded.has(path.resolve(file))); } @@ -1200,13 +1200,31 @@ function sortFiles(files, sort, conf) { return files.sort((a, b) => key(stats.get(a)) - key(stats.get(b)) || ascii(a, b)); } +// A path that can not be looked at (like a name that is too long) is not a file, so it is globbed +function isFile(file) { + try { + return Boolean(fs.statSync(file, {throwIfNoEntry: false})?.isFile()); + } catch (e) { + return false; + } +} + +// When the file system gives an error, the files can not be known, so the run can not go on +function glob(patterns, globOptions) { + try { + return globs.sync(patterns, globOptions); + } catch (e) { + return panic('The files can not be found:', e.message); + } +} + // A path that is a file is used as it is, so a name with glob characters, like `a[1].md` or `my (1).md`, gives only that file. // The other paths are globbed together, so a pattern that starts with `!` can take files away from the other patterns. function findFiles(paths, globOptions, sort) { const files = []; const patterns = []; paths.forEach(file => { - if (fs.statSync(file, {throwIfNoEntry: false})?.isFile()) { + if (isFile(file)) { files.push(file); } else { patterns.push(file); @@ -1216,7 +1234,7 @@ function findFiles(paths, globOptions, sort) { // A file given by its name and also found by a glob is only handled once const seen = new Set(); // A glob has no order of its own, so with --sort none its files come after the paths, sorted with natural - const found = patterns.length ? globs.sync(patterns, globOptions) : []; + const found = patterns.length ? glob(patterns, globOptions) : []; return files.concat('none' === sort ? found.sort(natural) : found).filter(file => { const fullPath = path.resolve(file); if (seen.has(fullPath)) { diff --git a/test/fix/788.test.js b/test/fix/788.test.js new file mode 100644 index 00000000..b572225c --- /dev/null +++ b/test/fix/788.test.js @@ -0,0 +1,31 @@ +// https://github.com/mathiasrw/rexreplace/issues/788 +// A path the file system could not look at, like a name that is too long, crashed the run with a stack trace. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#788 a path that can not be looked at', () => { + test('is not a file, so it is globbed', async () => { + fs.writeFileSync('a.md', 'foo'); + const long = `{a,${'x'.repeat(300)}}.md`; + expect(await rr('foo', 'bar', {includeGlob: [long]})).toEqual({ + out: 'a.md\n', + err: '', + exit: null, + }); + expect(fs.readFileSync('a.md', 'utf8')).toBe('bar'); + }); + + test.each([ + ['as an argument', {includeGlob: ['a.md/x']}], + ['from --glob-pipe', {pipedData: 'a.md/x', globPipe: true}], + ['in --exclude-glob', {includeGlob: ['a.md'], excludeGlob: ['a.md/x']}], + ])('gives a panic when the glob can not be used, %s', async (name, config) => { + fs.writeFileSync('a.md', 'foo'); + const result = await rr('foo', 'bar', config); + expect(result.err).toStartWith(' ❌ The files can not be found: ENOTDIR'); + expect(result.err).toEndWith('\n No further processing.\n\n'); + expect(result.exit).toBe(3); + expect(fs.readFileSync('a.md', 'utf8')).toBe('foo'); + }); +}); From e366dd68d2879781af6b87a81b1cbd9120edb757 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 22:05:23 +1100 Subject: [PATCH 02/15] Stop starting new files after a panic with --halt --- src/engine.ts | 11 +++++++++-- test/fix/789.test.js | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test/fix/789.test.js diff --git a/src/engine.ts b/src/engine.ts index 39ca4fa2..de4447cb 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -230,10 +230,17 @@ async function run(config) { ? files.length : Math.max(16, os.availableParallelism() * 2); let next = 0; + // After a panic no worker starts a new file. A file that is already in work is finished. + let stopped = false; const worker = async () => { - while (next < files.length) { + while (next < files.length && !stopped) { const index = next++; - await openFile(files[index], config, index + 1, files.length); + try { + await openFile(files[index], config, index + 1, files.length); + } catch (e) { + stopped = true; + throw e; + } } }; diff --git a/test/fix/789.test.js b/test/fix/789.test.js new file mode 100644 index 00000000..ca8c6159 --- /dev/null +++ b/test/fix/789.test.js @@ -0,0 +1,22 @@ +// https://github.com/mathiasrw/rexreplace/issues/789 +// With --halt and the async flow, the engine threw the panic, but the workers went on taking new files. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import os from 'os'; +import {rr} from '../helper.js'; + +const workers = Math.max(16, os.availableParallelism() * 2); + +describe('#789 a panic with --halt', () => { + test('stops the workers from starting new files', async () => { + const names = Array.from({length: workers * 10}, (_, i) => `${String(i).padStart(4, '0')}.txt`); + names.forEach(name => fs.writeFileSync(name, 'foo')); + fs.chmodSync(names[0], 0); + const result = await rr('foo', 'bar', {includeGlob: ['*.txt'], voidAsync: false, halt: true}); + expect(result.exit).toBe(3); + // The files that were already in work when the panic came are finished, but no new file is started + await Bun.sleep(300); + const changed = names.slice(1).filter(name => 'bar' === fs.readFileSync(name, 'utf8')); + expect(changed.length).toBeLessThan(workers); + }); +}); From ff4c0e51444ac7b4601122acbbdab4e76dc1de31 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 22:07:07 +1100 Subject: [PATCH 03/15] Read the piped paths of --glob-pipe as UTF-8 --- src/cli.ts | 3 ++- test/fix/790.test.js | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test/fix/790.test.js diff --git a/src/cli.ts b/src/cli.ts index 9d170219..bee07621 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -665,7 +665,8 @@ if (!pipeDataExpected || Boolean(process.stdin.isTTY)) { // Stays null when nothing comes through the pipe let pipeData = null; - process.stdin.setEncoding(config.encoding); + // With --glob-pipe the piped data is paths, and they are UTF-8 and not in the encoding of the files + process.stdin.setEncoding(config.globPipe ? 'utf8' : config.encoding); process.stdin.on('data', chunk => { pipeData = (pipeData ?? '') + chunk; diff --git a/test/fix/790.test.js b/test/fix/790.test.js new file mode 100644 index 00000000..c467024d --- /dev/null +++ b/test/fix/790.test.js @@ -0,0 +1,21 @@ +// https://github.com/mathiasrw/rexreplace/issues/790 +// With --glob-pipe the piped paths were decoded with --encoding, so with -e latin1 a name that is not ASCII found no file. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {$} from 'bun'; +import {cli, sh} from '../helper.js'; + +describe('#790 the paths piped in with --glob-pipe', () => { + test.each([ + ['line breaks', 'æ.file\nb.file\n', ''], + ['null bytes', 'æ.file\0b.file\0', '--null-in'], + ])('are read as UTF-8 with -e latin1, split by %s', async (name, paths, flag) => { + fs.writeFileSync('æ.file', 'foo'); + fs.writeFileSync('b.file', 'foo'); + const result = await sh( + $`bun ${cli} foo bar -g -e latin1 -A ${flag || []} < ${new Response(paths)}` + ); + expect(result).toEqual({out: 'æ.file\nb.file\n', err: '', exit: 0}); + expect(fs.readFileSync('æ.file', 'utf8')).toBe('bar'); + }); +}); From 9b6aec073f114e73c996d4ca227cda26d96c41e8 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:06:58 +1100 Subject: [PATCH 04/15] Remove control characters from the new names of --rename --- README.md | 2 +- src/cli.ts | 2 +- src/engine.ts | 11 ++++++++++- test/fix/826.test.js | 24 ++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 test/fix/826.test.js diff --git a/README.md b/README.md index efb9389c..ad9f9239 100644 --- a/README.md +++ b/README.md @@ -355,7 +355,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-z` | **`--expect-diff`** Check that at least one file is changed. The exit code is 1 if not. [boolean] | | `-Z` | **`--expect-zero-diff`** Check that no file is changed. The exit code is 1 if one is. [boolean] | | | **`--expect-diff-all`** Check that all files found are changed. The exit code is 1 if not. [boolean] | -| `-r` | **`--rename`** Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. [boolean] | +| `-r` | **`--rename`** Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. Control characters, like a line break, are removed from the new name. [boolean] | | | **`--rename-path`** Like --rename, but on the whole path of each file, so a / in the replacement moves the file. Folders that are not there are made. [boolean] | | | **`--rename-force`** With --rename or --rename-path, write over a file that is there already. [boolean] | | `-m` | **`--output-match`** Output each match on a new line. Will not replace any content but you still need to provide a dummy value (like `_`) as replacement parameter. The replacement is not used, so --js code is not run. If search pattern does not contain matching groups the full match will be outputted. If search pattern does contain matching groups only matching groups will be outputted (same line with no delimiter). [boolean] | diff --git a/src/cli.ts b/src/cli.ts index bee07621..6d49a484 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -300,7 +300,7 @@ const yargs = require('yargs') .boolean('r') .describe( 'r', - 'Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file.' + 'Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. Control characters, like a line break, are removed from the new name.' ) .alias('r', 'rename') diff --git a/src/engine.ts b/src/engine.ts index de4447cb..ba3419be 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -377,7 +377,7 @@ function renameFiles(files, config) { config.replacementJs && !config.outputMatch ? codeReplacement(file, config, text, fileNumbers(config, index + 1, files.length)) : config.replacement; - const result = text.replace(config.regex, replacement); + const result = withoutControlChars(text.replace(config.regex, replacement), file); countFile(config, text, result); if (result === text) { return {old: file, new: file}; @@ -418,6 +418,15 @@ function renameFiles(files, config) { }); } +// No name should have the chars below space in ASCII (like a line break, a tab or a null byte), or DEL +function withoutControlChars(name, file) { + const clean = name.replace(/[\x00-\x1f\x7f]/g, ''); + if (clean !== name) { + chat('Removed control characters from the new name of: ' + file); + } + return clean; +} + // With --simulate and --output the old name goes to stderr and the new one to stdout, so they come in pairs in a terminal function nameRename(item, config) { if (config.simulate || config.output) { diff --git a/test/fix/826.test.js b/test/fix/826.test.js new file mode 100644 index 00000000..36989218 --- /dev/null +++ b/test/fix/826.test.js @@ -0,0 +1,24 @@ +// https://github.com/mathiasrw/rexreplace/issues/826 +// --rename put control characters into the new name, like the line break at the end of a replacement file. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#826 control characters in a new name', () => { + test('are removed, like the line break at the end of a replacement file', async () => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('name.txt', 'b\n'); + const config = {includeGlob: ['a.md'], rename: true, replacementFile: true, verbose: true}; + const result = await rr('^a', 'name.txt', config); + expect(result.out).toBe('b.md\n'); + expect(result.err).toContain('Removed control characters from the new name of: a.md'); + expect(fs.readdirSync('.').sort()).toEqual(['b.md', 'name.txt']); + }); + + test('are removed with --rename-path too, also a tab, a null byte and DEL', async () => { + fs.writeFileSync('a.md', 'foo'); + const config = {includeGlob: ['a.md'], renamePath: true}; + expect((await rr('^a', 'b\t\0\x7f\x01c', config)).out).toBe('bc.md\n'); + expect(fs.readdirSync('.')).toEqual(['bc.md']); + }); +}); From a2c62df1dda41e22801876f61769ba7846f8ee3d Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:16:52 +1100 Subject: [PATCH 05/15] Give a message and not a crash when --js code throws something that is not an error --- src/engine.ts | 21 ++++++++++++++++++++- test/fix/822.test.js | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 test/fix/822.test.js diff --git a/src/engine.ts b/src/engine.ts index ba3419be..54f9ddd8 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -819,6 +819,25 @@ function numbersCode(numbers) { .join(''); } +// The text of what code threw. Code can throw anything, also a value that can not be made into text. +function thrownText(thrown) { + let text = ''; + try { + text = String(thrown); + } catch (e) {} + if (text && '[object Object]' !== text) { + return text; + } + if ('object' === typeof thrown && null !== thrown) { + let json = ''; + try { + json = JSON.stringify(thrown); + } catch (e) {} + return 'an object was thrown' + (json && '{}' !== json ? ': ' + json : ''); + } + return 'an empty ' + typeof thrown + ' was thrown'; +} + function dynamicReplacement( _file_rr, _config_rr, @@ -927,7 +946,7 @@ function dynamicReplacement( return dynamicContent(...args); } catch (e) { const where = _config_rr.dataIsPiped ? 'the piped data' : _file_rr; - return panic('The code of --js failed for ' + where + ':', String(e)); + return panic('The code of --js failed for ' + where + ':', thrownText(e)); } }; diff --git a/test/fix/822.test.js b/test/fix/822.test.js new file mode 100644 index 00000000..27f047d6 --- /dev/null +++ b/test/fix/822.test.js @@ -0,0 +1,22 @@ +// https://github.com/mathiasrw/rexreplace/issues/822 +// --js code that threw something that is not an error crashed the error handling, or gave an empty message. +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +const failed = text => + ` ❌ The code of --js failed for the piped data: ${text}\n No further processing.\n\n`; + +describe('#822 code that throws something that is not an error', () => { + test.each([ + ['throw Object.create(null)', 'an object was thrown'], + ['throw {code: 42}', 'an object was thrown: {"code":42}'], + ['throw ""', 'an empty string was thrown'], + ['throw 7', '7'], + ])('%s gives a message, and not a crash', async (code, text) => { + expect(await rr('o', code, {pipedData: 'foo', replacementJs: true})).toEqual({ + out: '', + err: failed(text), + exit: 3, + }); + }); +}); From 8be509baacc61bb5995a6cac2686b9fa062a94d5 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:11:38 +1100 Subject: [PATCH 06/15] Name a thrown error whose message can not be read, and cut a big thrown object short --- src/engine.ts | 12 +++++++++++- test/fix/822.test.js | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/engine.ts b/src/engine.ts index 54f9ddd8..d5725ccc 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -829,11 +829,21 @@ function thrownText(thrown) { return text; } if ('object' === typeof thrown && null !== thrown) { + // Like an Error whose message can not be read: its name is still known + let name = ''; + try { + name = thrown.constructor?.name; + } catch (e) {} + if (name && 'Object' !== name) { + return 'a ' + name + ' was thrown'; + } let json = ''; try { json = JSON.stringify(thrown); } catch (e) {} - return 'an object was thrown' + (json && '{}' !== json ? ': ' + json : ''); + // The object can hold a whole file, so only the start of it is told + const start = json.length > 300 ? json.slice(0, 300) + '…' : json; + return 'an object was thrown' + (json && '{}' !== json ? ': ' + start : ''); } return 'an empty ' + typeof thrown + ' was thrown'; } diff --git a/test/fix/822.test.js b/test/fix/822.test.js index 27f047d6..cfa1b874 100644 --- a/test/fix/822.test.js +++ b/test/fix/822.test.js @@ -12,6 +12,10 @@ describe('#822 code that throws something that is not an error', () => { ['throw {code: 42}', 'an object was thrown: {"code":42}'], ['throw ""', 'an empty string was thrown'], ['throw 7', '7'], + [ + 'const e = new TypeError("x"); Object.defineProperty(e, "message", {get() { throw 1; }}); throw e', + 'a TypeError was thrown', + ], ])('%s gives a message, and not a crash', async (code, text) => { expect(await rr('o', code, {pipedData: 'foo', replacementJs: true})).toEqual({ out: '', @@ -19,4 +23,13 @@ describe('#822 code that throws something that is not an error', () => { exit: 3, }); }); + + test('only tells the start of a big object', async () => { + const result = await rr('o', 'throw {text: "y".repeat(1000)}', { + pipedData: 'foo', + replacementJs: true, + }); + expect(result.err).toContain('an object was thrown: {"text":"' + 'y'.repeat(291) + '…'); + expect(result.err.length).toBeLessThan(400); + }); }); From eab2c2b8e9256aacc26e4d0fd6997e8f2dce42f3 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:20:44 +1100 Subject: [PATCH 07/15] Tell where --js code failed in a module with --verbose --- src/engine.ts | 31 ++++++++++++++++++++++++++++++- test/fix/824.test.js | 22 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 test/fix/824.test.js diff --git a/src/engine.ts b/src/engine.ts index d5725ccc..87df4410 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -819,6 +819,30 @@ function numbersCode(numbers) { .join(''); } +// A place in a stack, like `at fn (/path/file.js:2:15)`, also with `file://` in front +const stackPlaces = /\(?(?:file:\/\/)?((?:\/|[A-Za-z]:\\)[^()]+?)(:\d+:\d+)\)?\s*$/gm; + +// The folder of RexReplace itself, from the first place in a stack made here +const ownFolder = (() => { + const [place] = String(new Error().stack).matchAll(stackPlaces); + return place ? path.dirname(place[1]) + path.sep : null; +})(); + +// The first place in the stack of an error that is in a file of the user, like `helper.js:2:15`. +// Only the places above the code of RexReplace are from the code of the user, so the search stops there. +function stackPlace(thrown) { + const stack = 'object' === typeof thrown && null !== thrown ? thrown.stack : ''; + for (const [, file, place] of String(stack ?? '').matchAll(stackPlaces)) { + if (ownFolder && file.startsWith(ownFolder)) { + return ''; + } + if (!file.includes('node:')) { + return path.relative(process.cwd(), file) + place; + } + } + return ''; +} + // The text of what code threw. Code can throw anything, also a value that can not be made into text. function thrownText(thrown) { let text = ''; @@ -956,7 +980,12 @@ function dynamicReplacement( return dynamicContent(...args); } catch (e) { const where = _config_rr.dataIsPiped ? 'the piped data' : _file_rr; - return panic('The code of --js failed for ' + where + ':', thrownText(e)); + // With --verbose it also tells where the code failed, when that is in a file of the user, like a module loaded with r() + const at = _config_rr.verbose ? stackPlace(e) : ''; + return panic( + 'The code of --js failed for ' + where + ':', + thrownText(e) + (at ? '\n at ' + at : '') + ); } }; diff --git a/test/fix/824.test.js b/test/fix/824.test.js new file mode 100644 index 00000000..02f74383 --- /dev/null +++ b/test/fix/824.test.js @@ -0,0 +1,22 @@ +// https://github.com/mathiasrw/rexreplace/issues/824 +// Since #808 --js code that failed only gave the message, also with --verbose, so a module loaded with r() did not tell where. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#824 --js code that fails in a module', () => { + test('tells the file and the line with --verbose', async () => { + fs.writeFileSync('helper.js', 'module.exports = value => {\n\treturn value.nope.deeper;\n};\n'); + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + const result = await rr('o', 'r("helper")($0)', config); + expect(result.err).toMatch(/\n {4}at helper\.js:2:\d+\n {4}No further processing\.\n/); + expect(result.exit).toBe(3); + }); + + test('does not tell where without --verbose, or when the code in the command fails', async () => { + fs.writeFileSync('helper.js', 'module.exports = value => {\n\treturn value.nope.deeper;\n};\n'); + const config = {pipedData: 'foo', replacementJs: true}; + expect((await rr('o', 'r("helper")($0)', config)).err).not.toContain(' at '); + expect((await rr('o', 'nope()', {...config, verbose: true})).err).not.toContain(' at '); + }); +}); From e22597762c09fb3f4dceabf2a3bb527a03b1b99d Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:13:46 +1100 Subject: [PATCH 08/15] Read each line of the stack, skip what is not the code of the user, and never crash on it --- src/engine.ts | 34 ++++++++++++++++++++++------------ test/fix/824.test.js | 26 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index 87df4410..d6945541 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -819,27 +819,37 @@ function numbersCode(numbers) { .join(''); } -// A place in a stack, like `at fn (/path/file.js:2:15)`, also with `file://` in front -const stackPlaces = /\(?(?:file:\/\/)?((?:\/|[A-Za-z]:\\)[^()]+?)(:\d+:\d+)\)?\s*$/gm; +// The places in a stack, one for each line, like `at fn (/path/file.js:2:15)` or `at /path/file.js:2:15`. +// The place is at the end of the line, so a path with parentheses is read as a whole. +function stackPlaces(stack) { + return String(stack) + .split('\n') + .map(line => line.match(/^\s*at (?:.*? \()?(?:file:\/\/)?(.+?)(:\d+:\d+)\)?\s*$/)) + .filter(Boolean) + .map(([, file, place]) => ({file, place})); +} // The folder of RexReplace itself, from the first place in a stack made here const ownFolder = (() => { - const [place] = String(new Error().stack).matchAll(stackPlaces); - return place ? path.dirname(place[1]) + path.sep : null; + const [own] = stackPlaces(new Error().stack); + return own ? path.dirname(own.file) + path.sep : null; })(); // The first place in the stack of an error that is in a file of the user, like `helper.js:2:15`. // Only the places above the code of RexReplace are from the code of the user, so the search stops there. +// Places in Node itself, in eval and in node_modules are skipped. Reading the stack can throw, so it is tried. function stackPlace(thrown) { - const stack = 'object' === typeof thrown && null !== thrown ? thrown.stack : ''; - for (const [, file, place] of String(stack ?? '').matchAll(stackPlaces)) { - if (ownFolder && file.startsWith(ownFolder)) { - return ''; - } - if (!file.includes('node:')) { - return path.relative(process.cwd(), file) + place; + try { + for (const {file, place} of stackPlaces(thrown?.stack ?? '')) { + if (ownFolder && file.startsWith(ownFolder)) { + return ''; + } + // A file is an absolute path, so the frames of code run with eval are skipped too + if (/^(\/|[A-Za-z]:[\\/])/.test(file) && !file.includes('node_modules')) { + return path.relative(process.cwd(), file) + place; + } } - } + } catch (e) {} return ''; } diff --git a/test/fix/824.test.js b/test/fix/824.test.js index 02f74383..ae2d4bc9 100644 --- a/test/fix/824.test.js +++ b/test/fix/824.test.js @@ -19,4 +19,30 @@ describe('#824 --js code that fails in a module', () => { expect((await rr('o', 'r("helper")($0)', config)).err).not.toContain(' at '); expect((await rr('o', 'nope()', {...config, verbose: true})).err).not.toContain(' at '); }); + + test('reads a path with parentheses as a whole', async () => { + fs.mkdirSync('a (copy)'); + fs.writeFileSync( + 'a (copy)/helper.js', + 'module.exports = value => {\n\treturn value.nope.deeper;\n};\n' + ); + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + expect((await rr('o', 'r("a (copy)/helper.js")($0)', config)).err).toMatch( + /\n {4}at a \(copy\)\/helper\.js:2:\d+\n/ + ); + }); + + test('does not tell places in the runtime itself', async () => { + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + expect((await rr('o', 'new URL("x")', config)).err).not.toContain(' at '); + }); + + test('does not crash when the stack can not be read', async () => { + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + const result = await rr('o', 'throw new Proxy({}, {get() { throw 1; }})', config); + expect(result.err).toContain( + 'The code of --js failed for the piped data: an object was thrown' + ); + expect(result.exit).toBe(3); + }); }); From 2ad9cd282d30f5a2350aed5e5819a297bf5aafca Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:23:04 +1100 Subject: [PATCH 09/15] Only make the --js numbers the code uses, and only count files for $i and $c --- src/engine.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index d6945541..bbf5edb6 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -132,6 +132,9 @@ async function run(config) { config.replacementOri = config.replacement; + // $i and $c need one more search of each file, so the files are only counted when the code uses them + config.countFiles = config.replacementJs && /\$[ic](?![A-Za-z])/.test(config.replacementOri); + // The most specific delimiter wins. Without one there is a line break after each item, unless it already ends with one. const delimiter = config.delimiter ?? (config.nullOut ? '\0' : null); config.nameDelimiter = config.nameDelimiter ?? delimiter; @@ -784,8 +787,12 @@ function fileNumbers(config, number, total) { return {n: number, t: total, i: config.numbers.matched + 1, c: config.numbers.changed + 1}; } -// When a file is done, it counts for $i if the pattern found something, and for $c if it was changed +// When a file is done, it counts for $i if the pattern found something, and for $c if it was changed. +// It takes one more search of the file, so it is only done when the code uses $i or $c. function countFile(config, text, result) { + if (!config.countFiles) { + return; + } if (-1 !== text.search(config.regex)) { config.numbers.matched++; } @@ -805,8 +812,10 @@ function usesNamedGroups(config) { // Code that makes the numbers and their cousins, put in front of the code of --js. // $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, and the ones with _ at the end have a space at the end. -function numbersCode(numbers) { +// Only the numbers the code uses are made, as each one costs time when the code runs once per match. +function numbersCode(numbers, code) { return Object.entries(numbers) + .filter(([name]) => new RegExp('\\$' + name + '(?![A-Za-z])').test(code)) .map(([name, value]) => { const cousins = [`$${name} = ${value}`, `$${name}_ = '${value} '`]; for (let width = 2; width <= 10; width++) { @@ -893,7 +902,7 @@ function dynamicReplacement( const _pipe = _config_rr.pipedData, _text = _data_rr, _find = _config_rr.pattern, - code_rr = numbersCode(_numbers_rr) + _config_rr.replacementOri, + code_rr = numbersCode(_numbers_rr, _config_rr.replacementOri) + _config_rr.replacementOri, _cwd = process.cwd(), _now = _time, _ = ' ', @@ -1122,7 +1131,10 @@ function dynamicReplacement( _match_rr++; _config_rr.numbers.matches++; - var capturedGroups = numbersCode({m: _match_rr, M: _config_rr.numbers.matches}); + var capturedGroups = numbersCode( + {m: _match_rr, M: _config_rr.numbers.matches}, + _config_rr.replacementOri + ); // The match and the groups are followed by the offset and the full text, and by the named groups if the pattern has any const named = arguments[arguments.length - 1]; const hasNamed = 'object' === typeof named && null !== named; From 71b1a5141414b6394eedc0a673858769fd3c3c73 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:15:17 +1100 Subject: [PATCH 10/15] Tell in the help that only the numbers the code names are made --- README.md | 2 +- src/cli.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ad9f9239..0d9ac876 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A number is only made when the code names it, so a name built when the code runs, like `eval('$' + 'n')`, is not there. A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 6d49a484..5dd977a2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -409,7 +409,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f \`mtime_obj\`: date object representing \`mtime\`, \`ctime_obj\`: date object representing \`ctime\`. -The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). +The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). A number is only made when the code names it, so a name built when the code runs, like \`eval('$' + 'n')\`, is not there. A named group is a variable with \`_\` in front: \`_year\` for \`(?...)\`, and \`_year_\` has a space at the end. From 1b890fe89ddf344214fa9c1adf7dd1e7bc7b6502 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:26:25 +1100 Subject: [PATCH 11/15] Leave the README line of --js to #838, so the stack can be rebased without a conflict --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d9ac876..ad9f9239 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A number is only made when the code names it, so a name built when the code runs, like `eval('$' + 'n')`, is not there. A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | From 07dbd4d23faf5e6900d6a9c882cc37d493ba1069 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:25:00 +1100 Subject: [PATCH 12/15] Let --js code declare the names of the numbers and groups with let and const --- src/engine.ts | 5 ++++- test/fix/815.test.js | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 test/fix/815.test.js diff --git a/src/engine.ts b/src/engine.ts index bbf5edb6..1ebacddf 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -902,7 +902,10 @@ function dynamicReplacement( const _pipe = _config_rr.pipedData, _text = _data_rr, _find = _config_rr.pattern, - code_rr = numbersCode(_numbers_rr, _config_rr.replacementOri) + _config_rr.replacementOri, + // The code of the user is in a block of its own, so it can declare the same names with let and const. + // The value of the block is the value of its last statement, like before. The line break ends a comment at the end. + code_rr = + numbersCode(_numbers_rr, _config_rr.replacementOri) + '{' + _config_rr.replacementOri + '\n}', _cwd = process.cwd(), _now = _time, _ = ' ', diff --git a/test/fix/815.test.js b/test/fix/815.test.js new file mode 100644 index 00000000..95941206 --- /dev/null +++ b/test/fix/815.test.js @@ -0,0 +1,20 @@ +// https://github.com/mathiasrw/rexreplace/issues/815 +// The numbers and the named groups were made with var in front of the code, so code that declared the same name with let or const did not run. +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +describe('#815 --js code can declare its own variables', () => { + test.each([ + ['a number with let', 'o', 'let $t = "T"; $t', 'fTT'], + ['a group with const', '(o)', 'const $1 = "X"; $1', 'fXX'], + ['a named group with let', '(?o)', 'let _o = "N"; _o', 'fNN'], + ])('with the name of %s', async (name, pattern, code, out) => { + expect((await rr(pattern, code, {pipedData: 'foo', replacementJs: true})).out).toBe(out); + }); + + test('the value is still the value of the last statement, also with a comment at the end', async () => { + expect( + (await rr('o', 'const a = 1; a + 1 // two', {pipedData: 'foo', replacementJs: true})).out + ).toBe('f22'); + }); +}); From f7b634f5f31194ab5d14735711d8da0bbcfe3480 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:26:59 +1100 Subject: [PATCH 13/15] Use the prefix rr_ for named groups in --js, and panic on a name that ends with _ --- README.md | 2 +- src/cli.ts | 2 +- src/engine.ts | 25 ++++++++++++++++++++----- test/fix/799.test.js | 8 ++++---- test/fix/815.test.js | 2 +- test/fix/825.test.js | 26 ++++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 test/fix/825.test.js diff --git a/README.md b/README.md index ad9f9239..d5311a9e 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `rr_` in front: `rr_year` for `(?...)`, and `rr_year_` has a space at the end. A named group can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 5dd977a2..ac0067e4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -411,7 +411,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). A number is only made when the code names it, so a name built when the code runs, like \`eval('$' + 'n')\`, is not there. -A named group is a variable with \`_\` in front: \`_year\` for \`(?...)\`, and \`_year_\` has a space at the end. +A named group is a variable with \`rr_\` in front: \`rr_year\` for \`(?...)\`, and \`rr_year_\` has a space at the end. A named group can not end with \`_\`. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 1ebacddf..d0ac52b7 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -145,6 +145,16 @@ async function run(config) { config.regex = getFinalRegex(config.pattern, config) || ''; + // In --js a named group is a variable like rr_year, and rr_year_ has a space at the end. + // A group that ends with _ would have the same name as another group with a space, so it can not be used. + const endsWith_ = + config.replacementJs && config.regex + ? namedGroups(config.regex).find(name => name.endsWith('_')) + : null; + if (endsWith_) { + panic('A named group can not end with _:', endsWith_); + } + // What the regex engine is asked to do. With --replacement-js and --output-match it gets a function. if (config.statusInfo.engine) { config.statusInfo.engine.execution = { @@ -801,15 +811,20 @@ function countFile(config, text, result) { } } -// The code uses a named group of the pattern, like _year for (?…) +// The code uses a named group of the pattern, like rr_year for (?…) function usesNamedGroups(config) { - return [...config.regex.source.matchAll(/\(\?<([A-Za-z_$][\w$]*)>/g)].some(([, name]) => - new RegExp('(^|[^\\w$])_' + name.replace(/\$/g, '\\$') + '(?![\\w$])').test( + return namedGroups(config.regex).some(name => + new RegExp('(^|[^\\w$])rr_' + name.replace(/\$/g, '\\$') + '(?![\\w$])').test( config.replacementOri ) ); } +// The names of the named groups of a regex, like year for (?…) +function namedGroups(regex) { + return [...regex.source.matchAll(/\(\?<([A-Za-z_$][\w$]*)>/g)].map(([, name]) => name); +} + // Code that makes the numbers and their cousins, put in front of the code of --js. // $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, and the ones with _ at the end have a space at the end. // Only the numbers the code uses are made, as each one costs time when the code runs once per match. @@ -1144,10 +1159,10 @@ function dynamicReplacement( for (var i = 0; i < arguments.length - (hasNamed ? 3 : 2); i++) { capturedGroups += 'var $' + i + '=' + JSON.stringify(arguments[i]) + '; '; } - // A named group is a variable with _ in front, like _year for (?…), and _year_ has a space at the end + // A named group is a variable with rr_ in front, like rr_year for (?…), and rr_year_ has a space at the end if (hasNamed) { for (const [name, value] of Object.entries(named)) { - capturedGroups += `var _${name} = ${JSON.stringify(value)}, _${name}_ = _${name} + ' '; `; + capturedGroups += `var rr_${name} = ${JSON.stringify(value)}, rr_${name}_ = rr_${name} + ' '; `; } } diff --git a/test/fix/799.test.js b/test/fix/799.test.js index f4f4f568..d7c4ddf8 100644 --- a/test/fix/799.test.js +++ b/test/fix/799.test.js @@ -9,16 +9,16 @@ describe('#799 groups in --js', () => { expect((await rr('(?a)(b)', '$1 + $2 + typeof $3', config)).out).toBe('abundefined'); }); - test('a named group is a variable with _ in front, and _ at the end has a space', async () => { + test('a named group is a variable with rr_ in front, and _ at the end has a space', async () => { const config = {pipedData: '2024-05 1999-12', replacementJs: true}; const pattern = '(?\\d{4})-(?\\d\\d)'; - expect((await rr(pattern, '_month + "/" + _year', config)).out).toBe('05/2024 12/1999'); - expect((await rr(pattern, '_month_ + _year', config)).out).toBe('05 2024 12 1999'); + expect((await rr(pattern, 'rr_month + "/" + rr_year', config)).out).toBe('05/2024 12/1999'); + expect((await rr(pattern, 'rr_month_ + rr_year', config)).out).toBe('05 2024 12 1999'); }); test('a named group that took no part in the match is undefined', async () => { const config = {pipedData: 'ab', replacementJs: true}; - expect((await rr('(?a)|(?b)', 'String(_x) + String(_y) + ","', config)).out).toBe( + expect((await rr('(?a)|(?b)', 'String(rr_x) + String(rr_y) + ","', config)).out).toBe( 'aundefined,undefinedb,' ); }); diff --git a/test/fix/815.test.js b/test/fix/815.test.js index 95941206..efb3413b 100644 --- a/test/fix/815.test.js +++ b/test/fix/815.test.js @@ -7,7 +7,7 @@ describe('#815 --js code can declare its own variables', () => { test.each([ ['a number with let', 'o', 'let $t = "T"; $t', 'fTT'], ['a group with const', '(o)', 'const $1 = "X"; $1', 'fXX'], - ['a named group with let', '(?o)', 'let _o = "N"; _o', 'fNN'], + ['a named group with let', '(?o)', 'let rr_o = "N"; rr_o', 'fNN'], ])('with the name of %s', async (name, pattern, code, out) => { expect((await rr(pattern, code, {pipedData: 'foo', replacementJs: true})).out).toBe(out); }); diff --git a/test/fix/825.test.js b/test/fix/825.test.js new file mode 100644 index 00000000..c7c1901c --- /dev/null +++ b/test/fix/825.test.js @@ -0,0 +1,26 @@ +// https://github.com/mathiasrw/rexreplace/issues/825 +// A named group was a variable like _year, which could clash: with groups a and a_, _a_ was both a_ and a with a space, +// and a group named _require_ broke r(). +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +describe('#825 named groups in --js', () => { + test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { + const config = {pipedData: 'ab', replacementJs: true}; + expect((await rr('(?a)', 'rr_require + typeof r', config)).out).toBe('afunctionb'); + }); + + test('can not end with _, as that is the name of another group with a space', async () => { + expect(await rr('(?\\d)(?\\w)', 'rr_a', {pipedData: '1x', replacementJs: true})).toEqual( + { + out: '', + err: ' ❌ A named group can not end with _: a_\n No further processing.\n\n', + exit: 3, + } + ); + }); + + test('can end with _ without --js, as there are no variables then', async () => { + expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); + }); +}); From c3fcc3604eb71c105045f09573bcffa3ee1c101d Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:17:32 +1100 Subject: [PATCH 14/15] Do not panic on a group name that ends with _ under --output-match --- src/engine.ts | 3 ++- test/fix/825.test.js | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index d0ac52b7..676d97d3 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -147,8 +147,9 @@ async function run(config) { // In --js a named group is a variable like rr_year, and rr_year_ has a space at the end. // A group that ends with _ would have the same name as another group with a space, so it can not be used. + // With --output-match the code is not run, so there are no variables. const endsWith_ = - config.replacementJs && config.regex + config.replacementJs && !config.outputMatch && config.regex ? namedGroups(config.regex).find(name => name.endsWith('_')) : null; if (endsWith_) { diff --git a/test/fix/825.test.js b/test/fix/825.test.js index c7c1901c..58bb3614 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -6,8 +6,10 @@ import {rr} from '../helper.js'; describe('#825 named groups in --js', () => { test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { + // With _ in front, the group _require with a space was __require_, which r() uses itself. + // $1 makes the code run once per match, where the variables of the groups are made. const config = {pipedData: 'ab', replacementJs: true}; - expect((await rr('(?a)', 'rr_require + typeof r', config)).out).toBe('afunctionb'); + expect((await rr('(?<_require>a)', '$1 + r("path").sep', config)).out).toBe('a/b'); }); test('can not end with _, as that is the name of another group with a space', async () => { @@ -20,6 +22,11 @@ describe('#825 named groups in --js', () => { ); }); + test('can end with _ with --output-match, as the code is not run', async () => { + const config = {pipedData: 'ab', replacementJs: true, outputMatch: true}; + expect((await rr('(?a)', 'x', config)).exit).toBe(null); + }); + test('can end with _ without --js, as there are no variables then', async () => { expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); }); From ad127c64ca6cf0e4c552560f6ad794fceb600284 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 02:10:00 +1100 Subject: [PATCH 15/15] Test named groups with uppercase, numbers, $ and the space cousin, and say the rr_ help shorter --- src/cli.ts | 2 +- test/fix/825.test.js | 51 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index ac0067e4..4989cba0 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -411,7 +411,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). A number is only made when the code names it, so a name built when the code runs, like \`eval('$' + 'n')\`, is not there. -A named group is a variable with \`rr_\` in front: \`rr_year\` for \`(?...)\`, and \`rr_year_\` has a space at the end. A named group can not end with \`_\`. +Access a named group via the name prepended with \`rr_\`. \`(?...)\` gives you \`rr_year\`. A named group name can not end with \`_\`. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/test/fix/825.test.js b/test/fix/825.test.js index 58bb3614..067420b2 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -2,9 +2,29 @@ // A named group was a variable like _year, which could clash: with groups a and a_, _a_ was both a_ and a with a space, // and a group named _require_ broke r(). import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; import {rr} from '../helper.js'; +const js = {pipedData: 'in 1999 and 2024', replacementJs: true}; + describe('#825 named groups in --js', () => { + test.each([ + ['a name in lowercase', '(?\\d+)', '"<" + rr_year + ">"', 'in <1999> and <2024>'], + [ + 'the cousin with a space at the end', + '(?\\d+)', + 'rr_year + "," + rr_year_', + 'in 1999,1999 and 2024,2024 ', + ], + ['a name in uppercase', '(?\\d+)', '"Y" + rr_Year', 'in Y1999 and Y2024'], + ['a name with a number', '(?\\d+)', '"Y" + rr_y2k', 'in Y1999 and Y2024'], + ['a name with $', '(?\\d+)', '"Y" + rr_a$', 'in Y1999 and Y2024'], + ['two names that only differ by case', '(?\\d)(?\\d)', 'rr_Y + rr_y', 'in 9199 and 0242'], + ['a name next to its number', '(?\\d+)', '$1 + rr_year', 'in 19991999 and 20242024'], + ])('gives %s', async (name, pattern, code, out) => { + expect(await rr(pattern, code, js)).toEqual({out, err: '', exit: null}); + }); + test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { // With _ in front, the group _require with a space was __require_, which r() uses itself. // $1 makes the code run once per match, where the variables of the groups are made. @@ -12,22 +32,35 @@ describe('#825 named groups in --js', () => { expect((await rr('(?<_require>a)', '$1 + r("path").sep', config)).out).toBe('a/b'); }); - test('can not end with _, as that is the name of another group with a space', async () => { - expect(await rr('(?\\d)(?\\w)', 'rr_a', {pipedData: '1x', replacementJs: true})).toEqual( - { + test('are gone with _ in front', async () => { + expect((await rr('(?\\d+)', '_year', js)).err).toContain('ReferenceError'); + }); +}); + +describe('#825 a named group that ends with _', () => { + test.each([['a_'], ['Year_'], ['_'], ['y2_']])( + 'panics for %s, as that is the name of another group with a space, and nothing is changed', + async name => { + fs.writeFileSync('a.md', 'in 1999'); + const result = await rr(`(?<${name}>\\d+)`, '"x"', { + includeGlob: ['a.md'], + replacementJs: true, + }); + expect(result).toEqual({ out: '', - err: ' ❌ A named group can not end with _: a_\n No further processing.\n\n', + err: ` ❌ A named group can not end with _: ${name}\n No further processing.\n\n`, exit: 3, - } - ); - }); + }); + expect(fs.readFileSync('a.md', 'utf8')).toBe('in 1999'); + } + ); - test('can end with _ with --output-match, as the code is not run', async () => { + test('is fine with --output-match, as the code is not run', async () => { const config = {pipedData: 'ab', replacementJs: true, outputMatch: true}; expect((await rr('(?a)', 'x', config)).exit).toBe(null); }); - test('can end with _ without --js, as there are no variables then', async () => { + test('is fine without --js, as there are no variables then', async () => { expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); }); });