From 918855de9c0d31d7ed3862646fdeb52fe59c0bce Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 22:03:28 +1100 Subject: [PATCH 01/29] Give a panic and not a stack trace when a path can not be looked at --- src/engine.ts | 24 +++++++++++++++++++++--- test/fix/788.test.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 test/fix/788.test.js diff --git a/src/engine.ts b/src/engine.ts index 49c83ad9..39ca4fa2 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1135,7 +1135,7 @@ function getFilePaths(conf) { // --exclude-glob is always globs, also for the name of a file. They are used together, and the files they find are taken away. if (excludeGlob.length) { - const excluded = new Set(globs.sync(excludeGlob, globOptions).map(file => path.resolve(file))); + const excluded = new Set(glob(excludeGlob, globOptions).map(file => path.resolve(file))); files = files.filter(file => !excluded.has(path.resolve(file))); } @@ -1200,13 +1200,31 @@ function sortFiles(files, sort, conf) { return files.sort((a, b) => key(stats.get(a)) - key(stats.get(b)) || ascii(a, b)); } +// A path that can not be looked at (like a name that is too long) is not a file, so it is globbed +function isFile(file) { + try { + return Boolean(fs.statSync(file, {throwIfNoEntry: false})?.isFile()); + } catch (e) { + return false; + } +} + +// When the file system gives an error, the files can not be known, so the run can not go on +function glob(patterns, globOptions) { + try { + return globs.sync(patterns, globOptions); + } catch (e) { + return panic('The files can not be found:', e.message); + } +} + // A path that is a file is used as it is, so a name with glob characters, like `a[1].md` or `my (1).md`, gives only that file. // The other paths are globbed together, so a pattern that starts with `!` can take files away from the other patterns. function findFiles(paths, globOptions, sort) { const files = []; const patterns = []; paths.forEach(file => { - if (fs.statSync(file, {throwIfNoEntry: false})?.isFile()) { + if (isFile(file)) { files.push(file); } else { patterns.push(file); @@ -1216,7 +1234,7 @@ function findFiles(paths, globOptions, sort) { // A file given by its name and also found by a glob is only handled once const seen = new Set(); // A glob has no order of its own, so with --sort none its files come after the paths, sorted with natural - const found = patterns.length ? globs.sync(patterns, globOptions) : []; + const found = patterns.length ? glob(patterns, globOptions) : []; return files.concat('none' === sort ? found.sort(natural) : found).filter(file => { const fullPath = path.resolve(file); if (seen.has(fullPath)) { diff --git a/test/fix/788.test.js b/test/fix/788.test.js new file mode 100644 index 00000000..b572225c --- /dev/null +++ b/test/fix/788.test.js @@ -0,0 +1,31 @@ +// https://github.com/mathiasrw/rexreplace/issues/788 +// A path the file system could not look at, like a name that is too long, crashed the run with a stack trace. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#788 a path that can not be looked at', () => { + test('is not a file, so it is globbed', async () => { + fs.writeFileSync('a.md', 'foo'); + const long = `{a,${'x'.repeat(300)}}.md`; + expect(await rr('foo', 'bar', {includeGlob: [long]})).toEqual({ + out: 'a.md\n', + err: '', + exit: null, + }); + expect(fs.readFileSync('a.md', 'utf8')).toBe('bar'); + }); + + test.each([ + ['as an argument', {includeGlob: ['a.md/x']}], + ['from --glob-pipe', {pipedData: 'a.md/x', globPipe: true}], + ['in --exclude-glob', {includeGlob: ['a.md'], excludeGlob: ['a.md/x']}], + ])('gives a panic when the glob can not be used, %s', async (name, config) => { + fs.writeFileSync('a.md', 'foo'); + const result = await rr('foo', 'bar', config); + expect(result.err).toStartWith(' ❌ The files can not be found: ENOTDIR'); + expect(result.err).toEndWith('\n No further processing.\n\n'); + expect(result.exit).toBe(3); + expect(fs.readFileSync('a.md', 'utf8')).toBe('foo'); + }); +}); From e366dd68d2879781af6b87a81b1cbd9120edb757 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 22:05:23 +1100 Subject: [PATCH 02/29] Stop starting new files after a panic with --halt --- src/engine.ts | 11 +++++++++-- test/fix/789.test.js | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 test/fix/789.test.js diff --git a/src/engine.ts b/src/engine.ts index 39ca4fa2..de4447cb 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -230,10 +230,17 @@ async function run(config) { ? files.length : Math.max(16, os.availableParallelism() * 2); let next = 0; + // After a panic no worker starts a new file. A file that is already in work is finished. + let stopped = false; const worker = async () => { - while (next < files.length) { + while (next < files.length && !stopped) { const index = next++; - await openFile(files[index], config, index + 1, files.length); + try { + await openFile(files[index], config, index + 1, files.length); + } catch (e) { + stopped = true; + throw e; + } } }; diff --git a/test/fix/789.test.js b/test/fix/789.test.js new file mode 100644 index 00000000..ca8c6159 --- /dev/null +++ b/test/fix/789.test.js @@ -0,0 +1,22 @@ +// https://github.com/mathiasrw/rexreplace/issues/789 +// With --halt and the async flow, the engine threw the panic, but the workers went on taking new files. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import os from 'os'; +import {rr} from '../helper.js'; + +const workers = Math.max(16, os.availableParallelism() * 2); + +describe('#789 a panic with --halt', () => { + test('stops the workers from starting new files', async () => { + const names = Array.from({length: workers * 10}, (_, i) => `${String(i).padStart(4, '0')}.txt`); + names.forEach(name => fs.writeFileSync(name, 'foo')); + fs.chmodSync(names[0], 0); + const result = await rr('foo', 'bar', {includeGlob: ['*.txt'], voidAsync: false, halt: true}); + expect(result.exit).toBe(3); + // The files that were already in work when the panic came are finished, but no new file is started + await Bun.sleep(300); + const changed = names.slice(1).filter(name => 'bar' === fs.readFileSync(name, 'utf8')); + expect(changed.length).toBeLessThan(workers); + }); +}); From ff4c0e51444ac7b4601122acbbdab4e76dc1de31 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 22:07:07 +1100 Subject: [PATCH 03/29] Read the piped paths of --glob-pipe as UTF-8 --- src/cli.ts | 3 ++- test/fix/790.test.js | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test/fix/790.test.js diff --git a/src/cli.ts b/src/cli.ts index 9d170219..bee07621 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -665,7 +665,8 @@ if (!pipeDataExpected || Boolean(process.stdin.isTTY)) { // Stays null when nothing comes through the pipe let pipeData = null; - process.stdin.setEncoding(config.encoding); + // With --glob-pipe the piped data is paths, and they are UTF-8 and not in the encoding of the files + process.stdin.setEncoding(config.globPipe ? 'utf8' : config.encoding); process.stdin.on('data', chunk => { pipeData = (pipeData ?? '') + chunk; diff --git a/test/fix/790.test.js b/test/fix/790.test.js new file mode 100644 index 00000000..c467024d --- /dev/null +++ b/test/fix/790.test.js @@ -0,0 +1,21 @@ +// https://github.com/mathiasrw/rexreplace/issues/790 +// With --glob-pipe the piped paths were decoded with --encoding, so with -e latin1 a name that is not ASCII found no file. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {$} from 'bun'; +import {cli, sh} from '../helper.js'; + +describe('#790 the paths piped in with --glob-pipe', () => { + test.each([ + ['line breaks', 'æ.file\nb.file\n', ''], + ['null bytes', 'æ.file\0b.file\0', '--null-in'], + ])('are read as UTF-8 with -e latin1, split by %s', async (name, paths, flag) => { + fs.writeFileSync('æ.file', 'foo'); + fs.writeFileSync('b.file', 'foo'); + const result = await sh( + $`bun ${cli} foo bar -g -e latin1 -A ${flag || []} < ${new Response(paths)}` + ); + expect(result).toEqual({out: 'æ.file\nb.file\n', err: '', exit: 0}); + expect(fs.readFileSync('æ.file', 'utf8')).toBe('bar'); + }); +}); From 9b6aec073f114e73c996d4ca227cda26d96c41e8 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:06:58 +1100 Subject: [PATCH 04/29] Remove control characters from the new names of --rename --- README.md | 2 +- src/cli.ts | 2 +- src/engine.ts | 11 ++++++++++- test/fix/826.test.js | 24 ++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 test/fix/826.test.js diff --git a/README.md b/README.md index efb9389c..ad9f9239 100644 --- a/README.md +++ b/README.md @@ -355,7 +355,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-z` | **`--expect-diff`** Check that at least one file is changed. The exit code is 1 if not. [boolean] | | `-Z` | **`--expect-zero-diff`** Check that no file is changed. The exit code is 1 if one is. [boolean] | | | **`--expect-diff-all`** Check that all files found are changed. The exit code is 1 if not. [boolean] | -| `-r` | **`--rename`** Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. [boolean] | +| `-r` | **`--rename`** Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. Control characters, like a line break, are removed from the new name. [boolean] | | | **`--rename-path`** Like --rename, but on the whole path of each file, so a / in the replacement moves the file. Folders that are not there are made. [boolean] | | | **`--rename-force`** With --rename or --rename-path, write over a file that is there already. [boolean] | | `-m` | **`--output-match`** Output each match on a new line. Will not replace any content but you still need to provide a dummy value (like `_`) as replacement parameter. The replacement is not used, so --js code is not run. If search pattern does not contain matching groups the full match will be outputted. If search pattern does contain matching groups only matching groups will be outputted (same line with no delimiter). [boolean] | diff --git a/src/cli.ts b/src/cli.ts index bee07621..6d49a484 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -300,7 +300,7 @@ const yargs = require('yargs') .boolean('r') .describe( 'r', - 'Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file.' + 'Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. Control characters, like a line break, are removed from the new name.' ) .alias('r', 'rename') diff --git a/src/engine.ts b/src/engine.ts index de4447cb..ba3419be 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -377,7 +377,7 @@ function renameFiles(files, config) { config.replacementJs && !config.outputMatch ? codeReplacement(file, config, text, fileNumbers(config, index + 1, files.length)) : config.replacement; - const result = text.replace(config.regex, replacement); + const result = withoutControlChars(text.replace(config.regex, replacement), file); countFile(config, text, result); if (result === text) { return {old: file, new: file}; @@ -418,6 +418,15 @@ function renameFiles(files, config) { }); } +// No name should have the chars below space in ASCII (like a line break, a tab or a null byte), or DEL +function withoutControlChars(name, file) { + const clean = name.replace(/[\x00-\x1f\x7f]/g, ''); + if (clean !== name) { + chat('Removed control characters from the new name of: ' + file); + } + return clean; +} + // With --simulate and --output the old name goes to stderr and the new one to stdout, so they come in pairs in a terminal function nameRename(item, config) { if (config.simulate || config.output) { diff --git a/test/fix/826.test.js b/test/fix/826.test.js new file mode 100644 index 00000000..36989218 --- /dev/null +++ b/test/fix/826.test.js @@ -0,0 +1,24 @@ +// https://github.com/mathiasrw/rexreplace/issues/826 +// --rename put control characters into the new name, like the line break at the end of a replacement file. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#826 control characters in a new name', () => { + test('are removed, like the line break at the end of a replacement file', async () => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('name.txt', 'b\n'); + const config = {includeGlob: ['a.md'], rename: true, replacementFile: true, verbose: true}; + const result = await rr('^a', 'name.txt', config); + expect(result.out).toBe('b.md\n'); + expect(result.err).toContain('Removed control characters from the new name of: a.md'); + expect(fs.readdirSync('.').sort()).toEqual(['b.md', 'name.txt']); + }); + + test('are removed with --rename-path too, also a tab, a null byte and DEL', async () => { + fs.writeFileSync('a.md', 'foo'); + const config = {includeGlob: ['a.md'], renamePath: true}; + expect((await rr('^a', 'b\t\0\x7f\x01c', config)).out).toBe('bc.md\n'); + expect(fs.readdirSync('.')).toEqual(['bc.md']); + }); +}); From a2c62df1dda41e22801876f61769ba7846f8ee3d Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:16:52 +1100 Subject: [PATCH 05/29] Give a message and not a crash when --js code throws something that is not an error --- src/engine.ts | 21 ++++++++++++++++++++- test/fix/822.test.js | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 test/fix/822.test.js diff --git a/src/engine.ts b/src/engine.ts index ba3419be..54f9ddd8 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -819,6 +819,25 @@ function numbersCode(numbers) { .join(''); } +// The text of what code threw. Code can throw anything, also a value that can not be made into text. +function thrownText(thrown) { + let text = ''; + try { + text = String(thrown); + } catch (e) {} + if (text && '[object Object]' !== text) { + return text; + } + if ('object' === typeof thrown && null !== thrown) { + let json = ''; + try { + json = JSON.stringify(thrown); + } catch (e) {} + return 'an object was thrown' + (json && '{}' !== json ? ': ' + json : ''); + } + return 'an empty ' + typeof thrown + ' was thrown'; +} + function dynamicReplacement( _file_rr, _config_rr, @@ -927,7 +946,7 @@ function dynamicReplacement( return dynamicContent(...args); } catch (e) { const where = _config_rr.dataIsPiped ? 'the piped data' : _file_rr; - return panic('The code of --js failed for ' + where + ':', String(e)); + return panic('The code of --js failed for ' + where + ':', thrownText(e)); } }; diff --git a/test/fix/822.test.js b/test/fix/822.test.js new file mode 100644 index 00000000..27f047d6 --- /dev/null +++ b/test/fix/822.test.js @@ -0,0 +1,22 @@ +// https://github.com/mathiasrw/rexreplace/issues/822 +// --js code that threw something that is not an error crashed the error handling, or gave an empty message. +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +const failed = text => + ` ❌ The code of --js failed for the piped data: ${text}\n No further processing.\n\n`; + +describe('#822 code that throws something that is not an error', () => { + test.each([ + ['throw Object.create(null)', 'an object was thrown'], + ['throw {code: 42}', 'an object was thrown: {"code":42}'], + ['throw ""', 'an empty string was thrown'], + ['throw 7', '7'], + ])('%s gives a message, and not a crash', async (code, text) => { + expect(await rr('o', code, {pipedData: 'foo', replacementJs: true})).toEqual({ + out: '', + err: failed(text), + exit: 3, + }); + }); +}); From 8be509baacc61bb5995a6cac2686b9fa062a94d5 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:11:38 +1100 Subject: [PATCH 06/29] Name a thrown error whose message can not be read, and cut a big thrown object short --- src/engine.ts | 12 +++++++++++- test/fix/822.test.js | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/engine.ts b/src/engine.ts index 54f9ddd8..d5725ccc 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -829,11 +829,21 @@ function thrownText(thrown) { return text; } if ('object' === typeof thrown && null !== thrown) { + // Like an Error whose message can not be read: its name is still known + let name = ''; + try { + name = thrown.constructor?.name; + } catch (e) {} + if (name && 'Object' !== name) { + return 'a ' + name + ' was thrown'; + } let json = ''; try { json = JSON.stringify(thrown); } catch (e) {} - return 'an object was thrown' + (json && '{}' !== json ? ': ' + json : ''); + // The object can hold a whole file, so only the start of it is told + const start = json.length > 300 ? json.slice(0, 300) + '…' : json; + return 'an object was thrown' + (json && '{}' !== json ? ': ' + start : ''); } return 'an empty ' + typeof thrown + ' was thrown'; } diff --git a/test/fix/822.test.js b/test/fix/822.test.js index 27f047d6..cfa1b874 100644 --- a/test/fix/822.test.js +++ b/test/fix/822.test.js @@ -12,6 +12,10 @@ describe('#822 code that throws something that is not an error', () => { ['throw {code: 42}', 'an object was thrown: {"code":42}'], ['throw ""', 'an empty string was thrown'], ['throw 7', '7'], + [ + 'const e = new TypeError("x"); Object.defineProperty(e, "message", {get() { throw 1; }}); throw e', + 'a TypeError was thrown', + ], ])('%s gives a message, and not a crash', async (code, text) => { expect(await rr('o', code, {pipedData: 'foo', replacementJs: true})).toEqual({ out: '', @@ -19,4 +23,13 @@ describe('#822 code that throws something that is not an error', () => { exit: 3, }); }); + + test('only tells the start of a big object', async () => { + const result = await rr('o', 'throw {text: "y".repeat(1000)}', { + pipedData: 'foo', + replacementJs: true, + }); + expect(result.err).toContain('an object was thrown: {"text":"' + 'y'.repeat(291) + '…'); + expect(result.err.length).toBeLessThan(400); + }); }); From eab2c2b8e9256aacc26e4d0fd6997e8f2dce42f3 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:20:44 +1100 Subject: [PATCH 07/29] Tell where --js code failed in a module with --verbose --- src/engine.ts | 31 ++++++++++++++++++++++++++++++- test/fix/824.test.js | 22 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 test/fix/824.test.js diff --git a/src/engine.ts b/src/engine.ts index d5725ccc..87df4410 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -819,6 +819,30 @@ function numbersCode(numbers) { .join(''); } +// A place in a stack, like `at fn (/path/file.js:2:15)`, also with `file://` in front +const stackPlaces = /\(?(?:file:\/\/)?((?:\/|[A-Za-z]:\\)[^()]+?)(:\d+:\d+)\)?\s*$/gm; + +// The folder of RexReplace itself, from the first place in a stack made here +const ownFolder = (() => { + const [place] = String(new Error().stack).matchAll(stackPlaces); + return place ? path.dirname(place[1]) + path.sep : null; +})(); + +// The first place in the stack of an error that is in a file of the user, like `helper.js:2:15`. +// Only the places above the code of RexReplace are from the code of the user, so the search stops there. +function stackPlace(thrown) { + const stack = 'object' === typeof thrown && null !== thrown ? thrown.stack : ''; + for (const [, file, place] of String(stack ?? '').matchAll(stackPlaces)) { + if (ownFolder && file.startsWith(ownFolder)) { + return ''; + } + if (!file.includes('node:')) { + return path.relative(process.cwd(), file) + place; + } + } + return ''; +} + // The text of what code threw. Code can throw anything, also a value that can not be made into text. function thrownText(thrown) { let text = ''; @@ -956,7 +980,12 @@ function dynamicReplacement( return dynamicContent(...args); } catch (e) { const where = _config_rr.dataIsPiped ? 'the piped data' : _file_rr; - return panic('The code of --js failed for ' + where + ':', thrownText(e)); + // With --verbose it also tells where the code failed, when that is in a file of the user, like a module loaded with r() + const at = _config_rr.verbose ? stackPlace(e) : ''; + return panic( + 'The code of --js failed for ' + where + ':', + thrownText(e) + (at ? '\n at ' + at : '') + ); } }; diff --git a/test/fix/824.test.js b/test/fix/824.test.js new file mode 100644 index 00000000..02f74383 --- /dev/null +++ b/test/fix/824.test.js @@ -0,0 +1,22 @@ +// https://github.com/mathiasrw/rexreplace/issues/824 +// Since #808 --js code that failed only gave the message, also with --verbose, so a module loaded with r() did not tell where. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#824 --js code that fails in a module', () => { + test('tells the file and the line with --verbose', async () => { + fs.writeFileSync('helper.js', 'module.exports = value => {\n\treturn value.nope.deeper;\n};\n'); + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + const result = await rr('o', 'r("helper")($0)', config); + expect(result.err).toMatch(/\n {4}at helper\.js:2:\d+\n {4}No further processing\.\n/); + expect(result.exit).toBe(3); + }); + + test('does not tell where without --verbose, or when the code in the command fails', async () => { + fs.writeFileSync('helper.js', 'module.exports = value => {\n\treturn value.nope.deeper;\n};\n'); + const config = {pipedData: 'foo', replacementJs: true}; + expect((await rr('o', 'r("helper")($0)', config)).err).not.toContain(' at '); + expect((await rr('o', 'nope()', {...config, verbose: true})).err).not.toContain(' at '); + }); +}); From e22597762c09fb3f4dceabf2a3bb527a03b1b99d Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:13:46 +1100 Subject: [PATCH 08/29] Read each line of the stack, skip what is not the code of the user, and never crash on it --- src/engine.ts | 34 ++++++++++++++++++++++------------ test/fix/824.test.js | 26 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index 87df4410..d6945541 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -819,27 +819,37 @@ function numbersCode(numbers) { .join(''); } -// A place in a stack, like `at fn (/path/file.js:2:15)`, also with `file://` in front -const stackPlaces = /\(?(?:file:\/\/)?((?:\/|[A-Za-z]:\\)[^()]+?)(:\d+:\d+)\)?\s*$/gm; +// The places in a stack, one for each line, like `at fn (/path/file.js:2:15)` or `at /path/file.js:2:15`. +// The place is at the end of the line, so a path with parentheses is read as a whole. +function stackPlaces(stack) { + return String(stack) + .split('\n') + .map(line => line.match(/^\s*at (?:.*? \()?(?:file:\/\/)?(.+?)(:\d+:\d+)\)?\s*$/)) + .filter(Boolean) + .map(([, file, place]) => ({file, place})); +} // The folder of RexReplace itself, from the first place in a stack made here const ownFolder = (() => { - const [place] = String(new Error().stack).matchAll(stackPlaces); - return place ? path.dirname(place[1]) + path.sep : null; + const [own] = stackPlaces(new Error().stack); + return own ? path.dirname(own.file) + path.sep : null; })(); // The first place in the stack of an error that is in a file of the user, like `helper.js:2:15`. // Only the places above the code of RexReplace are from the code of the user, so the search stops there. +// Places in Node itself, in eval and in node_modules are skipped. Reading the stack can throw, so it is tried. function stackPlace(thrown) { - const stack = 'object' === typeof thrown && null !== thrown ? thrown.stack : ''; - for (const [, file, place] of String(stack ?? '').matchAll(stackPlaces)) { - if (ownFolder && file.startsWith(ownFolder)) { - return ''; - } - if (!file.includes('node:')) { - return path.relative(process.cwd(), file) + place; + try { + for (const {file, place} of stackPlaces(thrown?.stack ?? '')) { + if (ownFolder && file.startsWith(ownFolder)) { + return ''; + } + // A file is an absolute path, so the frames of code run with eval are skipped too + if (/^(\/|[A-Za-z]:[\\/])/.test(file) && !file.includes('node_modules')) { + return path.relative(process.cwd(), file) + place; + } } - } + } catch (e) {} return ''; } diff --git a/test/fix/824.test.js b/test/fix/824.test.js index 02f74383..ae2d4bc9 100644 --- a/test/fix/824.test.js +++ b/test/fix/824.test.js @@ -19,4 +19,30 @@ describe('#824 --js code that fails in a module', () => { expect((await rr('o', 'r("helper")($0)', config)).err).not.toContain(' at '); expect((await rr('o', 'nope()', {...config, verbose: true})).err).not.toContain(' at '); }); + + test('reads a path with parentheses as a whole', async () => { + fs.mkdirSync('a (copy)'); + fs.writeFileSync( + 'a (copy)/helper.js', + 'module.exports = value => {\n\treturn value.nope.deeper;\n};\n' + ); + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + expect((await rr('o', 'r("a (copy)/helper.js")($0)', config)).err).toMatch( + /\n {4}at a \(copy\)\/helper\.js:2:\d+\n/ + ); + }); + + test('does not tell places in the runtime itself', async () => { + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + expect((await rr('o', 'new URL("x")', config)).err).not.toContain(' at '); + }); + + test('does not crash when the stack can not be read', async () => { + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + const result = await rr('o', 'throw new Proxy({}, {get() { throw 1; }})', config); + expect(result.err).toContain( + 'The code of --js failed for the piped data: an object was thrown' + ); + expect(result.exit).toBe(3); + }); }); From 2ad9cd282d30f5a2350aed5e5819a297bf5aafca Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:23:04 +1100 Subject: [PATCH 09/29] Only make the --js numbers the code uses, and only count files for $i and $c --- src/engine.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index d6945541..bbf5edb6 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -132,6 +132,9 @@ async function run(config) { config.replacementOri = config.replacement; + // $i and $c need one more search of each file, so the files are only counted when the code uses them + config.countFiles = config.replacementJs && /\$[ic](?![A-Za-z])/.test(config.replacementOri); + // The most specific delimiter wins. Without one there is a line break after each item, unless it already ends with one. const delimiter = config.delimiter ?? (config.nullOut ? '\0' : null); config.nameDelimiter = config.nameDelimiter ?? delimiter; @@ -784,8 +787,12 @@ function fileNumbers(config, number, total) { return {n: number, t: total, i: config.numbers.matched + 1, c: config.numbers.changed + 1}; } -// When a file is done, it counts for $i if the pattern found something, and for $c if it was changed +// When a file is done, it counts for $i if the pattern found something, and for $c if it was changed. +// It takes one more search of the file, so it is only done when the code uses $i or $c. function countFile(config, text, result) { + if (!config.countFiles) { + return; + } if (-1 !== text.search(config.regex)) { config.numbers.matched++; } @@ -805,8 +812,10 @@ function usesNamedGroups(config) { // Code that makes the numbers and their cousins, put in front of the code of --js. // $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, and the ones with _ at the end have a space at the end. -function numbersCode(numbers) { +// Only the numbers the code uses are made, as each one costs time when the code runs once per match. +function numbersCode(numbers, code) { return Object.entries(numbers) + .filter(([name]) => new RegExp('\\$' + name + '(?![A-Za-z])').test(code)) .map(([name, value]) => { const cousins = [`$${name} = ${value}`, `$${name}_ = '${value} '`]; for (let width = 2; width <= 10; width++) { @@ -893,7 +902,7 @@ function dynamicReplacement( const _pipe = _config_rr.pipedData, _text = _data_rr, _find = _config_rr.pattern, - code_rr = numbersCode(_numbers_rr) + _config_rr.replacementOri, + code_rr = numbersCode(_numbers_rr, _config_rr.replacementOri) + _config_rr.replacementOri, _cwd = process.cwd(), _now = _time, _ = ' ', @@ -1122,7 +1131,10 @@ function dynamicReplacement( _match_rr++; _config_rr.numbers.matches++; - var capturedGroups = numbersCode({m: _match_rr, M: _config_rr.numbers.matches}); + var capturedGroups = numbersCode( + {m: _match_rr, M: _config_rr.numbers.matches}, + _config_rr.replacementOri + ); // The match and the groups are followed by the offset and the full text, and by the named groups if the pattern has any const named = arguments[arguments.length - 1]; const hasNamed = 'object' === typeof named && null !== named; From 71b1a5141414b6394eedc0a673858769fd3c3c73 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:15:17 +1100 Subject: [PATCH 10/29] Tell in the help that only the numbers the code names are made --- README.md | 2 +- src/cli.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ad9f9239..0d9ac876 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A number is only made when the code names it, so a name built when the code runs, like `eval('$' + 'n')`, is not there. A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 6d49a484..5dd977a2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -409,7 +409,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f \`mtime_obj\`: date object representing \`mtime\`, \`ctime_obj\`: date object representing \`ctime\`. -The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). +The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). A number is only made when the code names it, so a name built when the code runs, like \`eval('$' + 'n')\`, is not there. A named group is a variable with \`_\` in front: \`_year\` for \`(?...)\`, and \`_year_\` has a space at the end. From 1b890fe89ddf344214fa9c1adf7dd1e7bc7b6502 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:26:25 +1100 Subject: [PATCH 11/29] Leave the README line of --js to #838, so the stack can be rebased without a conflict --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d9ac876..ad9f9239 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A number is only made when the code names it, so a name built when the code runs, like `eval('$' + 'n')`, is not there. A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | From 07dbd4d23faf5e6900d6a9c882cc37d493ba1069 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:25:00 +1100 Subject: [PATCH 12/29] Let --js code declare the names of the numbers and groups with let and const --- src/engine.ts | 5 ++++- test/fix/815.test.js | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 test/fix/815.test.js diff --git a/src/engine.ts b/src/engine.ts index bbf5edb6..1ebacddf 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -902,7 +902,10 @@ function dynamicReplacement( const _pipe = _config_rr.pipedData, _text = _data_rr, _find = _config_rr.pattern, - code_rr = numbersCode(_numbers_rr, _config_rr.replacementOri) + _config_rr.replacementOri, + // The code of the user is in a block of its own, so it can declare the same names with let and const. + // The value of the block is the value of its last statement, like before. The line break ends a comment at the end. + code_rr = + numbersCode(_numbers_rr, _config_rr.replacementOri) + '{' + _config_rr.replacementOri + '\n}', _cwd = process.cwd(), _now = _time, _ = ' ', diff --git a/test/fix/815.test.js b/test/fix/815.test.js new file mode 100644 index 00000000..95941206 --- /dev/null +++ b/test/fix/815.test.js @@ -0,0 +1,20 @@ +// https://github.com/mathiasrw/rexreplace/issues/815 +// The numbers and the named groups were made with var in front of the code, so code that declared the same name with let or const did not run. +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +describe('#815 --js code can declare its own variables', () => { + test.each([ + ['a number with let', 'o', 'let $t = "T"; $t', 'fTT'], + ['a group with const', '(o)', 'const $1 = "X"; $1', 'fXX'], + ['a named group with let', '(?o)', 'let _o = "N"; _o', 'fNN'], + ])('with the name of %s', async (name, pattern, code, out) => { + expect((await rr(pattern, code, {pipedData: 'foo', replacementJs: true})).out).toBe(out); + }); + + test('the value is still the value of the last statement, also with a comment at the end', async () => { + expect( + (await rr('o', 'const a = 1; a + 1 // two', {pipedData: 'foo', replacementJs: true})).out + ).toBe('f22'); + }); +}); From f7b634f5f31194ab5d14735711d8da0bbcfe3480 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:26:59 +1100 Subject: [PATCH 13/29] Use the prefix rr_ for named groups in --js, and panic on a name that ends with _ --- README.md | 2 +- src/cli.ts | 2 +- src/engine.ts | 25 ++++++++++++++++++++----- test/fix/799.test.js | 8 ++++---- test/fix/815.test.js | 2 +- test/fix/825.test.js | 26 ++++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 test/fix/825.test.js diff --git a/README.md b/README.md index ad9f9239..d5311a9e 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `_` in front: `_year` for `(?...)`, and `_year_` has a space at the end. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `rr_` in front: `rr_year` for `(?...)`, and `rr_year_` has a space at the end. A named group can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index 5dd977a2..ac0067e4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -411,7 +411,7 @@ The following values defaults to \`❌\` if haystack does not originate from a f The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). A number is only made when the code names it, so a name built when the code runs, like \`eval('$' + 'n')\`, is not there. -A named group is a variable with \`_\` in front: \`_year\` for \`(?...)\`, and \`_year_\` has a space at the end. +A named group is a variable with \`rr_\` in front: \`rr_year\` for \`(?...)\`, and \`rr_year_\` has a space at the end. A named group can not end with \`_\`. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 1ebacddf..d0ac52b7 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -145,6 +145,16 @@ async function run(config) { config.regex = getFinalRegex(config.pattern, config) || ''; + // In --js a named group is a variable like rr_year, and rr_year_ has a space at the end. + // A group that ends with _ would have the same name as another group with a space, so it can not be used. + const endsWith_ = + config.replacementJs && config.regex + ? namedGroups(config.regex).find(name => name.endsWith('_')) + : null; + if (endsWith_) { + panic('A named group can not end with _:', endsWith_); + } + // What the regex engine is asked to do. With --replacement-js and --output-match it gets a function. if (config.statusInfo.engine) { config.statusInfo.engine.execution = { @@ -801,15 +811,20 @@ function countFile(config, text, result) { } } -// The code uses a named group of the pattern, like _year for (?…) +// The code uses a named group of the pattern, like rr_year for (?…) function usesNamedGroups(config) { - return [...config.regex.source.matchAll(/\(\?<([A-Za-z_$][\w$]*)>/g)].some(([, name]) => - new RegExp('(^|[^\\w$])_' + name.replace(/\$/g, '\\$') + '(?![\\w$])').test( + return namedGroups(config.regex).some(name => + new RegExp('(^|[^\\w$])rr_' + name.replace(/\$/g, '\\$') + '(?![\\w$])').test( config.replacementOri ) ); } +// The names of the named groups of a regex, like year for (?…) +function namedGroups(regex) { + return [...regex.source.matchAll(/\(\?<([A-Za-z_$][\w$]*)>/g)].map(([, name]) => name); +} + // Code that makes the numbers and their cousins, put in front of the code of --js. // $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, and the ones with _ at the end have a space at the end. // Only the numbers the code uses are made, as each one costs time when the code runs once per match. @@ -1144,10 +1159,10 @@ function dynamicReplacement( for (var i = 0; i < arguments.length - (hasNamed ? 3 : 2); i++) { capturedGroups += 'var $' + i + '=' + JSON.stringify(arguments[i]) + '; '; } - // A named group is a variable with _ in front, like _year for (?…), and _year_ has a space at the end + // A named group is a variable with rr_ in front, like rr_year for (?…), and rr_year_ has a space at the end if (hasNamed) { for (const [name, value] of Object.entries(named)) { - capturedGroups += `var _${name} = ${JSON.stringify(value)}, _${name}_ = _${name} + ' '; `; + capturedGroups += `var rr_${name} = ${JSON.stringify(value)}, rr_${name}_ = rr_${name} + ' '; `; } } diff --git a/test/fix/799.test.js b/test/fix/799.test.js index f4f4f568..d7c4ddf8 100644 --- a/test/fix/799.test.js +++ b/test/fix/799.test.js @@ -9,16 +9,16 @@ describe('#799 groups in --js', () => { expect((await rr('(?a)(b)', '$1 + $2 + typeof $3', config)).out).toBe('abundefined'); }); - test('a named group is a variable with _ in front, and _ at the end has a space', async () => { + test('a named group is a variable with rr_ in front, and _ at the end has a space', async () => { const config = {pipedData: '2024-05 1999-12', replacementJs: true}; const pattern = '(?\\d{4})-(?\\d\\d)'; - expect((await rr(pattern, '_month + "/" + _year', config)).out).toBe('05/2024 12/1999'); - expect((await rr(pattern, '_month_ + _year', config)).out).toBe('05 2024 12 1999'); + expect((await rr(pattern, 'rr_month + "/" + rr_year', config)).out).toBe('05/2024 12/1999'); + expect((await rr(pattern, 'rr_month_ + rr_year', config)).out).toBe('05 2024 12 1999'); }); test('a named group that took no part in the match is undefined', async () => { const config = {pipedData: 'ab', replacementJs: true}; - expect((await rr('(?a)|(?b)', 'String(_x) + String(_y) + ","', config)).out).toBe( + expect((await rr('(?a)|(?b)', 'String(rr_x) + String(rr_y) + ","', config)).out).toBe( 'aundefined,undefinedb,' ); }); diff --git a/test/fix/815.test.js b/test/fix/815.test.js index 95941206..efb3413b 100644 --- a/test/fix/815.test.js +++ b/test/fix/815.test.js @@ -7,7 +7,7 @@ describe('#815 --js code can declare its own variables', () => { test.each([ ['a number with let', 'o', 'let $t = "T"; $t', 'fTT'], ['a group with const', '(o)', 'const $1 = "X"; $1', 'fXX'], - ['a named group with let', '(?o)', 'let _o = "N"; _o', 'fNN'], + ['a named group with let', '(?o)', 'let rr_o = "N"; rr_o', 'fNN'], ])('with the name of %s', async (name, pattern, code, out) => { expect((await rr(pattern, code, {pipedData: 'foo', replacementJs: true})).out).toBe(out); }); diff --git a/test/fix/825.test.js b/test/fix/825.test.js new file mode 100644 index 00000000..c7c1901c --- /dev/null +++ b/test/fix/825.test.js @@ -0,0 +1,26 @@ +// https://github.com/mathiasrw/rexreplace/issues/825 +// A named group was a variable like _year, which could clash: with groups a and a_, _a_ was both a_ and a with a space, +// and a group named _require_ broke r(). +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +describe('#825 named groups in --js', () => { + test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { + const config = {pipedData: 'ab', replacementJs: true}; + expect((await rr('(?a)', 'rr_require + typeof r', config)).out).toBe('afunctionb'); + }); + + test('can not end with _, as that is the name of another group with a space', async () => { + expect(await rr('(?\\d)(?\\w)', 'rr_a', {pipedData: '1x', replacementJs: true})).toEqual( + { + out: '', + err: ' ❌ A named group can not end with _: a_\n No further processing.\n\n', + exit: 3, + } + ); + }); + + test('can end with _ without --js, as there are no variables then', async () => { + expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); + }); +}); From c3fcc3604eb71c105045f09573bcffa3ee1c101d Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:17:32 +1100 Subject: [PATCH 14/29] Do not panic on a group name that ends with _ under --output-match --- src/engine.ts | 3 ++- test/fix/825.test.js | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index d0ac52b7..676d97d3 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -147,8 +147,9 @@ async function run(config) { // In --js a named group is a variable like rr_year, and rr_year_ has a space at the end. // A group that ends with _ would have the same name as another group with a space, so it can not be used. + // With --output-match the code is not run, so there are no variables. const endsWith_ = - config.replacementJs && config.regex + config.replacementJs && !config.outputMatch && config.regex ? namedGroups(config.regex).find(name => name.endsWith('_')) : null; if (endsWith_) { diff --git a/test/fix/825.test.js b/test/fix/825.test.js index c7c1901c..58bb3614 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -6,8 +6,10 @@ import {rr} from '../helper.js'; describe('#825 named groups in --js', () => { test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { + // With _ in front, the group _require with a space was __require_, which r() uses itself. + // $1 makes the code run once per match, where the variables of the groups are made. const config = {pipedData: 'ab', replacementJs: true}; - expect((await rr('(?a)', 'rr_require + typeof r', config)).out).toBe('afunctionb'); + expect((await rr('(?<_require>a)', '$1 + r("path").sep', config)).out).toBe('a/b'); }); test('can not end with _, as that is the name of another group with a space', async () => { @@ -20,6 +22,11 @@ describe('#825 named groups in --js', () => { ); }); + test('can end with _ with --output-match, as the code is not run', async () => { + const config = {pipedData: 'ab', replacementJs: true, outputMatch: true}; + expect((await rr('(?a)', 'x', config)).exit).toBe(null); + }); + test('can end with _ without --js, as there are no variables then', async () => { expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); }); From 9e521417978c15d6c934976910a8e1974fa8067b Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:28:37 +1100 Subject: [PATCH 15/29] Find named groups with letters that are not ASCII in --js --- src/engine.ts | 13 ++++++++----- test/fix/816.test.js | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 test/fix/816.test.js diff --git a/src/engine.ts b/src/engine.ts index 676d97d3..ca19caaf 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -815,15 +815,18 @@ function countFile(config, text, result) { // The code uses a named group of the pattern, like rr_year for (?…) function usesNamedGroups(config) { return namedGroups(config.regex).some(name => - new RegExp('(^|[^\\w$])rr_' + name.replace(/\$/g, '\\$') + '(?![\\w$])').test( - config.replacementOri - ) + new RegExp( + '(^|[^\\p{ID_Continue}$])rr_' + name.replace(/\$/g, '\\$') + '(?![\\p{ID_Continue}$])', + 'u' + ).test(config.replacementOri) ); } -// The names of the named groups of a regex, like year for (?…) +// The names of the named groups of a regex, like year for (?…). A name can have any letter, like año. function namedGroups(regex) { - return [...regex.source.matchAll(/\(\?<([A-Za-z_$][\w$]*)>/g)].map(([, name]) => name); + return [ + ...regex.source.matchAll(/\(\?<([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)>/gu), + ].map(([, name]) => name); } // Code that makes the numbers and their cousins, put in front of the code of --js. diff --git a/test/fix/816.test.js b/test/fix/816.test.js new file mode 100644 index 00000000..286b6142 --- /dev/null +++ b/test/fix/816.test.js @@ -0,0 +1,19 @@ +// https://github.com/mathiasrw/rexreplace/issues/816 +// A named group with a letter that is not ASCII was not seen in the code, so the code ran once per file, where the group was not there. +import {describe, test, expect} from 'bun:test'; +import {rr} from '../helper.js'; + +describe('#816 named groups with letters that are not ASCII', () => { + test.each([ + ['año', '(?\\d)', '"[" + rr_año + "]"', 'a[1]b[2]'], + ['日', '(?<日>\\d)', 'rr_日 * 2', 'a2b4'], + ])('can be used, like %s', async (name, pattern, code, out) => { + expect((await rr(pattern, code, {pipedData: 'a1b2', replacementJs: true})).out).toBe(out); + }); + + test('can not end with _ either', async () => { + const result = await rr('(?x)', 'rr_año', {pipedData: 'x', replacementJs: true}); + expect(result.err).toStartWith(' ❌ A named group can not end with _: año_'); + expect(result.exit).toBe(3); + }); +}); From 6cb7e62a656390c8c154a8917c2d9904aeab26e3 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:19:17 +1100 Subject: [PATCH 16/29] Read the named groups from the regex, and see rr_year_ on its own --- src/engine.ts | 10 ++++++---- test/fix/816.test.js | 11 +++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index ca19caaf..66edb195 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -816,17 +816,19 @@ function countFile(config, text, result) { function usesNamedGroups(config) { return namedGroups(config.regex).some(name => new RegExp( - '(^|[^\\p{ID_Continue}$])rr_' + name.replace(/\$/g, '\\$') + '(?![\\p{ID_Continue}$])', + // rr_year_ is the group with a space at the end, so the code uses the group then too + '(^|[^\\p{ID_Continue}$])rr_' + name.replace(/\$/g, '\\$') + '_?(?![\\p{ID_Continue}$])', 'u' ).test(config.replacementOri) ); } // The names of the named groups of a regex, like year for (?…). A name can have any letter, like año. +// They come from the regex itself, so text that only looks like a group, like \(? or [(?], is not one. +// With | at the end the regex matches an empty text, and the match has all the groups. function namedGroups(regex) { - return [ - ...regex.source.matchAll(/\(\?<([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)>/gu), - ].map(([, name]) => name); + const match = new RegExp(regex.source + '|', regex.flags.replace(/[gy]/g, '')).exec(''); + return Object.keys(match?.groups ?? {}); } // Code that makes the numbers and their cousins, put in front of the code of --js. diff --git a/test/fix/816.test.js b/test/fix/816.test.js index 286b6142..11cde209 100644 --- a/test/fix/816.test.js +++ b/test/fix/816.test.js @@ -16,4 +16,15 @@ describe('#816 named groups with letters that are not ASCII', () => { expect(result.err).toStartWith(' ❌ A named group can not end with _: año_'); expect(result.exit).toBe(3); }); + + test('are read from the regex, so text that only looks like a group is not one', async () => { + const config = {replacementJs: true}; + expect((await rr('\\(?b', '"z"', {...config, pipedData: 'x(?b'})).out).toBe('x(?z'); + expect((await rr('[(?]', '"z"', {...config, pipedData: 'ab'})).out).toBe('zb'); + }); + + test('can be used with a space at the end on its own, like rr_year_', async () => { + const config = {pipedData: '2024', replacementJs: true}; + expect((await rr('(?\\d+)', 'rr_year_ + "|"', config)).out).toBe('2024 |'); + }); }); From f736bf43c7e8abc3792bc85e99eb7126e55dabe4 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:30:22 +1100 Subject: [PATCH 17/29] Use the path of --replacement-file as it is, with no escapes and no aliases --- src/alias.ts | 5 ++++- src/cli.ts | 3 ++- test/fix/817.test.js | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 test/fix/817.test.js diff --git a/src/alias.ts b/src/alias.ts index e872b519..8eace72c 100644 --- a/src/alias.ts +++ b/src/alias.ts @@ -15,7 +15,10 @@ export function alias(config) { }; config.pattern = swap(config.pattern); - config.replacement = swap(config.replacement); + // With --replacement-file the replacement is the name of a file, unless --js code gives the name + if (!config.replacementFile || config.replacementJs) { + config.replacement = swap(config.replacement); + } config.excludeRe = config.excludeRe.map(swap); config.delimiter = swap(config.delimiter); config.nameDelimiter = swap(config.nameDelimiter); diff --git a/src/cli.ts b/src/cli.ts index ac0067e4..d9626a88 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -624,7 +624,8 @@ const unescape = (key, name) => { } }; -if (!config.replacementJs) { +// The path of --replacement-file is the name of a file, so it has no escapes +if (!config.replacementJs && !config.replacementFile) { unescape('replacement', 'replacement'); } diff --git a/test/fix/817.test.js b/test/fix/817.test.js new file mode 100644 index 00000000..35d54fdc --- /dev/null +++ b/test/fix/817.test.js @@ -0,0 +1,32 @@ +// https://github.com/mathiasrw/rexreplace/issues/817 +// The path of --replacement-file went through the escapes and the € and § aliases of a typed replacement, +// so a backslash in a Windows path was lost, and price€.txt was looked for as price$.txt. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {$} from 'bun'; +import {cli, sh} from '../helper.js'; + +describe('#817 the path of --replacement-file', () => { + test('keeps a backslash', async () => { + fs.mkdirSync('my\\dir'); + fs.writeFileSync('my\\dir/new.txt', 'NEW'); + fs.writeFileSync('page.txt', 'X'); + const result = await sh($`bun ${cli} X ${'my\\dir/new.txt'} page.txt -o --replacement-file`); + expect(result).toEqual({out: 'NEW\n', err: 'page.txt\n', exit: 0}); + }); + + test('keeps € and §, as the aliases are not used in file names', async () => { + fs.writeFileSync('price€§.txt', 'PRICE'); + fs.writeFileSync('page.txt', 'X'); + const result = await sh($`bun ${cli} X ${'price€§.txt'} page.txt -o --replacement-file`); + expect(result.out).toBe('PRICE\n'); + }); + + test('still uses the aliases in the code of --js, which gives the path', async () => { + fs.writeFileSync('new.txt', 'NEW'); + fs.writeFileSync('page.txt', 'X'); + const code = '"new" + €0.replace("X", ".txt")'; + const result = await sh($`bun ${cli} X ${code} page.txt -o --replacement-file --js`); + expect(result.out).toBe('NEW\n'); + }); +}); From d74c21360883e712830446b8cbb4c0e123decc9c Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:32:14 +1100 Subject: [PATCH 18/29] Tell why a replacement file can not be read, and only once --- src/engine.ts | 17 +++++++++++------ test/engine/replacement-file.test.js | 4 ++-- test/fix/818.test.js | 26 ++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 test/fix/818.test.js diff --git a/src/engine.ts b/src/engine.ts index 66edb195..0098b011 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -669,13 +669,18 @@ function getFinalReplacement(replacement, conf: any) { try { return readReplacementFile(replacement, conf); } catch (e) { - return panic('The replacement file can not be read:', replacement); + return panic('The replacement file can not be read:', unreadable(replacement, e)); } } return replacement; } +// The path as JSON, so a line break or a space at the end can be seen, and the reason, like ENOENT +function unreadable(filepath, e) { + return JSON.stringify(filepath) + ' (' + (e.code || e.message) + ')'; +} + // Each replacement file is read once, also when the code of --js gives the same path for many matches function readReplacementFile(filepath, conf) { conf.replacementFiles ??= new Map(); @@ -693,16 +698,16 @@ function codeReplacement(file, config, text, numbers) { if (!config.replacementFile) { return code; } - // A path that can not be read is an error once, and its matches are left as they are - const failed = new Set(); + // A path that can not be read is an error once in the run, and its matches are left as they are + config.unreadableFiles ??= new Set(); return (...args) => { const filepath = String(code(...args)); - if (!failed.has(filepath)) { + if (!config.unreadableFiles.has(filepath)) { try { return readReplacementFile(filepath, config); } catch (e) { - failed.add(filepath); - error('The replacement file can not be read:', filepath); + config.unreadableFiles.add(filepath); + error('The replacement file can not be read:', unreadable(filepath, e)); } } return args[0]; diff --git a/test/engine/replacement-file.test.js b/test/engine/replacement-file.test.js index 121bccd2..9d3c9854 100644 --- a/test/engine/replacement-file.test.js +++ b/test/engine/replacement-file.test.js @@ -59,7 +59,7 @@ describe('--replacement-file', () => { test('that can not be read stops the run before any file is changed', async () => { fs.writeFileSync('my.file', 'foo'); const result = await rr('foo', 'nope.txt', {includeGlob: ['my.file'], replacementFile: true}); - expect(result.err).toStartWith(' ❌ The replacement file can not be read: nope.txt'); + expect(result.err).toStartWith(' ❌ The replacement file can not be read: "nope.txt" (ENOENT)'); expect(result.exit).toBe(3); expect(fs.readFileSync('my.file', 'utf8')).toBe('foo'); }); @@ -68,7 +68,7 @@ describe('--replacement-file', () => { const config = {pipedData: 'foo', replacementJs: true, replacementFile: true}; expect(await rr('o', '"nope.txt"', config)).toEqual({ out: 'foo', - err: ' ❌ The replacement file can not be read: nope.txt\n\n', + err: ' ❌ The replacement file can not be read: "nope.txt" (ENOENT)\n\n', exit: 2, }); }); diff --git a/test/fix/818.test.js b/test/fix/818.test.js new file mode 100644 index 00000000..160ca3a6 --- /dev/null +++ b/test/fix/818.test.js @@ -0,0 +1,26 @@ +// https://github.com/mathiasrw/rexreplace/issues/818 +// A replacement file that could not be read gave no reason, and with --js the error came once for each file. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr, rrStatus} from '../helper.js'; + +describe('#818 a replacement file that can not be read', () => { + test.each([ + ['is not there', 'nope.txt', '"nope.txt" (ENOENT)'], + ['is a folder', 'adir', '"adir" (EISDIR)'], + ['has a line break at the end, like from echo', 'a.txt\n', '"a.txt\\n" (ENOENT)'], + ])('tells the reason when the path %s', async (name, filepath, text) => { + fs.mkdirSync('adir'); + fs.writeFileSync('a.txt', 'X'); + const result = await rr('X', filepath, {includeGlob: ['a.txt'], replacementFile: true}); + expect(result.err).toStartWith(' ❌ The replacement file can not be read: ' + text); + }); + + test('with --js is told once in the run, not once for each file', async () => { + fs.writeFileSync('a.txt', 'X'); + fs.writeFileSync('b.txt', 'X'); + const config = {includeGlob: ['*.txt'], replacementJs: true, replacementFile: true}; + const status = await rrStatus('X', '"nope.txt"', config); + expect(status.errors).toEqual(['The replacement file can not be read: "nope.txt" (ENOENT)']); + }); +}); From 02e974e6a17de1dfb98638ca2776c3ee2a5904fd Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:21:08 +1100 Subject: [PATCH 19/29] Show the reason in words, and the path as it is unless it hides a char --- src/engine.ts | 8 ++++++-- test/engine/replacement-file.test.js | 6 ++++-- test/fix/818.test.js | 19 +++++++++++++++---- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index 0098b011..895c378e 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -676,9 +676,13 @@ function getFinalReplacement(replacement, conf: any) { return replacement; } -// The path as JSON, so a line break or a space at the end can be seen, and the reason, like ENOENT +// The path as it is, and the reason, like `nope.txt (ENOENT: no such file or directory)`. +// A path that starts or ends with a space, or has a control char, like a line break from echo, is shown as JSON, so it can be seen. function unreadable(filepath, e) { - return JSON.stringify(filepath) + ' (' + (e.code || e.message) + ')'; + const shown = /^\s|\s$|[\x00-\x1f\x7f]/.test(filepath) ? JSON.stringify(filepath) : filepath; + // The message ends with the call and the path, like `, open 'nope.txt'`, which is already told + const reason = String(e.message || e.code).replace(/, \w+ '.*'$/s, ''); + return shown + ' (' + reason + ')'; } // Each replacement file is read once, also when the code of --js gives the same path for many matches diff --git a/test/engine/replacement-file.test.js b/test/engine/replacement-file.test.js index 9d3c9854..19abca35 100644 --- a/test/engine/replacement-file.test.js +++ b/test/engine/replacement-file.test.js @@ -59,7 +59,9 @@ describe('--replacement-file', () => { test('that can not be read stops the run before any file is changed', async () => { fs.writeFileSync('my.file', 'foo'); const result = await rr('foo', 'nope.txt', {includeGlob: ['my.file'], replacementFile: true}); - expect(result.err).toStartWith(' ❌ The replacement file can not be read: "nope.txt" (ENOENT)'); + expect(result.err).toStartWith( + ' ❌ The replacement file can not be read: nope.txt (ENOENT: no such file or directory)' + ); expect(result.exit).toBe(3); expect(fs.readFileSync('my.file', 'utf8')).toBe('foo'); }); @@ -68,7 +70,7 @@ describe('--replacement-file', () => { const config = {pipedData: 'foo', replacementJs: true, replacementFile: true}; expect(await rr('o', '"nope.txt"', config)).toEqual({ out: 'foo', - err: ' ❌ The replacement file can not be read: "nope.txt" (ENOENT)\n\n', + err: ' ❌ The replacement file can not be read: nope.txt (ENOENT: no such file or directory)\n\n', exit: 2, }); }); diff --git a/test/fix/818.test.js b/test/fix/818.test.js index 160ca3a6..a5b97e98 100644 --- a/test/fix/818.test.js +++ b/test/fix/818.test.js @@ -6,9 +6,18 @@ import {rr, rrStatus} from '../helper.js'; describe('#818 a replacement file that can not be read', () => { test.each([ - ['is not there', 'nope.txt', '"nope.txt" (ENOENT)'], - ['is a folder', 'adir', '"adir" (EISDIR)'], - ['has a line break at the end, like from echo', 'a.txt\n', '"a.txt\\n" (ENOENT)'], + ['is not there', 'nope.txt', 'nope.txt (ENOENT: no such file or directory)'], + ['is a folder', 'adir', 'adir (EISDIR: illegal operation on a directory, read)'], + [ + 'has a line break at the end, like from echo', + 'a.txt\n', + '"a.txt\\n" (ENOENT: no such file or directory)', + ], + [ + 'has a backslash, like a Windows path', + 'my\\dir/nope.txt', + 'my\\dir/nope.txt (ENOENT: no such file or directory)', + ], ])('tells the reason when the path %s', async (name, filepath, text) => { fs.mkdirSync('adir'); fs.writeFileSync('a.txt', 'X'); @@ -21,6 +30,8 @@ describe('#818 a replacement file that can not be read', () => { fs.writeFileSync('b.txt', 'X'); const config = {includeGlob: ['*.txt'], replacementJs: true, replacementFile: true}; const status = await rrStatus('X', '"nope.txt"', config); - expect(status.errors).toEqual(['The replacement file can not be read: "nope.txt" (ENOENT)']); + expect(status.errors).toEqual([ + 'The replacement file can not be read: nope.txt (ENOENT: no such file or directory)', + ]); }); }); From 24879e8300801af504ad4fac1682f183af39ba36 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:33:56 +1100 Subject: [PATCH 20/29] Sort a file that is gone before the sort last, and do not crash --- src/engine.ts | 18 ++++++++++++++---- test/fix/819.test.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 test/fix/819.test.js diff --git a/src/engine.ts b/src/engine.ts index 895c378e..04e62eec 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1310,8 +1310,17 @@ function sortFiles(files, sort, conf) { return files.sort(ascii); } - // By a number from each file, and when two are the same, by ascii - const stats = new Map(files.map(file => [file, fs.statSync(file)])); + // By a number from each file, and when two are the same, by ascii. + // A file that can not be looked at, like one deleted since it was found, is sorted last, and is told about when it is opened. + const stats = new Map( + files.map(file => { + try { + return [file, fs.statSync(file)]; + } catch (e) { + return [file, null]; + } + }) + ); let key = { modified: stat => stat.mtimeMs, created: stat => stat.birthtimeMs, @@ -1319,14 +1328,15 @@ function sortFiles(files, sort, conf) { }[sort]; // Not all file systems keep the time a file was made. Then Node gives 0. - if ('created' === sort && [...stats.values()].some(stat => !stat.birthtimeMs)) { + if ('created' === sort && [...stats.values()].some(stat => stat && !stat.birthtimeMs)) { warn( 'The file system does not keep the time a file was made, so --sort created uses the modified time.' ); key = stat => stat.mtimeMs; } - return files.sort((a, b) => key(stats.get(a)) - key(stats.get(b)) || ascii(a, b)); + const value = file => (stats.get(file) ? key(stats.get(file)) : Infinity); + return files.sort((a, b) => value(a) - value(b) || ascii(a, b)); } // A path that can not be looked at (like a name that is too long) is not a file, so it is globbed diff --git a/test/fix/819.test.js b/test/fix/819.test.js new file mode 100644 index 00000000..9d4816d7 --- /dev/null +++ b/test/fix/819.test.js @@ -0,0 +1,31 @@ +// https://github.com/mathiasrw/rexreplace/issues/819 +// With --sort modified, created or size, a file that was gone between the glob and the sort crashed the run with a stack trace. +import {describe, test, expect, spyOn} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#819 a file that is gone before the sort', () => { + test.each([['modified'], ['created'], ['size']])( + 'is sorted last with --sort %s, and the run goes on', + async sort => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('gone.md', 'foo'); + fs.writeFileSync('z.md', 'foo'); + // The sort looks at gone.md as if it was deleted after the glob found it + const statSync = fs.statSync; + const spy = spyOn(fs, 'statSync').mockImplementation((file, options) => { + if ('gone.md' === file && !options) { + throw Object.assign(new Error('ENOENT: no such file or directory'), {code: 'ENOENT'}); + } + return statSync(file, options); + }); + try { + const result = await rr('foo', 'bar', {includeGlob: ['*.md'], sort}); + expect(result.out).toBe('a.md\nz.md\ngone.md\n'); + expect(result.exit).toBe(null); + } finally { + spy.mockRestore(); + } + } + ); +}); From 870b6911cb4e856c0b5ff54edd6f71d5208cbd91 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:22:50 +1100 Subject: [PATCH 21/29] Only sort a deleted file last, and give other stat errors their reason --- src/engine.ts | 22 ++++++++++-------- test/fix/819.test.js | 53 +++++++++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index 04e62eec..b691d475 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1311,16 +1311,20 @@ function sortFiles(files, sort, conf) { } // By a number from each file, and when two are the same, by ascii. - // A file that can not be looked at, like one deleted since it was found, is sorted last, and is told about when it is opened. - const stats = new Map( - files.map(file => { - try { - return [file, fs.statSync(file)]; - } catch (e) { - return [file, null]; + // A file deleted since it was found is sorted last, and is told about as not found when it is opened. + // A file that can not be looked at for another reason is an error with that reason, and is left out. + const stats = new Map(); + files = files.filter(file => { + try { + stats.set(file, fs.statSync(file)); + } catch (e) { + if ('ENOENT' !== e.code) { + return error(e.message); } - }) - ); + stats.set(file, null); + } + return true; + }); let key = { modified: stat => stat.mtimeMs, created: stat => stat.birthtimeMs, diff --git a/test/fix/819.test.js b/test/fix/819.test.js index 9d4816d7..632d1d60 100644 --- a/test/fix/819.test.js +++ b/test/fix/819.test.js @@ -4,28 +4,49 @@ import {describe, test, expect, spyOn} from 'bun:test'; import fs from 'fs'; import {rr} from '../helper.js'; -describe('#819 a file that is gone before the sort', () => { +// The sort looks at each file with statSync and no options. When it looks at the given file, the file is made to fail. +function failInSort(name, fail) { + const statSync = fs.statSync; + return spyOn(fs, 'statSync').mockImplementation((file, options) => { + if (name === file && !options) { + fail(); + } + return statSync(file, options); + }); +} + +describe('#819 a file that can not be looked at in the sort', () => { test.each([['modified'], ['created'], ['size']])( - 'is sorted last with --sort %s, and the run goes on', + 'when deleted, is sorted last with --sort %s, and told about as not found', async sort => { - fs.writeFileSync('a.md', 'foo'); - fs.writeFileSync('gone.md', 'foo'); - fs.writeFileSync('z.md', 'foo'); - // The sort looks at gone.md as if it was deleted after the glob found it - const statSync = fs.statSync; - const spy = spyOn(fs, 'statSync').mockImplementation((file, options) => { - if ('gone.md' === file && !options) { - throw Object.assign(new Error('ENOENT: no such file or directory'), {code: 'ENOENT'}); - } - return statSync(file, options); - }); + ['a.md', 'gone.md', 'z.md'].forEach(name => fs.writeFileSync(name, 'foo')); + const spy = failInSort('gone.md', () => fs.unlinkSync('gone.md')); try { - const result = await rr('foo', 'bar', {includeGlob: ['*.md'], sort}); - expect(result.out).toBe('a.md\nz.md\ngone.md\n'); - expect(result.exit).toBe(null); + expect(await rr('foo', 'bar', {includeGlob: ['*.md'], sort})).toEqual({ + out: 'a.md\nz.md\n', + err: ' ❌ File not found: gone.md\n\n', + exit: 2, + }); } finally { spy.mockRestore(); } } ); + + test('for another reason, is an error with that reason, and is left out', async () => { + ['a.md', 'b.md'].forEach(name => fs.writeFileSync(name, 'foo')); + const spy = failInSort('b.md', () => { + throw Object.assign(new Error('EACCES: permission denied'), {code: 'EACCES'}); + }); + try { + expect(await rr('foo', 'bar', {includeGlob: ['*.md'], sort: 'size'})).toEqual({ + out: 'a.md\n', + err: ' ❌ EACCES: permission denied\n\n', + exit: 2, + }); + expect(fs.readFileSync('b.md', 'utf8')).toBe('foo'); + } finally { + spy.mockRestore(); + } + }); }); From 16161a482a4036b8b4bdda408cb073e7bf3df391 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:35:44 +1100 Subject: [PATCH 22/29] Make the natural order the same on Node and Bun, and in every locale --- src/engine.ts | 5 ++++- test/fix/820.test.js | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 test/fix/820.test.js diff --git a/src/engine.ts b/src/engine.ts index b691d475..3407cfd6 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1295,7 +1295,10 @@ function sortName(value) { return name; } -const natural = new Intl.Collator(undefined, {numeric: true, sensitivity: 'base'}).compare; +// The same order everywhere: a fixed locale, as Node follows the locale of the user and Bun does not, +// and ascii for the names natural sees as the same, like file1 and file01, so the order of the glob does not decide +const collator = new Intl.Collator('en', {numeric: true, sensitivity: 'base'}).compare; +const natural = (a, b) => collator(a, b) || ascii(a, b); const ascii = (a, b) => (a < b ? -1 : a > b ? 1 : 0); function sortFiles(files, sort, conf) { diff --git a/test/fix/820.test.js b/test/fix/820.test.js new file mode 100644 index 00000000..989f14c9 --- /dev/null +++ b/test/fix/820.test.js @@ -0,0 +1,17 @@ +// https://github.com/mathiasrw/rexreplace/issues/820 +// Names that natural sees as the same kept the order of the glob, which differs between Node and Bun, +// and natural followed the locale of the user on Node, but not on Bun. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('#820 the natural order', () => { + test('orders the names it sees as the same by ascii', async () => { + for (const name of ['file1.md', 'file01.md', 'file001.md', 'cafe.md', 'café.md']) { + fs.writeFileSync(name, 'foo'); + } + expect((await rr('foo', 'bar', {includeGlob: ['*.md'], simulate: true})).out).toBe( + 'cafe.md\ncafé.md\nfile001.md\nfile01.md\nfile1.md\n' + ); + }); +}); From 9c69a0d9ae8301bd3cf4b21a295806a450b4a919 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:24:11 +1100 Subject: [PATCH 23/29] Run the test of the natural order with --sort --- test/fix/820.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/fix/820.test.js b/test/fix/820.test.js index 989f14c9..70a93b25 100644 --- a/test/fix/820.test.js +++ b/test/fix/820.test.js @@ -10,8 +10,8 @@ describe('#820 the natural order', () => { for (const name of ['file1.md', 'file01.md', 'file001.md', 'cafe.md', 'café.md']) { fs.writeFileSync(name, 'foo'); } - expect((await rr('foo', 'bar', {includeGlob: ['*.md'], simulate: true})).out).toBe( - 'cafe.md\ncafé.md\nfile001.md\nfile01.md\nfile1.md\n' - ); + expect( + (await rr('foo', 'bar', {includeGlob: ['*.md'], simulate: true, sort: 'natural'})).out + ).toBe('cafe.md\ncafé.md\nfile001.md\nfile01.md\nfile1.md\n'); }); }); From 29146e2eacf5c39bae87287a59bc77ced3645765 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 23:37:51 +1100 Subject: [PATCH 24/29] Make the README and the help true about --quiet, --debug, --simulate, --sort and more --- README.md | 16 ++++++++-------- src/cli.ts | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d5311a9e..32da1395 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ Use --js-pipe to access the pipe data in your javascript code. #### Templates -With `--replacement-file` the replacement is the path to a file, and the content of the file goes in. `--trim-file` takes away the line break at the end of the file. +With `--replacement-file` the replacement is the path to a file, and the content of the file goes in. `--trim-file` trims the content, like the line break at the end of the file. ```bash > rexreplace '\{\{footer\}\}' partials/footer.html 'site/*.html' --replacement-file --trim-file @@ -316,7 +316,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-v` | **`--version`** Print rexreplace version (can be given as only argument) [boolean] | | `-V` | **`--verbose`** More chatty output [boolean] | | `-L` | **`--literal`** Literal string search (no regex used when searching) [boolean] | -| `-l` | **`--literal-replacement`** Use the replacement as it is. Without it, `$1`, `$&` and the other `$` signs in the replacement mean the groups and the match, as in javascript. [boolean] | +| `-l` | **`--literal-replacement`** The `$` signs in the replacement have no meaning, so `$1` and `$&` are just text. Without it they mean the groups and the match, as in javascript. The `€` and `§` aliases and escapes like `\n` are part of what is typed, and are still used. [boolean] | | | **`--replacement-file`** The replacement is the path to a file, and the content of the file is used as the replacement. With --js the code gives the path. [boolean] | | `-I` | **`--void-ignore-case`** Void case insensitive search. Applies to the pattern and to all globs. [boolean] | | `-G` | **`--void-global`** Void global search (stop looking after the first match). [boolean] | @@ -348,8 +348,8 @@ If planned renames clash, the process is halted and no change is applied to any | `-A` | **`--void-async`** Handle files in a synchronous flow. Good to limit memory usage when handling large files. [boolean] | | `-B` | **`--void-backup`** Avoid temporary backing up file. Works async (independent of -A flag) and will speed up things but at one point data lives only in memory, and you will lose the content if the process is interrupted. [boolean] | | `-b` | **`--keep-backup`** Keep a backup file of the original content. With --rename the old file is kept. [boolean] | -| `-S` | **`--simulate`** Execute everything during a run except changes to the file system. Names of affected files and content are still printed as in a usual run. [boolean] | -| | **`--sort`** Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (alphabetic), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order. [string] [choices: "natural", "ascii", "modified", "created", "size", "none"] | +| `-S` | **`--simulate`** Execute everything during a run except the changes RexReplace would normally make to the file system. Code from --js still runs, and can change data even with --simulate. [boolean] | +| | **`--sort`** Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (by char code, so `B` comes before `a`), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order. [string] [choices: "natural", "ascii", "modified", "created", "size", "none"] | | | **`--sort-reverse`** Turn the order of the files around. [boolean] | | `-F` | **`--expect-files`** Check that at least one file is found. The exit code is 1 if not. [boolean] | | `-z` | **`--expect-diff`** Check that at least one file is changed. The exit code is 1 if not. [boolean] | @@ -473,7 +473,7 @@ a_b #### The replacement -1. The replacement as typed, with `€` and `§` as above. Or the piped data with `--replacement-pipe` (`-R`), used as it is. +1. The replacement as typed: `€` is `$`, `§` is `\`, and escapes like `\n` and `\t` are read. Or the piped data with `--replacement-pipe` (`-R`), used as it is. 2. `--js` (`-j`) runs it as code, and the value of the code goes on. `--js-pipe` also gives the code the piped data as `pipe`. 3. `--replacement-file`: what came so far is a path, and the content of that file goes on. 4. `--trim-file` trims it. @@ -538,14 +538,14 @@ b.file 3. The tails: the delimiter also goes after the last item, unless `--no-name-delimiter-tail`, `--no-content-delimiter-tail` or `--no-tail`. Piped data that is printed is left as it came in, with nothing after it. ```bash -> rexreplace o x a.file b.file --name-delimiter ', ' --no-name-delimiter-tail +> rexreplace o x a.file b.file --name-delimiter ', ' --no-name-delimiter-tail --sort natural a.file, b.file > rexreplace o x a.file b.file --null-out | xargs -0 ls -l ``` 4. `--nl` ends stdout and stderr with a line break, if they do not end with one. -5. How much: `--quiet` (`-q`) shows only the errors (❌), and no names or warnings (🟡). `--quiet-total` (`-Q`) shows nothing, and `--verbose` (`-V`) tells each step to stderr. An error that stops the run ends with `No further processing.` -6. The status report: `--status` and `--status-compact` (`-c`) print it as the last thing, also with `--quiet-total`. `--debug` (`-d`) prints only the status report, with what went into the engine and what it did. +5. How much: `--quiet` (`-q`) hides the names and the warnings (🟡), but not the errors (❌), and not the content of `--output` or the matches of `--output-match`. `--quiet-total` (`-Q`) shows nothing, and `--verbose` (`-V`) tells each step to stderr. An error that stops the run ends with `No further processing.` +6. The status report: `--status` and `--status-compact` (`-c`) print it as the last thing, also with `--quiet-total`. `--debug` (`-d`) prints the status report, with what went into the engine and what it did, and hides the rest, except the content of `--output` and the matches of `--output-match`. 7. The exit code, see [Exit codes](#exit-codes). ### Other details diff --git a/src/cli.ts b/src/cli.ts index d9626a88..65c23dfa 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -78,7 +78,7 @@ const yargs = require('yargs') .boolean('l') .describe( 'l', - 'Use the replacement as it is. Without it, `$1`, `$&` and the other `$` signs in the replacement mean the groups and the match, as in javascript.' + 'The `$` signs in the replacement have no meaning, so `$1` and `$&` are just text. Without it they mean the groups and the match, as in javascript. The `€` and `§` aliases and escapes like `\\n` are part of what is typed, and are still used.' ) .alias('l', 'literal-replacement') @@ -261,14 +261,14 @@ const yargs = require('yargs') .boolean('S') .describe( 'S', - 'Execute everything during a run except changes to the file system. Names of affected files and content are still printed as in a usual run.' + 'Execute everything during a run except the changes RexReplace would normally make to the file system. Code from --js still runs, and can change data even with --simulate.' ) .alias('S', 'simulate') .string('sort') .describe( 'sort', - 'Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (alphabetic), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order.' + 'Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (by char code, so `B` comes before `a`), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order.' ) // Any case, and true and yes are natural .coerce('sort', value => { From c200a428ea55f0afed1b85a8b783fb5ae82bc14a Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:25:52 +1100 Subject: [PATCH 25/29] Make the README and help true about --quiet, --debug and the escapes --- README.md | 8 ++++---- devguide.md | 2 +- src/cli.ts | 5 ++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 32da1395..1b235f71 100644 --- a/README.md +++ b/README.md @@ -325,7 +325,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | | `-E` | **`--engine`** What regex engine to use. `default` is the engine of the runtime: `V8` with Node and `JSC` (JavaScriptCore) with Bun. The engine of the other runtime can not be picked. [choices: "default", "V8", "JSC"] [default: "default"] | -| `-q` | **`--quiet`** Only display errors (no other info) [boolean] | +| `-q` | **`--quiet`** Hide the names of the files and the warnings. Errors are still shown, and so is the content of --output and the matches of --output-match. [boolean] | | `-Q` | **`--quiet-total`** Never display errors or info [boolean] | | `-f` | **`--full-path`** Print the full path of the named files instead of the relative path. [boolean] | | | **`--name-all`** Print the names of all files found by the globs, also the ones where nothing was changed. [boolean] | @@ -405,7 +405,7 @@ Files identified for potential processing are identified by: - Per default `§` is treated as an alias for `\` in the pattern, the replacement and `--exclude-re`. The main reason is for you not to worry about escaping the escape of an escape. Use the `-§` flag if you need to search or replace the actual section char. -- The `€` and `§` aliases apply to pattern, replacement, `--exclude-re` values and delimiters. File globs, `--exclude-glob` and piped data are used exactly as they are, so a file named `price€.txt` is found without any flags. +- The `€` and `§` aliases apply to pattern, replacement, `--exclude-re` values and delimiters. File globs, `--exclude-glob`, the path of `--replacement-file` and piped data are used exactly as they are, so a file named `price€.txt` is found without any flags. - The default delimiter is a magic newline ensuring that there is _at least_ one `\n` after printed file content, matches or names. This means that if the content already ends on a `\n` no additional `\n` is added. This behaviour can not be recreated with other delimiter values. @@ -473,9 +473,9 @@ a_b #### The replacement -1. The replacement as typed: `€` is `$`, `§` is `\`, and escapes like `\n` and `\t` are read. Or the piped data with `--replacement-pipe` (`-R`), used as it is. +1. The replacement as typed: `€` is `$`, `§` is `\`, and escapes like `\n` and `\t` are read. In `--js` code javascript reads the escapes, and the path of `--replacement-file` is used as it is, with no aliases or escapes. Or the piped data with `--replacement-pipe` (`-R`), used as it is. 2. `--js` (`-j`) runs it as code, and the value of the code goes on. `--js-pipe` also gives the code the piped data as `pipe`. -3. `--replacement-file`: what came so far is a path, and the content of that file goes on. +3. `--replacement-file`: what came so far is the path of a file, and the content of the file goes on. 4. `--trim-file` trims it. 5. It goes into the text. `$1` is group 1, `$&` the whole match, and `$$` one `$`, unless the value came from `--js` code, or `--literal-replacement` (`-l`) is given. diff --git a/devguide.md b/devguide.md index b27fc433..4ba7bae8 100644 --- a/devguide.md +++ b/devguide.md @@ -29,7 +29,7 @@ All output goes through `src/output.ts`. Pick the function by who the message is | `chat(msg)` | `--verbose`, also with `--quiet`, but not with `--quiet-total` | The user: what happens to each file. Only ever to stderr. Also kept in `engine.steps` of the status report. | `Open sync: a.md`, `Nothing changed in: a.md` | - What the run did is in the status report: the engine resolves with it, and `--status` prints it. So there is no extra debug output to print along the way. -- `--debug` prints only the status report, with what went into the engine and what it did. +- `--debug` prints the status report, with what went into the engine and what it did, and hides the rest, except the content of `--output` and the matches of `--output-match`. - `warn()`, `error()`, `checkFailed()` and `panic()` add to the status report as they go, also when the quiet flags hide what they print. - The exit code is 0 when the run is done, 1 when a check from `--expect-*` is not met, 2 when a file gave an error, and 3 for a panic or wrong flags. See Exit codes in the README. diff --git a/src/cli.ts b/src/cli.ts index 65c23dfa..8db8bca3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -146,7 +146,10 @@ const yargs = require('yargs') .default('E', 'default') .boolean('q') - .describe('q', 'Only display errors (no other info)') + .describe( + 'q', + 'Hide the names of the files and the warnings. Errors are still shown, and so is the content of --output and the matches of --output-match.' + ) .alias('q', 'quiet') .boolean('Q') From d618a25026ef1a1503e631b4e1a6b02f2d7fab12 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:26:35 +1100 Subject: [PATCH 26/29] Tell in the README that only the --js numbers the code names are made --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b235f71..15f265b0 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A named group is a variable with `rr_` in front: `rr_year` for `(?...)`, and `rr_year_` has a space at the end. A named group can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A number is only made when the code names it, so a name built when the code runs, like `eval('$' + 'n')`, is not there. A named group is a variable with `rr_` in front: `rr_year` for `(?...)`, and `rr_year_` has a space at the end. A named group can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | From 2533c6cb81011e4900066e58a0a6b974a8bb3bac Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 00:46:44 +1100 Subject: [PATCH 27/29] End the run as usual when the reader of a pipe stops early --- src/cli.ts | 6 +++--- src/output.ts | 25 ++++++++++++++++++++++++- test/fix/839.test.js | 24 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 test/fix/839.test.js diff --git a/src/cli.ts b/src/cli.ts index 8db8bca3..cf22585f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,7 +1,7 @@ // CLI interface for rexreplace import * as rexreplace from './engine.ts'; -import {outputConfig, panic} from './output.ts'; +import {outputConfig, panic, writeTo} from './output.ts'; import {alias} from './alias.ts'; // Make it visible when Bun is the runtime (like with rexreplace-bun) as the regex engine is then not the one from Node. @@ -526,9 +526,9 @@ Z z // The CLI is the only part that acts on the process. function printStatus(config, status) { if (config.status) { - process.stdout.write('\n' + JSON.stringify(status, null, 2) + '\n'); + writeTo(process.stdout, '\n' + JSON.stringify(status, null, 2) + '\n'); } else if (config.statusCompact) { - process.stdout.write(JSON.stringify(status) + '\n'); + writeTo(process.stdout, JSON.stringify(status) + '\n'); } } diff --git a/src/output.ts b/src/output.ts index 3b77f490..147f46cf 100644 --- a/src/output.ts +++ b/src/output.ts @@ -23,9 +23,32 @@ export class Panic extends Error { status = null; } +// A pipe the reader has closed, like `| head`, can not take more. That is fine: nobody reads the rest. +// Bun throws when it is written to, and Node gives an error event, so both are caught. +const closed = new Set(); +const whenClosed = stream => e => { + if ('EPIPE' !== e?.code) { + throw e; + } + closed.add(stream); +}; +process.stdout.on?.('error', whenClosed(process.stdout)); +process.stderr.on?.('error', whenClosed(process.stderr)); + +export const writeTo = function (stream, text) { + if (closed.has(stream)) { + return; + } + try { + stream.write(text); + } catch (e) { + whenClosed(stream)(e); + } +}; + // Everything is printed from here, so the last character of each stream is known const write = function (text, stream) { - stream.write(text); + writeTo(stream, text); if (text) { lastPrintedChar.set(stream, text.charCodeAt(text.length - 1)); } diff --git a/test/fix/839.test.js b/test/fix/839.test.js new file mode 100644 index 00000000..23af8e68 --- /dev/null +++ b/test/fix/839.test.js @@ -0,0 +1,24 @@ +// https://github.com/mathiasrw/rexreplace/issues/839 +// When the reader of a pipe stopped early, like head, the run crashed with EPIPE and a stack trace. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {$} from 'bun'; +import {cli, sh} from '../helper.js'; + +describe('#839 a pipe the reader closes early', () => { + test('ends the run as usual, with no error', async () => { + fs.writeFileSync('big.txt', Array.from({length: 200000}, (_, i) => `${i} foo`).join('\n')); + expect(await sh($`bun ${cli} foo bar big.txt -o | head -1`)).toEqual({ + out: '0 bar\n', + err: 'big.txt\n', + exit: 0, + }); + }); + + test('also when the status report comes after the reader is gone', async () => { + fs.writeFileSync('big.txt', Array.from({length: 200000}, (_, i) => `${i} foo`).join('\n')); + const result = await sh($`bun ${cli} foo bar big.txt -o -d | head -1`); + expect(result.out).toBe('0 bar\n'); + expect(result.err).toBe(''); + }); +}); From c9079a44b360529eb37448ea3089aee3425e0039 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 03:19:54 +1100 Subject: [PATCH 28/29] Take in the newest commits of the PRs below, so the cleanup is made on the code that will be merged --- README.md | 12 +-- src/cli.ts | 15 ++-- src/engine.ts | 210 +++++++++++++++++++++++++++++-------------- src/output.ts | 20 +++-- test/cli/run.sh | 12 ++- test/fix/788.test.js | 64 +++++++++++-- test/fix/789.test.js | 9 +- test/fix/790.test.js | 26 ++++++ test/fix/808.test.js | 8 +- test/fix/813.test.js | 53 +++++++++++ test/fix/815.test.js | 3 + test/fix/816.test.js | 11 +++ test/fix/818.test.js | 8 +- test/fix/819.test.js | 105 ++++++++++++++++------ test/fix/820.test.js | 41 +++++++-- test/fix/822.test.js | 25 ++++-- test/fix/824.test.js | 12 ++- test/fix/825.test.js | 51 +++++++++-- test/fix/826.test.js | 88 +++++++++++++++--- 19 files changed, 601 insertions(+), 172 deletions(-) create mode 100644 test/fix/813.test.js diff --git a/README.md b/README.md index 15f265b0..5b7120fc 100644 --- a/README.md +++ b/README.md @@ -316,7 +316,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-v` | **`--version`** Print rexreplace version (can be given as only argument) [boolean] | | `-V` | **`--verbose`** More chatty output [boolean] | | `-L` | **`--literal`** Literal string search (no regex used when searching) [boolean] | -| `-l` | **`--literal-replacement`** The `$` signs in the replacement have no meaning, so `$1` and `$&` are just text. Without it they mean the groups and the match, as in javascript. The `€` and `§` aliases and escapes like `\n` are part of what is typed, and are still used. [boolean] | +| `-l` | **`--literal-replacement`** No "smart" values in the replacement, so you can have text like `$1` and `$&` without escaping them. [boolean] | | | **`--replacement-file`** The replacement is the path to a file, and the content of the file is used as the replacement. With --js the code gives the path. [boolean] | | `-I` | **`--void-ignore-case`** Void case insensitive search. Applies to the pattern and to all globs. [boolean] | | `-G` | **`--void-global`** Void global search (stop looking after the first match). [boolean] | @@ -325,7 +325,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-u` | **`--unicode`** Treat pattern as a sequence of unicode code points. [boolean] | | `-e` | **`--encoding`** Encoding of files/piped data. [default: "utf8"] | | `-E` | **`--engine`** What regex engine to use. `default` is the engine of the runtime: `V8` with Node and `JSC` (JavaScriptCore) with Bun. The engine of the other runtime can not be picked. [choices: "default", "V8", "JSC"] [default: "default"] | -| `-q` | **`--quiet`** Hide the names of the files and the warnings. Errors are still shown, and so is the content of --output and the matches of --output-match. [boolean] | +| `-q` | **`--quiet`** Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown. [boolean] | | `-Q` | **`--quiet-total`** Never display errors or info [boolean] | | `-f` | **`--full-path`** Print the full path of the named files instead of the relative path. [boolean] | | | **`--name-all`** Print the names of all files found by the globs, also the ones where nothing was changed. [boolean] | @@ -348,14 +348,14 @@ If planned renames clash, the process is halted and no change is applied to any | `-A` | **`--void-async`** Handle files in a synchronous flow. Good to limit memory usage when handling large files. [boolean] | | `-B` | **`--void-backup`** Avoid temporary backing up file. Works async (independent of -A flag) and will speed up things but at one point data lives only in memory, and you will lose the content if the process is interrupted. [boolean] | | `-b` | **`--keep-backup`** Keep a backup file of the original content. With --rename the old file is kept. [boolean] | -| `-S` | **`--simulate`** Execute everything during a run except the changes RexReplace would normally make to the file system. Code from --js still runs, and can change data even with --simulate. [boolean] | -| | **`--sort`** Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (by char code, so `B` comes before `a`), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order. [string] [choices: "natural", "ascii", "modified", "created", "size", "none"] | +| `-S` | **`--simulate`** Execute everything during a run except changes RexReplace would normally make to the file system. Code from --js still runs, and can change data even with --simulate. [boolean] | +| | **`--sort`** Process files sync in a sorted order chosen from 6 values: `natural` (numbers compared as numbers and case insensitive), `ascii` (by char code, so `B` comes before `a`), `modified`, `created`, `size`, or `none` (the order of the input: files first, then files found by globs). Files sorted the same (like the same size) are ordered by ascii sort. Without --sort files are processed asynchronously, so output order is not guaranteed, but the processing starts in natural order. [string] [choices: "natural", "ascii", "modified", "created", "size", "none"] | | | **`--sort-reverse`** Turn the order of the files around. [boolean] | | `-F` | **`--expect-files`** Check that at least one file is found. The exit code is 1 if not. [boolean] | | `-z` | **`--expect-diff`** Check that at least one file is changed. The exit code is 1 if not. [boolean] | | `-Z` | **`--expect-zero-diff`** Check that no file is changed. The exit code is 1 if one is. [boolean] | | | **`--expect-diff-all`** Check that all files found are changed. The exit code is 1 if not. [boolean] | -| `-r` | **`--rename`** Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. Control characters, like a line break, are removed from the new name. [boolean] | +| `-r` | **`--rename`** Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. The process will halt before renaming begins if old or new names contain ASCII control characters, end on a space or a dot, or consist only of spaces and dots. [boolean] | | | **`--rename-path`** Like --rename, but on the whole path of each file, so a / in the replacement moves the file. Folders that are not there are made. [boolean] | | | **`--rename-force`** With --rename or --rename-path, write over a file that is there already. [boolean] | | `-m` | **`--output-match`** Output each match on a new line. Will not replace any content but you still need to provide a dummy value (like `_`) as replacement parameter. The replacement is not used, so --js code is not run. If search pattern does not contain matching groups the full match will be outputted. If search pattern does contain matching groups only matching groups will be outputted (same line with no delimiter). [boolean] | @@ -366,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). A number is only made when the code names it, so a name built when the code runs, like `eval('$' + 'n')`, is not there. A named group is a variable with `rr_` in front: `rr_year` for `(?...)`, and `rr_year_` has a space at the end. A named group can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). For speed, only the numbers the code names are made, and the code runs once per file unless it names the match, a group, `$m` or `$M`. This is found with a simple text search, so it can be fooled, like by `eval('$' + 'n')`. Access a named group via the name prepended with `rr_`. `(?...)` gives you `rr_year`. A named group name can not end with `_`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index cf22585f..086483b3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -78,7 +78,7 @@ const yargs = require('yargs') .boolean('l') .describe( 'l', - 'The `$` signs in the replacement have no meaning, so `$1` and `$&` are just text. Without it they mean the groups and the match, as in javascript. The `€` and `§` aliases and escapes like `\\n` are part of what is typed, and are still used.' + 'No "smart" values in the replacement, so you can have text like `$1` and `$&` without escaping them.' ) .alias('l', 'literal-replacement') @@ -148,7 +148,7 @@ const yargs = require('yargs') .boolean('q') .describe( 'q', - 'Hide the names of the files and the warnings. Errors are still shown, and so is the content of --output and the matches of --output-match.' + 'Hide file names and warnings. Errors are still shown, and output from flags like --output and --output-match is still shown.' ) .alias('q', 'quiet') @@ -264,14 +264,14 @@ const yargs = require('yargs') .boolean('S') .describe( 'S', - 'Execute everything during a run except the changes RexReplace would normally make to the file system. Code from --js still runs, and can change data even with --simulate.' + 'Execute everything during a run except changes RexReplace would normally make to the file system. Code from --js still runs, and can change data even with --simulate.' ) .alias('S', 'simulate') .string('sort') .describe( 'sort', - 'Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (by char code, so `B` comes before `a`), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order.' + 'Process files sync in a sorted order chosen from 6 values: `natural` (numbers compared as numbers and case insensitive), `ascii` (by char code, so `B` comes before `a`), `modified`, `created`, `size`, or `none` (the order of the input: files first, then files found by globs). Files sorted the same (like the same size) are ordered by ascii sort. Without --sort files are processed asynchronously, so output order is not guaranteed, but the processing starts in natural order.' ) // Any case, and true and yes are natural .coerce('sort', value => { @@ -303,7 +303,7 @@ const yargs = require('yargs') .boolean('r') .describe( 'r', - 'Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. Control characters, like a line break, are removed from the new name.' + 'Search and replace in the name of each file, and rename the file to the result. The folder and the content are not touched. If planned renames clash the process is halted and no change is applied to any file. The process will halt before renaming begins if old or new names contain ASCII control characters, end on a space or a dot, or consist only of spaces and dots.' ) .alias('r', 'rename') @@ -380,7 +380,6 @@ The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. -If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: \`r\` as an alias for \`require\` with both expanded to understand a relative path even if it is not starting with \`./\`, @@ -412,9 +411,9 @@ The following values defaults to \`❌\` if haystack does not originate from a f \`mtime_obj\`: date object representing \`mtime\`, \`ctime_obj\`: date object representing \`ctime\`. -The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). A number is only made when the code names it, so a name built when the code runs, like \`eval('$' + 'n')\`, is not there. +The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). For speed, only the numbers the code names are made, and the code runs once per file unless it names the match, a group, \`$m\` or \`$M\`. This is found with a simple text search, so it can be fooled, like by \`eval('$' + 'n')\`. -A named group is a variable with \`rr_\` in front: \`rr_year\` for \`(?...)\`, and \`rr_year_\` has a space at the end. A named group can not end with \`_\`. +Access a named group via the name prepended with \`rr_\`. \`(?...)\` gives you \`rr_year\`. A named group name can not end with \`_\`. All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 3407cfd6..523fb7a2 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -33,6 +33,13 @@ const re = { // The numbers of #791 that count what came before: $i, $c, $m and $M with their cousins, like $i00 or $M_ runNumbers: /\$[icmM](?![A-Za-z])/, matchNumbers: /\$[mM](?![A-Za-z])/, + // A path that is not clear as it is printed: a space at the start or the end, or a char that can not be seen, + // like a line break from echo or a zero-width space + unclearPath: /^\s|\s$|[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2060-\u2064\ufeff]/, + // The chars of unclearPath that JSON.stringify does not escape + invisibleChars: /[\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2060-\u2064\ufeff]/g, + // The end of the message of a failed file call, like `, open 'nope.txt'` + fileCallInMessage: /, \w+( '.*')?$/s, }; // Only the version ends up in the bundle @@ -53,6 +60,8 @@ function validateEncoding(encoding = 'utf8') { // When the run can not go on, a panic is thrown with the status so far. export async function engine(config: any = {engine: 'default'}) { config.statusInfo = newStatus(config); + // A new run has had no panic yet, also when the same config is used again + config.stopped = false; outputConfig(config); try { await run(config); @@ -258,7 +267,12 @@ async function run(config) { } }; - await Promise.all(Array.from({length: workers}, worker)); + // Wait for every file in work before the first error is thrown, so no file is cut off halfway and left only as a backup + const results = await Promise.allSettled(Array.from({length: workers}, worker)); + const failed = results.find(result => 'rejected' === result.status); + if (failed) { + throw failed.reason; + } } function openFile(file, config, number, total) { @@ -383,6 +397,8 @@ function updated(file, config) { function renameFiles(files, config) { // With --rename a / in the new name would move the file const slashed = []; + // The new name of each file, before it is joined with its folder + const newNames = new Map(); const plan = files.map((file, index) => { config.statusInfo.files.searched.push(file); const text = config.renamePath ? file : path.basename(file); @@ -391,20 +407,43 @@ function renameFiles(files, config) { config.replacementJs && !config.outputMatch ? codeReplacement(file, config, text, fileNumbers(config, index + 1, files.length)) : config.replacement; - const result = withoutControlChars(text.replace(config.regex, replacement), file); + const result = text.replace(config.regex, replacement); countFile(config, text, result); if (result === text) { return {old: file, new: file}; } if (config.renamePath) { - return {old: file, new: result}; + const item = {old: file, new: result}; + newNames.set(item, path.basename(result)); + return item; } const item = {old: file, new: path.join(path.dirname(file), result)}; + newNames.set(item, result); if (/[\\/]/.test(result)) { slashed.push(item); } return item; }); + // Names that can not be made. Each one is a clash of its own, and is left out of the other checks. + const invalid = []; + // A path with a control character is not welcome, also when the pattern would not change it + plan.forEach(item => { + if (/[\x00-\x1f\x7f]/.test(item.old + item.new)) { + invalid.push([ + 'Names can not have control characters, like a line break or a tab.' + + (config.replacementFile + ? ' Use --trim-file for the line break at the end of a file.' + : ''), + item, + ]); + } else if (item.new !== item.old && /^[. ]*$|[. ]$/.test(newNames.get(item))) { + // Like an empty name, . or .., which are folders. Windows does not allow a space or a dot at the end. + invalid.push([ + 'A name must have more than dots and spaces, and can not end with a space or a dot.', + item, + ]); + } + }); const renames = plan.filter(item => item.new !== item.old); // With --output-match the matches are printed by the replacement, and the names go to stderr. @@ -415,7 +454,7 @@ function renameFiles(files, config) { ); } - findClashes(plan, renames, slashed, config); + findClashes(plan, renames, slashed, invalid, config); plan.forEach(item => { if (item.new === item.old) { @@ -432,15 +471,6 @@ function renameFiles(files, config) { }); } -// No name should have the chars below space in ASCII (like a line break, a tab or a null byte), or DEL -function withoutControlChars(name, file) { - const clean = name.replace(/[\x00-\x1f\x7f]/g, ''); - if (clean !== name) { - chat('Removed control characters from the new name of: ' + file); - } - return clean; -} - // With --simulate and --output the old name goes to stderr and the new one to stdout, so they come in pairs in a terminal function nameRename(item, config) { if (config.simulate || config.output) { @@ -454,9 +484,13 @@ function nameRename(item, config) { // and the case is ignored, as `A.md` and `a.md` are the same file on a file system that is case-insensitive. const pathKey = file => path.resolve(file).toLowerCase(); -function findClashes(plan, renames, slashed, config) { +function findClashes(plan, renames, slashed, invalid, config) { const byTarget = new Map(); + const left = new Set(invalid.map(([, item]) => item)); renames.forEach(item => { + if (left.has(item)) { + return; + } const target = pathKey(item.new); byTarget.set(target, (byTarget.get(target) || []).concat(item)); }); @@ -469,9 +503,12 @@ function findClashes(plan, renames, slashed, config) { const clash = (title, items) => title + items.map(item => '\n From: ' + item.old).join('') + '\n To: ' + items[0].new; - const clashes = slashed.map(item => - clash('"/" is not allowed in names. Use --rename-path to move files.', [item]) - ); + const clashes = [ + ...invalid.map(([title, item]) => clash(title, [item])), + ...slashed.map(item => + clash('"/" is not allowed in names. Use --rename-path to move files.', [item]) + ), + ]; byTarget.forEach((items, target) => { const [item] = items; // The new path is another file, also when it differs from the old one only in case (on a file system that is case-sensitive) @@ -669,20 +706,25 @@ function getFinalReplacement(replacement, conf: any) { try { return readReplacementFile(replacement, conf); } catch (e) { - return panic('The replacement file can not be read:', unreadable(replacement, e)); + return panic('The replacement file can not be read:', pathAndReason(replacement, e)); } } return replacement; } -// The path as it is, and the reason, like `nope.txt (ENOENT: no such file or directory)`. -// A path that starts or ends with a space, or has a control char, like a line break from echo, is shown as JSON, so it can be seen. -function unreadable(filepath, e) { - const shown = /^\s|\s$|[\x00-\x1f\x7f]/.test(filepath) ? JSON.stringify(filepath) : filepath; - // The message ends with the call and the path, like `, open 'nope.txt'`, which is already told - const reason = String(e.message || e.code).replace(/, \w+ '.*'$/s, ''); - return shown + ' (' + reason + ')'; +// The path and why it can not be read, like `nope.txt (ENOENT: no such file or directory)`. +// A path that is not clear as it is printed is shown as JSON, with each char that can not be seen escaped. +function pathAndReason(filepath, e) { + let shownPath = filepath; + if (re.unclearPath.test(filepath)) { + shownPath = JSON.stringify(filepath).replace( + re.invisibleChars, + char => '\\u' + char.charCodeAt(0).toString(16).padStart(4, '0') + ); + } + // The call and the path at the end of the message are told already + return shownPath + ' (' + e.message.replace(re.fileCallInMessage, '') + ')'; } // Each replacement file is read once, also when the code of --js gives the same path for many matches @@ -711,7 +753,7 @@ function codeReplacement(file, config, text, numbers) { return readReplacementFile(filepath, config); } catch (e) { config.unreadableFiles.add(filepath); - error('The replacement file can not be read:', unreadable(filepath, e)); + error('The replacement file can not be read:', pathAndReason(filepath, e)); } } return args[0]; @@ -858,34 +900,40 @@ function numbersCode(numbers, code) { .join(''); } -// The places in a stack, one for each line, like `at fn (/path/file.js:2:15)` or `at /path/file.js:2:15`. -// The place is at the end of the line, so a path with parentheses is read as a whole. -function stackPlaces(stack) { - return String(stack) +// The stack trace of an error lists the calls that led to it, one line each, like `at fn (/path/file.js:2:15)` or `at /path/file.js:2:15`. +// These are the locations in it. The location is at the end of the line, so a path with parentheses is read as a whole. +function stackTraceLocations(stackTrace) { + return String(stackTrace) .split('\n') - .map(line => line.match(/^\s*at (?:.*? \()?(?:file:\/\/)?(.+?)(:\d+:\d+)\)?\s*$/)) + .map( + line => + // Without a function name the location comes right after at, like `at /dir/a (copy)/anon.js:1:59` + line.match(/^\s*at (?:file:\/\/)?((?:\/|[A-Za-z]:[\\/]).*?)(:\d+:\d+)\s*$/) || + // With a name the location is in parentheses at the end, like `at fn (/path/file.js:2:15)` + line.match(/^\s*at .*? \((?:file:\/\/)?(.+?)(:\d+:\d+)\)\s*$/) + ) .filter(Boolean) - .map(([, file, place]) => ({file, place})); + .map(([, file, position]) => ({file, position})); } -// The folder of RexReplace itself, from the first place in a stack made here +// The folder of RexReplace itself, from the first location in a stack trace made here const ownFolder = (() => { - const [own] = stackPlaces(new Error().stack); + const [own] = stackTraceLocations(new Error().stack); return own ? path.dirname(own.file) + path.sep : null; })(); -// The first place in the stack of an error that is in a file of the user, like `helper.js:2:15`. -// Only the places above the code of RexReplace are from the code of the user, so the search stops there. -// Places in Node itself, in eval and in node_modules are skipped. Reading the stack can throw, so it is tried. -function stackPlace(thrown) { +// The first location in the stack trace of an error that is in a file of the user, like `helper.js:2:15`. +// Only the locations above the code of RexReplace are from the code of the user, so the search stops there. +// Locations in Node itself, in eval and in node_modules are skipped. Reading the stack trace can throw, so it is tried. +function userCodeLocation(thrown) { try { - for (const {file, place} of stackPlaces(thrown?.stack ?? '')) { + for (const {file, position} of stackTraceLocations(thrown?.stack ?? '')) { if (ownFolder && file.startsWith(ownFolder)) { return ''; } // A file is an absolute path, so the frames of code run with eval are skipped too if (/^(\/|[A-Za-z]:[\\/])/.test(file) && !file.includes('node_modules')) { - return path.relative(process.cwd(), file) + place; + return path.relative(process.cwd(), file) + position; } } } catch (e) {} @@ -901,24 +949,31 @@ function thrownText(thrown) { if (text && '[object Object]' !== text) { return text; } - if ('object' === typeof thrown && null !== thrown) { - // Like an Error whose message can not be read: its name is still known - let name = ''; - try { - name = thrown.constructor?.name; - } catch (e) {} - if (name && 'Object' !== name) { - return 'a ' + name + ' was thrown'; - } - let json = ''; - try { - json = JSON.stringify(thrown); - } catch (e) {} - // The object can hold a whole file, so only the start of it is told - const start = json.length > 300 ? json.slice(0, 300) + '…' : json; - return 'an object was thrown' + (json && '{}' !== json ? ': ' + start : ''); + if ('object' !== typeof thrown || null === thrown) { + return 'an empty ' + typeof thrown + ' was thrown'; } - return 'an empty ' + typeof thrown + ' was thrown'; + // The name of the class, like an Error whose message can not be read, or a class of the user + let name = ''; + try { + name = String(thrown.constructor?.name ?? ''); + } catch (e) {} + let json = ''; + try { + json = JSON.stringify(thrown) ?? ''; + } catch (e) {} + const what = name && 'Object' !== name ? name : 'object'; + return cut( + (/^[aeiou]/i.test(what) ? 'an ' : 'a ') + + what + + ' was thrown' + + (json && '{}' !== json ? ': ' + json : '') + ); +} + +// The thrown value can hold a whole file, so only the start is told. It is cut by characters, so an emoji is not split. +function cut(text, length = 300) { + const chars = Array.from(text); + return chars.length > length ? chars.slice(0, length).join('') + '…' : text; } function dynamicReplacement( @@ -1033,10 +1088,13 @@ function dynamicReplacement( } catch (e) { const where = _config_rr.dataIsPiped ? 'the piped data' : _file_rr; // With --verbose it also tells where the code failed, when that is in a file of the user, like a module loaded with r() - const at = _config_rr.verbose ? stackPlace(e) : ''; + const at = _config_rr.verbose ? userCodeLocation(e) : ''; return panic( - 'The code of --js failed for ' + where + ':', - thrownText(e) + (at ? '\n at ' + at : '') + 'The code of --js failed for ' + + where + + '\n ' + + thrownText(e) + + (at ? '\n at ' + at : '') ); } }; @@ -1264,7 +1322,7 @@ function getFilePaths(conf) { // --exclude-glob is always globs, also for the name of a file. They are used together, and the files they find are taken away. if (excludeGlob.length) { - const excluded = new Set(glob(excludeGlob, globOptions).map(file => path.resolve(file))); + const excluded = new Set(glob(excludeGlob, globOptions, true).map(file => path.resolve(file))); files = files.filter(file => !excluded.has(path.resolve(file))); } @@ -1337,7 +1395,7 @@ function sortFiles(files, sort, conf) { // Not all file systems keep the time a file was made. Then Node gives 0. if ('created' === sort && [...stats.values()].some(stat => stat && !stat.birthtimeMs)) { warn( - 'The file system does not keep the time a file was made, so --sort created uses the modified time.' + 'This file system does not know when files were created. Sorting via modified time instead.' ); key = stat => stat.mtimeMs; } @@ -1346,21 +1404,34 @@ function sortFiles(files, sort, conf) { return files.sort((a, b) => value(a) - value(b) || ascii(a, b)); } -// A path that can not be looked at (like a name that is too long) is not a file, so it is globbed +// Is the path a file (true), something to glob (false), or a path that can not be looked at (null, with an error)? +// A path that is not there, or has a file where a folder should be, is globbed like before. So is a glob, +// like a long brace glob that is too long as a name. A plain path that can not be looked at, like a name +// that is too long or a symlink loop, is an error, so it is not hidden as 0 files found. function isFile(file) { try { return Boolean(fs.statSync(file, {throwIfNoEntry: false})?.isFile()); } catch (e) { - return false; + if ('ENOENT' === e.code || 'ENOTDIR' === e.code || globs.isDynamicPattern(file)) { + return false; + } + error(e.message); + return null; } } -// When the file system gives an error, the files can not be known, so the run can not go on -function glob(patterns, globOptions) { +// A glob that finds files and meets a folder that can not be read, like one with no access, is an error, +// and the run goes on with the files that can be found. A glob of --exclude-glob that fails is a panic, +// as the files it should take away can not be known, and they must not be changed. +function glob(patterns, globOptions, exclude = false) { try { return globs.sync(patterns, globOptions); } catch (e) { - return panic('The files can not be found:', e.message); + if (exclude) { + return panic('The files to exclude can not be found:', e.message); + } + error('Not all paths could be accessed:', e.message); + return globs.sync(patterns, {...globOptions, suppressErrors: true}); } } @@ -1370,9 +1441,10 @@ function findFiles(paths, globOptions, sort) { const files = []; const patterns = []; paths.forEach(file => { - if (isFile(file)) { + const kind = isFile(file); + if (kind) { files.push(file); - } else { + } else if (false === kind) { patterns.push(file); } }); diff --git a/src/output.ts b/src/output.ts index 147f46cf..6486632a 100644 --- a/src/output.ts +++ b/src/output.ts @@ -127,14 +127,18 @@ const printError = function (text, stop) { } }; -// The run can not go on, so it stops here +// The run can not go on, so it stops here. +// The files in work are finished, and they can fail too. Only the first panic is told, the rest are only in the status. export const panic = function (msg = '', data = '', displayHelp = false) { - if (displayHelp && !config.quietTotal) { - config.showHelp(); - } const text = msg + (data ? ' ' + data : ''); config.statusInfo?.errors.push(text); - printError(text, true); + if (!config.stopped) { + config.stopped = true; + if (displayHelp && !config.quietTotal) { + config.showHelp(); + } + printError(text, true); + } throw new Panic(text); }; @@ -149,8 +153,12 @@ export const checkFailed = function (text) { export const error = function (msg, data = '') { const text = msg + (data ? ' ' + data : ''); config.statusInfo?.errors.push(text); - printError(text, config.halt); + // After a panic only the status is told about the files that are finished + if (!config.stopped) { + printError(text, config.halt); + } if (config.halt) { + config.stopped = true; throw new Panic(text); } return false; diff --git a/test/cli/run.sh b/test/cli/run.sh index b26ad19c..88777bdf 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -55,11 +55,11 @@ assert_failure "rexreplace o-ho" # The exit code: 0 when done, 2 when a file gave an error and the run went on, 3 when the run stopped -# --halt (or --bail) stops at the first error, here that my.file can not be read +# --halt (or --bail) stops at the first error, here that my.file can not be read. With -A the files come one at a time, so your.file is not started. reset chmod 000 my.file -assert_raises "rexreplace b x my.file your.file --halt" 3 -assert_raises "rexreplace b x my.file your.file --bail" 3 +assert_raises "rexreplace b x my.file your.file --halt -A" 3 +assert_raises "rexreplace b x my.file your.file --bail -A" 3 assert "cat your.file" "abc123" assert_raises "rexreplace b x my.file your.file" 2 assert "cat your.file" "axc123" @@ -275,6 +275,12 @@ assert_raises "rexreplace 'oo' 'x' nope.file --expect-files" 1 reset assert "rexreplace 'b' 'x' my.file your.file --sort-reverse --name-delimiter ','" 'your.file,my.file,' +# The natural order is the same on Node and Bun, and in every locale. In Danish aa sorts after z. +reset +echo 'foo' > aab.file +echo 'foo' > b.file +assert "LC_ALL=da_DK.UTF-8 rexreplace 'foo' 'x' b.file aab.file --sort natural --simulate --name-delimiter ','" 'aab.file,b.file,' + # --full-path prints the full path of each name reset diff --git a/test/fix/788.test.js b/test/fix/788.test.js index b572225c..f98d03c2 100644 --- a/test/fix/788.test.js +++ b/test/fix/788.test.js @@ -16,14 +16,66 @@ describe('#788 a path that can not be looked at', () => { expect(fs.readFileSync('a.md', 'utf8')).toBe('bar'); }); + test('is an error when it is a symlink loop, and the other files are still changed', async () => { + fs.writeFileSync('a.md', 'foo'); + fs.symlinkSync('l2', 'l1'); + fs.symlinkSync('l1', 'l2'); + const result = await rr('foo', 'bar', {includeGlob: ['l1', 'a.md']}); + expect(result.err).toStartWith(' ❌ ELOOP'); + expect(result.exit).toBe(2); + expect(fs.readFileSync('a.md', 'utf8')).toBe('bar'); + }); + + test('is an error when the name is too long, and the other files are still changed', async () => { + fs.writeFileSync('a.md', 'foo'); + const result = await rr('foo', 'bar', {includeGlob: ['x'.repeat(300) + '.md', 'a.md']}); + expect(result.err).toStartWith(' ❌ ENAMETOOLONG'); + expect(result.exit).toBe(2); + expect(fs.readFileSync('a.md', 'utf8')).toBe('bar'); + }); + test.each([ - ['as an argument', {includeGlob: ['a.md/x']}], - ['from --glob-pipe', {pipedData: 'a.md/x', globPipe: true}], - ['in --exclude-glob', {includeGlob: ['a.md'], excludeGlob: ['a.md/x']}], - ])('gives a panic when the glob can not be used, %s', async (name, config) => { + ['as an argument', {includeGlob: ['b.md', 'a.md/x']}], + ['from --glob-pipe', {includeGlob: ['b.md'], pipedData: 'a.md/x', globPipe: true}], + ])( + 'is an error when the glob can not be used, %s, and the other files are still changed', + async (name, config) => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('b.md', 'foo'); + const result = await rr('foo', 'bar', config); + expect(result.err).toStartWith(' ❌ Not all paths could be accessed: ENOTDIR'); + expect(result.exit).toBe(2); + expect(fs.readFileSync('b.md', 'utf8')).toBe('bar'); + } + ); + + test('is an error in a folder that can not be read, and the other files of the glob are still changed', async () => { + fs.mkdirSync('locked'); + fs.writeFileSync('locked/b.md', 'foo'); + fs.writeFileSync('a.md', 'foo'); + fs.chmodSync('locked', 0); + try { + const result = await rr('foo', 'bar', {includeGlob: ['**/*.md']}); + expect(result.err).toStartWith(' ❌ Not all paths could be accessed: EACCES'); + expect(result.out).toBe('a.md\n'); + expect(result.exit).toBe(2); + expect(fs.readFileSync('a.md', 'utf8')).toBe('bar'); + } finally { + fs.chmodSync('locked', 0o755); + } + }); + + test('stops the run with --halt', async () => { + fs.writeFileSync('a.md', 'foo'); + const result = await rr('foo', 'bar', {includeGlob: ['a.md', 'a.md/x/*'], halt: true}); + expect(result.exit).toBe(3); + expect(fs.readFileSync('a.md', 'utf8')).toBe('foo'); + }); + + test('is a panic in --exclude-glob, as the files to keep can not be known', async () => { fs.writeFileSync('a.md', 'foo'); - const result = await rr('foo', 'bar', config); - expect(result.err).toStartWith(' ❌ The files can not be found: ENOTDIR'); + const result = await rr('foo', 'bar', {includeGlob: ['a.md'], excludeGlob: ['a.md/x']}); + expect(result.err).toStartWith(' ❌ The files to exclude can not be found: ENOTDIR'); expect(result.err).toEndWith('\n No further processing.\n\n'); expect(result.exit).toBe(3); expect(fs.readFileSync('a.md', 'utf8')).toBe('foo'); diff --git a/test/fix/789.test.js b/test/fix/789.test.js index ca8c6159..801d6a38 100644 --- a/test/fix/789.test.js +++ b/test/fix/789.test.js @@ -14,9 +14,12 @@ describe('#789 a panic with --halt', () => { fs.chmodSync(names[0], 0); const result = await rr('foo', 'bar', {includeGlob: ['*.txt'], voidAsync: false, halt: true}); expect(result.exit).toBe(3); - // The files that were already in work when the panic came are finished, but no new file is started + // The files that were already in work when the panic came are finished before the engine stops, and no new file is started + const changed = () => + names.slice(1).filter(name => 'bar' === fs.readFileSync(name, 'utf8')).length; + const atStop = changed(); + expect(atStop).toBeLessThan(workers); await Bun.sleep(300); - const changed = names.slice(1).filter(name => 'bar' === fs.readFileSync(name, 'utf8')); - expect(changed.length).toBeLessThan(workers); + expect(changed()).toBe(atStop); }); }); diff --git a/test/fix/790.test.js b/test/fix/790.test.js index c467024d..954b7116 100644 --- a/test/fix/790.test.js +++ b/test/fix/790.test.js @@ -18,4 +18,30 @@ describe('#790 the paths piped in with --glob-pipe', () => { expect(result).toEqual({out: 'æ.file\nb.file\n', err: '', exit: 0}); expect(fs.readFileSync('æ.file', 'utf8')).toBe('bar'); }); + + test('are read as UTF-8 for names in other scripts and with emoji', async () => { + const names = ['日本.file', '🦖.file', 'b.file']; + names.forEach(name => fs.writeFileSync(name, 'foo')); + const result = await sh( + $`bun ${cli} foo bar -g -e latin1 -A < ${new Response(names.join('\n'))}` + ); + expect(result.exit).toBe(0); + names.forEach(name => expect(fs.readFileSync(name, 'utf8')).toBe('bar')); + }); + + test('are read as UTF-8 also when the files are UTF-16', async () => { + fs.writeFileSync('æ.file', Buffer.from('foo', 'utf16le')); + const result = await sh($`bun ${cli} foo bar -g -e utf16le < ${new Response('æ.file\n')}`); + expect(result).toEqual({out: 'æ.file\n', err: '', exit: 0}); + expect(fs.readFileSync('æ.file', 'utf16le')).toBe('bar'); + }); +}); + +describe('#790 piped content without --glob-pipe', () => { + test('is still read with --encoding', async () => { + // The byte 0xE6 is æ in latin1, and not valid UTF-8 + const latin1 = new Uint8Array([0x66, 0xe6, 0x6f]); + const result = await sh($`bun ${cli} ${'æ'} x -e latin1 < ${new Response(latin1)}`); + expect(result).toEqual({out: 'fxo', err: '', exit: 0}); + }); }); diff --git a/test/fix/808.test.js b/test/fix/808.test.js index 6836c92b..1b51ebd5 100644 --- a/test/fix/808.test.js +++ b/test/fix/808.test.js @@ -5,7 +5,7 @@ import fs from 'fs'; import {rr, rrStatus} from '../helper.js'; const failed = (where, message) => - ` ❌ The code of --js failed for ${where}: ${message}\n No further processing.\n\n`; + ` ❌ The code of --js failed for ${where}\n ${message}\n No further processing.\n\n`; describe('#808 code of --js that fails', () => { test('stops the run with only the message', async () => { @@ -18,7 +18,9 @@ describe('#808 code of --js that fails', () => { test('tells a syntax error the same way', async () => { const result = await rr('o', '1 +', {pipedData: 'foo', replacementJs: true}); - expect(result.err).toStartWith(' ❌ The code of --js failed for the piped data: SyntaxError'); + expect(result.err).toStartWith( + ' ❌ The code of --js failed for the piped data\n SyntaxError' + ); expect(result.exit).toBe(3); }); @@ -46,7 +48,7 @@ describe('#808 code of --js that fails', () => { test('is in the errors of the status report', async () => { const status = await rrStatus('o', 'nope()', {pipedData: 'foo', replacementJs: true}); expect(status.errors).toEqual([ - 'The code of --js failed for the piped data: ReferenceError: nope is not defined', + 'The code of --js failed for the piped data\n ReferenceError: nope is not defined', ]); }); }); diff --git a/test/fix/813.test.js b/test/fix/813.test.js new file mode 100644 index 00000000..2a76e2ca --- /dev/null +++ b/test/fix/813.test.js @@ -0,0 +1,53 @@ +// https://github.com/mathiasrw/rexreplace/issues/813 +// When a file failed in the middle of a run, the CLI ended while other files were halfway written, and they were left only as .backup. +import {describe, test, expect, spyOn} from 'bun:test'; +import fs from 'fs'; +import {$} from 'bun'; +import {cli, sh, rr, rrStatus} from '../helper.js'; +import {engine} from '../../src/engine.ts'; + +describe('#813 a run that stops', () => { + test('finishes the files in work, so none is left only as a backup', async () => { + const names = Array.from({length: 200}, (_, i) => `b${String(i).padStart(3, '0')}.txt`); + names.forEach(name => fs.writeFileSync(name, 'foo')); + const code = "if (filename === 'b050.txt') throw new Error('boom'); 'bar'"; + const result = await sh($`bun ${cli} foo ${code} 'b*.txt' --js`); + expect(result.exit).toBe(3); + expect(fs.readdirSync('.').sort()).toEqual(names); + expect(fs.readFileSync('b050.txt', 'utf8')).toBe('foo'); + }); + + test('tells only the first panic, and keeps the files that fail after it in the status', async () => { + const names = Array.from({length: 100}, (_, i) => `f${String(i).padStart(3, '0')}.txt`); + names.forEach(name => fs.writeFileSync(name, 'foo')); + const config = {includeGlob: ['*.txt'], replacementJs: true, voidAsync: false}; + const result = await rr('foo', 'nope()', config); + expect(result.err.match(/❌/g)).toHaveLength(1); + expect(result.exit).toBe(3); + names.forEach(name => fs.writeFileSync(name, 'foo')); + expect((await rrStatus('foo', 'nope()', config)).errors.length).toBeGreaterThan(1); + }); + + test('tells the panic of a second run with the same config again (#844)', async () => { + const config = { + engine: 'default', + encoding: 'utf8', + pipedData: 'foo', + includeGlob: [], + excludeGlob: [], + excludeRe: [], + pattern: 'o', + replacement: 'nope()', + replacementJs: true, + }; + const spy = spyOn(process.stderr, 'write').mockImplementation(() => true); + try { + await expect(engine(config)).rejects.toMatchObject({panic: true}); + spy.mockClear(); + await expect(engine(config)).rejects.toMatchObject({panic: true}); + expect(spy.mock.calls.join('')).toContain('❌'); + } finally { + spy.mockRestore(); + } + }); +}); diff --git a/test/fix/815.test.js b/test/fix/815.test.js index efb3413b..9a9028e0 100644 --- a/test/fix/815.test.js +++ b/test/fix/815.test.js @@ -5,6 +5,9 @@ import {rr} from '../helper.js'; describe('#815 --js code can declare its own variables', () => { test.each([ + ['a cousin with const', 'o', 'const $n00 = "C"; $n00', 'fCC'], + ['a number with var, as before', 'o', 'var $t = "V"; $t', 'fVV'], + ['a function', 'o', 'function $t() { return "F"; } $t()', 'fFF'], ['a number with let', 'o', 'let $t = "T"; $t', 'fTT'], ['a group with const', '(o)', 'const $1 = "X"; $1', 'fXX'], ['a named group with let', '(?o)', 'let rr_o = "N"; rr_o', 'fNN'], diff --git a/test/fix/816.test.js b/test/fix/816.test.js index 11cde209..60fe0ff9 100644 --- a/test/fix/816.test.js +++ b/test/fix/816.test.js @@ -7,6 +7,11 @@ describe('#816 named groups with letters that are not ASCII', () => { test.each([ ['año', '(?\\d)', '"[" + rr_año + "]"', 'a[1]b[2]'], ['日', '(?<日>\\d)', 'rr_日 * 2', 'a2b4'], + ['año with a space at the end', '(?\\d)', 'rr_año_ + "|"', 'a1 |b2 |'], + ['𝒜, outside the BMP', '(?<𝒜>\\d)', 'rr_𝒜 * 2', 'a2b4'], + ['$é, with $', '(?<$é>\\d)', 'rr_$é * 2', 'a2b4'], + ['Æble, in uppercase', '(?<Æble>\\d)', 'rr_Æble * 2', 'a2b4'], + ['x٣, with a digit that is not ASCII', '(?\\d)', 'rr_x٣ * 2', 'a2b4'], ])('can be used, like %s', async (name, pattern, code, out) => { expect((await rr(pattern, code, {pipedData: 'a1b2', replacementJs: true})).out).toBe(out); }); @@ -23,6 +28,12 @@ describe('#816 named groups with letters that are not ASCII', () => { expect((await rr('[(?]', '"z"', {...config, pipedData: 'ab'})).out).toBe('zb'); }); + test('are only seen as the whole name, so rr_rr_year is not the group year', async () => { + const result = await rr('(?\\d)', 'rr_rr_year', {pipedData: '1', replacementJs: true}); + expect(result.err).toContain('ReferenceError: rr_rr_year is not defined'); + expect(result.exit).toBe(3); + }); + test('can be used with a space at the end on its own, like rr_year_', async () => { const config = {pipedData: '2024', replacementJs: true}; expect((await rr('(?\\d+)', 'rr_year_ + "|"', config)).out).toBe('2024 |'); diff --git a/test/fix/818.test.js b/test/fix/818.test.js index a5b97e98..c4029dc5 100644 --- a/test/fix/818.test.js +++ b/test/fix/818.test.js @@ -7,7 +7,13 @@ import {rr, rrStatus} from '../helper.js'; describe('#818 a replacement file that can not be read', () => { test.each([ ['is not there', 'nope.txt', 'nope.txt (ENOENT: no such file or directory)'], - ['is a folder', 'adir', 'adir (EISDIR: illegal operation on a directory, read)'], + ['is a folder', 'adir', 'adir (EISDIR: illegal operation on a directory)'], + [ + 'has a zero-width space', + 'no\u200bpe.txt', + '"no\\u200bpe.txt" (ENOENT: no such file or directory)', + ], + ['has a C1 char', 'no\u0085pe.txt', '"no\\u0085pe.txt" (ENOENT: no such file or directory)'], [ 'has a line break at the end, like from echo', 'a.txt\n', diff --git a/test/fix/819.test.js b/test/fix/819.test.js index 632d1d60..5c0350aa 100644 --- a/test/fix/819.test.js +++ b/test/fix/819.test.js @@ -4,49 +4,96 @@ import {describe, test, expect, spyOn} from 'bun:test'; import fs from 'fs'; import {rr} from '../helper.js'; -// The sort looks at each file with statSync and no options. When it looks at the given file, the file is made to fail. -function failInSort(name, fail) { +// The sort looks at each file with statSync and no options. When it looks at one of the given files, that file is made to fail. +function failInSort(names, fail) { const statSync = fs.statSync; return spyOn(fs, 'statSync').mockImplementation((file, options) => { - if (name === file && !options) { - fail(); + if (names.includes(file) && !options) { + fail(file); } return statSync(file, options); }); } -describe('#819 a file that can not be looked at in the sort', () => { +// Each sort puts z.md before a.md, so the order is not the ascii order. The files to delete would come first. +async function makeFiles(gone) { + for (const name of [...gone, 'z.md', 'a.md']) { + fs.writeFileSync(name, 'z.md' === name ? 'foo' : 'foo foo'); + await Bun.sleep(20); + } + gone.forEach(name => fs.utimesSync(name, 500, 500)); + fs.utimesSync('z.md', 1000, 1000); + fs.utimesSync('a.md', 3000, 3000); +} + +async function sortWithGone(gone, config) { + await makeFiles(gone); + const spy = failInSort(gone, file => fs.unlinkSync(file)); + try { + return await rr('foo', 'bar', {includeGlob: ['*.md'], ...config}); + } finally { + spy.mockRestore(); + } +} + +describe('#819 a file deleted before --sort looks at it', () => { test.each([['modified'], ['created'], ['size']])( - 'when deleted, is sorted last with --sort %s, and told about as not found', + 'is told about as not found with --sort %s, and the other files are sorted', async sort => { - ['a.md', 'gone.md', 'z.md'].forEach(name => fs.writeFileSync(name, 'foo')); - const spy = failInSort('gone.md', () => fs.unlinkSync('gone.md')); + const result = await sortWithGone(['gone.md'], {sort}); + expect(result.out).toBe('z.md\na.md\n'); + expect(result.err).toEndWith(' ❌ File not found: gone.md\n\n'); + expect(result.exit).toBe(2); + } + ); + + test('is told about for each file, when more than one is deleted', async () => { + expect(await sortWithGone(['gone.md', 'gone2.md'], {sort: 'size'})).toEqual({ + out: 'z.md\na.md\n', + err: ' ❌ File not found: gone.md\n\n ❌ File not found: gone2.md\n\n', + exit: 2, + }); + }); + + test('does not change the order of the others with --sort-reverse', async () => { + expect(await sortWithGone(['gone.md'], {sort: 'size', sortReverse: true})).toEqual({ + out: 'a.md\nz.md\n', + err: ' ❌ File not found: gone.md\n\n', + exit: 2, + }); + }); + + test('stops the run with --halt, before any file is changed', async () => { + expect(await sortWithGone(['gone.md'], {sort: 'size', halt: true})).toEqual({ + out: '', + err: ' ❌ File not found: gone.md\n No further processing.\n\n', + exit: 3, + }); + expect(fs.readFileSync('a.md', 'utf8')).toBe('foo foo'); + }); +}); + +describe('#819 a file that can not be looked at in the sort for another reason', () => { + test.each([ + [false, '\n', 2, 'a.md\n'], + [true, '\n No further processing.\n', 3, ''], + ])( + 'is an error with that reason, and is left out, with --halt %s', + async (halt, end, exit, out) => { + ['a.md', 'b.md'].forEach(name => fs.writeFileSync(name, 'foo')); + const spy = failInSort(['b.md'], () => { + throw Object.assign(new Error('EACCES: permission denied'), {code: 'EACCES'}); + }); try { - expect(await rr('foo', 'bar', {includeGlob: ['*.md'], sort})).toEqual({ - out: 'a.md\nz.md\n', - err: ' ❌ File not found: gone.md\n\n', - exit: 2, + expect(await rr('foo', 'bar', {includeGlob: ['*.md'], sort: 'size', halt})).toEqual({ + out, + err: ' ❌ EACCES: permission denied' + end + '\n', + exit, }); + expect(fs.readFileSync('b.md', 'utf8')).toBe('foo'); } finally { spy.mockRestore(); } } ); - - test('for another reason, is an error with that reason, and is left out', async () => { - ['a.md', 'b.md'].forEach(name => fs.writeFileSync(name, 'foo')); - const spy = failInSort('b.md', () => { - throw Object.assign(new Error('EACCES: permission denied'), {code: 'EACCES'}); - }); - try { - expect(await rr('foo', 'bar', {includeGlob: ['*.md'], sort: 'size'})).toEqual({ - out: 'a.md\n', - err: ' ❌ EACCES: permission denied\n\n', - exit: 2, - }); - expect(fs.readFileSync('b.md', 'utf8')).toBe('foo'); - } finally { - spy.mockRestore(); - } - }); }); diff --git a/test/fix/820.test.js b/test/fix/820.test.js index 70a93b25..a4b4f512 100644 --- a/test/fix/820.test.js +++ b/test/fix/820.test.js @@ -1,17 +1,42 @@ // https://github.com/mathiasrw/rexreplace/issues/820 // Names that natural sees as the same kept the order of the glob, which differs between Node and Bun, -// and natural followed the locale of the user on Node, but not on Bun. +// and natural followed the locale of the user on Node, but not on Bun. The locale is checked in test/cli/run.sh. import {describe, test, expect} from 'bun:test'; import fs from 'fs'; import {rr} from '../helper.js'; +async function sorted(names, config = {}) { + names.forEach(name => fs.writeFileSync(name, 'foo')); + const result = await rr('foo', 'bar', { + includeGlob: ['*.md'], + simulate: true, + sort: 'natural', + ...config, + }); + return result.out.split('\n').filter(Boolean); +} + describe('#820 the natural order', () => { - test('orders the names it sees as the same by ascii', async () => { - for (const name of ['file1.md', 'file01.md', 'file001.md', 'cafe.md', 'café.md']) { - fs.writeFileSync(name, 'foo'); - } - expect( - (await rr('foo', 'bar', {includeGlob: ['*.md'], simulate: true, sort: 'natural'})).out - ).toBe('cafe.md\ncafé.md\nfile001.md\nfile01.md\nfile1.md\n'); + test.each([ + ['numbers as numbers', ['file10.md', 'file9.md'], ['file9.md', 'file10.md']], + ['case ignored', ['B.md', 'a.md'], ['a.md', 'B.md']], + ['accents ignored', ['cafe2.md', 'café1.md'], ['café1.md', 'cafe2.md']], + ['aa as two a, like in English', ['b.md', 'aab.md'], ['aab.md', 'b.md']], + [ + 'ascii for the names it sees as the same', + ['file1.md', 'file01.md', 'file001.md', 'cafe.md', 'café.md'], + ['cafe.md', 'café.md', 'file001.md', 'file01.md', 'file1.md'], + ], + ])('has %s', async (name, names, order) => { + expect(await sorted(names)).toEqual(order); + }); + + test('is turned around with --sort-reverse, also for the names it sees as the same', async () => { + expect(await sorted(['file1.md', 'file01.md', 'B.md', 'a.md'], {sortReverse: true})).toEqual([ + 'file1.md', + 'file01.md', + 'B.md', + 'a.md', + ]); }); }); diff --git a/test/fix/822.test.js b/test/fix/822.test.js index cfa1b874..bb0c81a1 100644 --- a/test/fix/822.test.js +++ b/test/fix/822.test.js @@ -4,7 +4,7 @@ import {describe, test, expect} from 'bun:test'; import {rr} from '../helper.js'; const failed = text => - ` ❌ The code of --js failed for the piped data: ${text}\n No further processing.\n\n`; + ` ❌ The code of --js failed for the piped data\n ${text}\n No further processing.\n\n`; describe('#822 code that throws something that is not an error', () => { test.each([ @@ -24,12 +24,21 @@ describe('#822 code that throws something that is not an error', () => { }); }); - test('only tells the start of a big object', async () => { - const result = await rr('o', 'throw {text: "y".repeat(1000)}', { - pipedData: 'foo', - replacementJs: true, - }); - expect(result.err).toContain('an object was thrown: {"text":"' + 'y'.repeat(291) + '…'); - expect(result.err.length).toBeLessThan(400); + test('only tells the start of a big object, cut by characters so an emoji is not split', async () => { + const config = {pipedData: 'foo', replacementJs: true}; + const result = await rr('o', 'throw {t: "🦖".repeat(400)}', config); + const text = result.err.split('\n')[1].replace(' ', ''); + expect(Array.from(text)).toHaveLength(301); + expect(text).toEndWith('🦖…'); + expect(text).not.toContain('\uFFFD'); + }); + + test.each([ + ['class Foo { constructor() { this.a = 1; } }; throw new Foo()', 'a Foo was thrown: {"a":1}'], + ['throw new (class Item {})()', 'an Item was thrown'], + ])('tells the class and what it holds: %s', async (code, text) => { + expect((await rr('o', code, {pipedData: 'foo', replacementJs: true})).err).toStartWith( + ' ❌ The code of --js failed for the piped data\n ' + text + '\n' + ); }); }); diff --git a/test/fix/824.test.js b/test/fix/824.test.js index ae2d4bc9..e7b19d31 100644 --- a/test/fix/824.test.js +++ b/test/fix/824.test.js @@ -41,8 +41,18 @@ describe('#824 --js code that fails in a module', () => { const config = {pipedData: 'foo', replacementJs: true, verbose: true}; const result = await rr('o', 'throw new Proxy({}, {get() { throw 1; }})', config); expect(result.err).toContain( - 'The code of --js failed for the piped data: an object was thrown' + 'The code of --js failed for the piped data\n an object was thrown' ); expect(result.exit).toBe(3); }); + + test('reads a place without a function name, also when the path has parentheses (#846)', async () => { + // Bun has no function name for the code at the top of a module + fs.mkdirSync('a (copy)'); + fs.writeFileSync('a (copy)/top.js', 'throw new Error("top");\n'); + const config = {pipedData: 'foo', replacementJs: true, verbose: true}; + expect((await rr('o', 'r("a (copy)/top.js")', config)).err).toMatch( + /\n {4}at a \(copy\)\/top\.js:1:\d+\n/ + ); + }); }); diff --git a/test/fix/825.test.js b/test/fix/825.test.js index 58bb3614..067420b2 100644 --- a/test/fix/825.test.js +++ b/test/fix/825.test.js @@ -2,9 +2,29 @@ // A named group was a variable like _year, which could clash: with groups a and a_, _a_ was both a_ and a with a space, // and a group named _require_ broke r(). import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; import {rr} from '../helper.js'; +const js = {pipedData: 'in 1999 and 2024', replacementJs: true}; + describe('#825 named groups in --js', () => { + test.each([ + ['a name in lowercase', '(?\\d+)', '"<" + rr_year + ">"', 'in <1999> and <2024>'], + [ + 'the cousin with a space at the end', + '(?\\d+)', + 'rr_year + "," + rr_year_', + 'in 1999,1999 and 2024,2024 ', + ], + ['a name in uppercase', '(?\\d+)', '"Y" + rr_Year', 'in Y1999 and Y2024'], + ['a name with a number', '(?\\d+)', '"Y" + rr_y2k', 'in Y1999 and Y2024'], + ['a name with $', '(?\\d+)', '"Y" + rr_a$', 'in Y1999 and Y2024'], + ['two names that only differ by case', '(?\\d)(?\\d)', 'rr_Y + rr_y', 'in 9199 and 0242'], + ['a name next to its number', '(?\\d+)', '$1 + rr_year', 'in 19991999 and 20242024'], + ])('gives %s', async (name, pattern, code, out) => { + expect(await rr(pattern, code, js)).toEqual({out, err: '', exit: null}); + }); + test('have rr_ in front, so they can not clash with what RexReplace uses', async () => { // With _ in front, the group _require with a space was __require_, which r() uses itself. // $1 makes the code run once per match, where the variables of the groups are made. @@ -12,22 +32,35 @@ describe('#825 named groups in --js', () => { expect((await rr('(?<_require>a)', '$1 + r("path").sep', config)).out).toBe('a/b'); }); - test('can not end with _, as that is the name of another group with a space', async () => { - expect(await rr('(?\\d)(?\\w)', 'rr_a', {pipedData: '1x', replacementJs: true})).toEqual( - { + test('are gone with _ in front', async () => { + expect((await rr('(?\\d+)', '_year', js)).err).toContain('ReferenceError'); + }); +}); + +describe('#825 a named group that ends with _', () => { + test.each([['a_'], ['Year_'], ['_'], ['y2_']])( + 'panics for %s, as that is the name of another group with a space, and nothing is changed', + async name => { + fs.writeFileSync('a.md', 'in 1999'); + const result = await rr(`(?<${name}>\\d+)`, '"x"', { + includeGlob: ['a.md'], + replacementJs: true, + }); + expect(result).toEqual({ out: '', - err: ' ❌ A named group can not end with _: a_\n No further processing.\n\n', + err: ` ❌ A named group can not end with _: ${name}\n No further processing.\n\n`, exit: 3, - } - ); - }); + }); + expect(fs.readFileSync('a.md', 'utf8')).toBe('in 1999'); + } + ); - test('can end with _ with --output-match, as the code is not run', async () => { + test('is fine with --output-match, as the code is not run', async () => { const config = {pipedData: 'ab', replacementJs: true, outputMatch: true}; expect((await rr('(?a)', 'x', config)).exit).toBe(null); }); - test('can end with _ without --js, as there are no variables then', async () => { + test('is fine without --js, as there are no variables then', async () => { expect((await rr('(?o)', 'x', {pipedData: 'foo'})).out).toBe('fxx'); }); }); diff --git a/test/fix/826.test.js b/test/fix/826.test.js index 36989218..590e3cbc 100644 --- a/test/fix/826.test.js +++ b/test/fix/826.test.js @@ -1,24 +1,88 @@ // https://github.com/mathiasrw/rexreplace/issues/826 -// --rename put control characters into the new name, like the line break at the end of a replacement file. +// https://github.com/mathiasrw/rexreplace/issues/848 +// https://github.com/mathiasrw/rexreplace/issues/843 +// --rename could make a name with a line break, like from the end of a replacement file. A path with a control character +// is now a clash, and so is a new name of only dots and spaces, or one that ends with a space or a dot. import {describe, test, expect} from 'bun:test'; import fs from 'fs'; import {rr} from '../helper.js'; -describe('#826 control characters in a new name', () => { - test('are removed, like the line break at the end of a replacement file', async () => { +const controls = ' ❌ Names can not have control characters, like a line break or a tab.'; +const dots = + ' ❌ A name must have more than dots and spaces, and can not end with a space or a dot.'; + +describe('#826 a new name with a control character', () => { + test('is a clash, also from the line break at the end of a replacement file', async () => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('name.txt', 'b\n'); + const result = await rr('^a', 'name.txt', { + includeGlob: ['a.md'], + rename: true, + replacementFile: true, + }); + expect(result.err).toStartWith( + controls + ' Use --trim-file for the line break at the end of a file.' + ); + expect(result.exit).toBe(3); + expect(fs.readdirSync('.').sort()).toEqual(['a.md', 'name.txt']); + }); + + test('is fine with --trim-file', async () => { fs.writeFileSync('a.md', 'foo'); fs.writeFileSync('name.txt', 'b\n'); - const config = {includeGlob: ['a.md'], rename: true, replacementFile: true, verbose: true}; - const result = await rr('^a', 'name.txt', config); - expect(result.out).toBe('b.md\n'); - expect(result.err).toContain('Removed control characters from the new name of: a.md'); - expect(fs.readdirSync('.').sort()).toEqual(['b.md', 'name.txt']); + const config = {includeGlob: ['a.md'], rename: true, replacementFile: true, trimFile: true}; + expect((await rr('^a', 'name.txt', config)).out).toBe('b.md\n'); + }); + + test('is a clash with --rename-path and --js too, and nothing is renamed', async () => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('b.md', 'foo'); + const config = {includeGlob: ['*.md'], renamePath: true, replacementJs: true}; + expect((await rr('^a', '"x\\t"', config)).err).toStartWith(controls); + expect(fs.readdirSync('.').sort()).toEqual(['a.md', 'b.md']); }); - test('are removed with --rename-path too, also a tab, a null byte and DEL', async () => { + test('is a clash for DEL too', async () => { fs.writeFileSync('a.md', 'foo'); - const config = {includeGlob: ['a.md'], renamePath: true}; - expect((await rr('^a', 'b\t\0\x7f\x01c', config)).out).toBe('bc.md\n'); - expect(fs.readdirSync('.')).toEqual(['bc.md']); + const config = {includeGlob: ['a.md'], rename: true, replacementJs: true}; + expect((await rr('^a', '"x\\x7f"', config)).err).toStartWith(controls); + expect(fs.readdirSync('.')).toEqual(['a.md']); + }); +}); + +describe('#848 a path with a control character in the run', () => { + test('is a clash, also when the pattern would not change it', async () => { + fs.writeFileSync('a\tb.md', 'foo'); + fs.writeFileSync('c.md', 'foo'); + const result = await rr('zzz', 'yyy', {includeGlob: ['*.md'], rename: true}); + expect(result.err).toStartWith(controls + '\n From: a\tb.md\n'); + expect(result.exit).toBe(3); + expect(fs.readdirSync('.').sort()).toEqual(['a\tb.md', 'c.md']); + }); +}); + +describe('#843 a new name of only dots and spaces, or with a space or a dot at the end', () => { + test.each([ + ['empty, in a folder', '^x\\.md$', ''], + ['a dot', '^x\\.md$', '.'], + ['two dots', '^x\\.md$', '..'], + ['spaces', '^x\\.md$', ' '], + ['ending with a space', '^x\\.md$', 'x '], + ['ending with a dot', '^x\\.md$', 'x.'], + ])('is a clash when it is %s', async (name, pattern, replacement) => { + fs.mkdirSync('d'); + fs.writeFileSync('d/x.md', 'foo'); + const result = await rr(pattern, replacement, {includeGlob: ['d/x.md'], rename: true}); + expect(result.err).toStartWith(dots); + expect(result.err.match(/❌/g)).toHaveLength(2); + expect(result.exit).toBe(3); + expect(fs.readdirSync('d')).toEqual(['x.md']); + }); + + test('is fine when it starts with a dot, like .env', async () => { + fs.writeFileSync('x.md', 'foo'); + expect((await rr('^x\\.md$', '.env', {includeGlob: ['x.md'], rename: true})).out).toBe( + '.env\n' + ); }); }); From 09cb129406509c0c3066178f6dc2eaf47ef27a9c Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Sat, 24 Oct 2026 03:19:54 +1100 Subject: [PATCH 29/29] Simplify the engine, split the status from the output, and name functions by what they do --- devguide.md | 10 +- src/cli.ts | 7 +- src/engine.ts | 1114 ++++++++++++++++++------------------------------- src/output.ts | 79 +--- src/shim.ts | 7 - src/status.ts | 93 +++++ 6 files changed, 522 insertions(+), 788 deletions(-) delete mode 100644 src/shim.ts create mode 100644 src/status.ts diff --git a/devguide.md b/devguide.md index 4ba7bae8..db476f9a 100644 --- a/devguide.md +++ b/devguide.md @@ -15,18 +15,18 @@ How we like the code in RexReplace to be. ## Telling what is going on -All output goes through `src/output.ts`. Pick the function by who the message is for: +All output goes through `src/output.ts`, which only prints. What a run records in its status, and the panic, are in `src/status.ts`, which prints through `output.ts`. Pick the function by who the message is for: | Function | Shown when | For | Example | | --- | --- | --- | --- | | `print(item, delimiter, tail, stream)` | Always | The result. Puts the delimiter after each item, or only between them without a tail. Without a delimiter it is a line break, unless the item already ends with one. | A name, the content of a file, a match | | `printDone()` | At the end of a run, also when it is stopped | Ends stdout and stderr with a line break when `--nl` is set | | -| `info(name)` | Unless `--quiet` | The name of an affected file. stdout, or stderr when stdout has content. | `a.md` | +| `printName(name)` | Unless `--quiet` | The name of an affected file. stdout, or stderr when stdout has content. | `a.md` | | `warn(msg)` | Unless `--quiet` | Something is likely a mistake, or there is nothing to do, but the run goes on. Starts with 🟡. Also added to `status.warnings`. | `0 files found` | | `error(msg, data)` | Unless `--quiet-total` | Something went wrong. Starts with ❌. The run goes on, unless `--halt` is set: then a panic is thrown. Also added to `status.errors`, so the CLI gives the exit code 2. | `File not found: a.md` | -| `checkFailed(text)` | Unless `--quiet-total` | A check from `--expect-*` that is not met. An error, also in `status.checks`, so the CLI gives the exit code 1 when the failed checks are the only errors. | `Expected a diff, but nothing was changed` | -| `panic(msg, data, help)` | Unless `--quiet-total` | Something went wrong and the run can not go on. Starts with ❌, ends with `No further processing.`, and throws a panic, so the CLI exits with 3. Also added to `status.errors`. | `No data piped into replacement` | -| `chat(msg)` | `--verbose`, also with `--quiet`, but not with `--quiet-total` | The user: what happens to each file. Only ever to stderr. Also kept in `engine.steps` of the status report. | `Open sync: a.md`, `Nothing changed in: a.md` | +| `checkFailed(text)` (inside `checkExpectations()`) | Unless `--quiet-total` | A check from `--expect-*` that is not met. An error, also in `status.checks`, so the CLI gives the exit code 1 when the failed checks are the only errors. | `Expected a diff, but nothing was changed` | +| `panic(msg, data)` | Unless `--quiet-total` | Something went wrong and the run can not go on. Starts with ❌, ends with `No further processing.`, and throws a panic, so the CLI exits with 3. Also added to `status.errors`. | `No data piped into replacement` | +| `verbose(msg)` | `--verbose`, also with `--quiet`, but not with `--quiet-total` | The user: what happens to each file. Only ever to stderr. Also kept in `engine.steps` of the status report. | `Open sync: a.md`, `Nothing changed in: a.md` | - What the run did is in the status report: the engine resolves with it, and `--status` prints it. So there is no extra debug output to print along the way. - `--debug` prints the status report, with what went into the engine and what it did, and hides the rest, except the content of `--output` and the matches of `--output-match`. diff --git a/src/cli.ts b/src/cli.ts index 086483b3..e04eb914 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,7 +1,8 @@ // CLI interface for rexreplace import * as rexreplace from './engine.ts'; -import {outputConfig, panic, writeTo} from './output.ts'; +import {writeTo} from './output.ts'; +import {statusConfig, panic} from './status.ts'; import {alias} from './alias.ts'; // Make it visible when Bun is the runtime (like with rexreplace-bun) as the regex engine is then not the one from Node. @@ -592,12 +593,10 @@ if (config.debug) { config.status = config.verbose = config.quietTotal = true; } -outputConfig(config); +statusConfig(config); config.pipedData = null; -config.showHelp = yargs.showHelp; - config.pattern = pattern; config.includeGlob = yargs.argv._; diff --git a/src/engine.ts b/src/engine.ts index 523fb7a2..3de1a7ae 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -6,33 +6,28 @@ const path = require('path'); const globs = require('fast-glob'); -const now = new Date(); - -import { - outputConfig, - chat, - info, - print, - printDone, - warn, - error, - panic, - checkFailed, -} from './output.ts'; - -import './shim.ts'; +import {print, printDone, printName} from './output.ts'; +import {statusConfig, verbose, warn, error, panic, checkExpectations} from './status.ts'; const re = { mctime: /[mc]time/, colon: /:/g, capturedGroupRef: /\$\d/, + startsWithVowel: /^[aeiou]/i, + // A path that is not welcome in --rename, also when the pattern would not change it + controlChar: /[\x00-\x1f\x7f]/, + slash: /[\\/]/, + // Like an empty name, . or .., which are folders. Windows does not allow a space or a dot at the end. + onlyDotsOrEndsWithDot: /^[. ]*$|[. ]$/, + absolutePath: /^(\/|[A-Za-z]:[\\/])/, + // A line of a stack trace without a function name has the location right after at, like `at /dir/a (copy)/anon.js:1:59`. + // With a name the location is in parentheses at the end, like `at fn (/path/file.js:2:15)`. + stackLineWithoutName: /^\s*at (?:file:\/\/)?((?:\/|[A-Za-z]:[\\/]).*?)(:\d+:\d+)\s*$/, + stackLineWithName: /^\s*at .*? \((?:file:\/\/)?(.+?)(:\d+:\d+)\)\s*$/, // Only the chars with a meaning outside a character class. Unicode mode does not allow to escape the others (#686). regexSpecialChars: /[\[\]{}()*+?.\/\\^$|]/g, byteOrSize: /bytes|size/, folderName: /[\\\/]+([^\\\/]+)[\\\/]+[^\\\/]+$/, - // The numbers of #791 that count what came before: $i, $c, $m and $M with their cousins, like $i00 or $M_ - runNumbers: /\$[icmM](?![A-Za-z])/, - matchNumbers: /\$[mM](?![A-Za-z])/, // A path that is not clear as it is printed: a space at the start or the end, or a char that can not be seen, // like a line break from echo or a zero-width space unclearPath: /^\s|\s$|[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2060-\u2064\ufeff]/, @@ -50,19 +45,16 @@ export {version}; const runtime = process.versions.bun ? 'Bun' : 'Node'; const runtimeEngine = process.versions.bun ? 'JSC' : 'V8'; -function validateEncoding(encoding = 'utf8') { - if (!Buffer.isEncoding(encoding)) { - panic('Unknown encoding:', encoding); - } -} - // Resolves with the status of the run when all files are done. // When the run can not go on, a panic is thrown with the status so far. export async function engine(config: any = {engine: 'default'}) { config.statusInfo = newStatus(config); - // A new run has had no panic yet, also when the same config is used again - config.stopped = false; - outputConfig(config); + // A new run has had no panic yet, has read no replacement file, and has its own time, also when the same config is used again + config.plzStop = false; + config.replacementFiles = new Map(); + config.unreadableFiles = new Set(); + config.startTime = new Date(); + statusConfig(config); try { await run(config); checkExpectations(config); @@ -109,9 +101,29 @@ function newStatus(config) { return status; } -async function run(config) { - validateEncoding(config.encoding); +// Flags that can not go together +function checkFlags(config) { + if (!Buffer.isEncoding(config.encoding ?? 'utf8')) { + panic('Unknown encoding:', config.encoding); + } + if (config.expectZeroDiff && (config.expectDiff || config.expectDiffAll)) { + panic('--expect-zero-diff can not be used with --expect-diff or --expect-diff-all'); + } + if (config.globPipe && (config.replacementPipe || config.pipe)) { + panic( + '--glob-pipe can not be used with --replacement-pipe or --js-pipe, as the piped data can only be one thing' + ); + } + if (config.rename && config.renamePath) { + panic('--rename can not be used with --rename-path'); + } + if (config.renameForce && !config.rename && !config.renamePath) { + panic('--rename-force needs --rename or --rename-path'); + } +} +// The config the run works with, made from the flags +function unpackConfig(config) { // For the numbers of --js: the files with a match, the files changed and the matches so far config.numbers = {matched: 0, changed: 0, matches: 0}; @@ -119,30 +131,20 @@ async function run(config) { warn('The code is not run with --output-match, as only the matches are printed.'); } - if (config.expectZeroDiff && (config.expectDiff || config.expectDiffAll)) { - panic('--expect-zero-diff can not be used with --expect-diff or --expect-diff-all'); - } - // With --glob-pipe each line of the piped data is a path or a glob, used together with the ones given as arguments if (config.globPipe) { - if (config.replacementPipe || config.pipe) { - panic( - '--glob-pipe can not be used with --replacement-pipe or --js-pipe, as the piped data can only be one thing' - ); - } config.pipedDataUsed = true; const lines = (config.pipedData ?? '').split(config.nullIn ? '\0' : /\r?\n/); config.includeGlob = config.includeGlob.concat(lines.filter(line => '' !== line)); } - config.pattern = getFinalPattern(config.pattern, config) || ''; - - config.replacement = getFinalReplacement(config.replacement, config) || ''; + config.pattern = regexPattern(config.pattern, config); + config.replacement = readReplacement(config) || ''; config.replacementOri = config.replacement; - - // $i and $c need one more search of each file, so the files are only counted when the code uses them - config.countFiles = config.replacementJs && /\$[ic](?![A-Za-z])/.test(config.replacementOri); + if (config.outputMatch) { + config.replacement = matchPrinter(config); + } // The most specific delimiter wins. Without one there is a line break after each item, unless it already ends with one. const delimiter = config.delimiter ?? (config.nullOut ? '\0' : null); @@ -152,15 +154,16 @@ async function run(config) { // --output-match wins over --output config.output = config.output && !config.outputMatch; - config.regex = getFinalRegex(config.pattern, config) || ''; + config.regex = getFinalRegex(config.pattern, config); + + config.codeUses = config.replacementJs ? codeUses(config.replacementOri, config.regex) : null; // In --js a named group is a variable like rr_year, and rr_year_ has a space at the end. // A group that ends with _ would have the same name as another group with a space, so it can not be used. // With --output-match the code is not run, so there are no variables. - const endsWith_ = - config.replacementJs && !config.outputMatch && config.regex - ? namedGroups(config.regex).find(name => name.endsWith('_')) - : null; + const endsWith_ = config.outputMatch + ? null + : config.codeUses?.groups.find(name => name.endsWith('_')); if (endsWith_) { panic('A named group can not end with _:', endsWith_); } @@ -186,12 +189,12 @@ async function run(config) { config.replacement = () => text; } - const piped = handlePipedData(config); + usePipeAsContent(config); if (null !== config.pipedData) { config.statusInfo.pipe = { bytes: Buffer.byteLength(config.pipedData, config.encoding), - type: pipeType(config, piped), + type: pipeType(config), updated: false, }; } @@ -199,21 +202,17 @@ async function run(config) { // The more specific tail wins. Piped data that is printed is left as it came in, but matches are matches, from a file or a pipe. config.nameDelimiterTail = config.nameDelimiterTail ?? config.tail ?? true; config.contentDelimiterTail = - config.contentDelimiterTail ?? config.tail ?? !(piped && config.output); - - if (config.rename && config.renamePath) { - panic('--rename can not be used with --rename-path'); - } - - if (config.renameForce && !config.rename && !config.renamePath) { - panic('--rename-force needs --rename or --rename-path'); - } + config.contentDelimiterTail ?? config.tail ?? !(config.dataIsPiped && config.output); +} - if ((config.rename || config.renamePath) && piped) { - panic('--rename and --rename-path need files, as piped data has no name to change'); - } +async function run(config) { + checkFlags(config); + unpackConfig(config); - if (piped) { + if (config.dataIsPiped) { + if (config.rename || config.renamePath) { + panic('--rename and --rename-path need files, as piped data has no name to change'); + } return doReplacement('Piped data', config, config.pipedData, 1, 1); } @@ -224,7 +223,7 @@ async function run(config) { return warn('0 files found'); } - chat(config.files.length + ' files found'); + verbose(config.files.length + ' files found'); // One file needs no async flow, and with -o, -m or --sort one file at a time gives the output in order config.voidAsync = @@ -235,7 +234,7 @@ async function run(config) { null != config.sort || config.sortReverse || // The numbers that count what came before are only the same on every run when the files come in order - (config.replacementJs && re.runNumbers.test(config.replacementOri)); + config.codeUses?.countsBefore; const files = config.files.filter(filepath => fs.existsSync(filepath) ? true : error('File not found:', filepath) @@ -244,9 +243,12 @@ async function run(config) { if (config.rename || config.renamePath) { return renameFiles(files, config); } + return processFiles(files, config); +} - // A limited number of files are in work at the same time, so the memory and the open files stay within bounds. - // Each worker takes the next file when it is done with one. Up to 16 files are all handled at once. +// A limited number of files are in work at the same time, so the memory and the open files stay within bounds. +// Each worker takes the next file when it is done with one. Up to 16 files are all handled at once. +async function processFiles(files, config) { const workers = config.voidAsync ? 1 : files.length <= 16 @@ -254,14 +256,13 @@ async function run(config) { : Math.max(16, os.availableParallelism() * 2); let next = 0; // After a panic no worker starts a new file. A file that is already in work is finished. - let stopped = false; const worker = async () => { - while (next < files.length && !stopped) { + while (next < files.length && !config.plzStop) { const index = next++; try { await openFile(files[index], config, index + 1, files.length); } catch (e) { - stopped = true; + config.plzStop = true; throw e; } } @@ -278,7 +279,7 @@ async function run(config) { function openFile(file, config, number, total) { config.statusInfo.files.searched.push(file); if (config.voidAsync) { - chat('Open sync: ' + file); + verbose('Open sync: ' + file); let data; try { data = fs.readFileSync(file, config.encoding); @@ -287,122 +288,137 @@ function openFile(file, config, number, total) { } return doReplacement(file, config, data, number, total); } else { - chat('Open async: ' + file); + verbose('Open async: ' + file); return fs.promises .readFile(file, config.encoding) .then(data => doReplacement(file, config, data, number, total), error); } } -// postfix argument names to limit the probabillity of user inputted javascript accidently using same values -function doReplacement( - _file_rr: string, - _config_rr: any, - _data_rr: string, - _number_rr: number, - _total_rr: number -) { - chat('Work on content from: ' + _file_rr); +function doReplacement(file: string, config: any, data: string, number: number, total: number) { + verbose('Work on content from: ' + file); // Main regexp of the whole thing. With --output-match the replacement prints the matches, so the code of --replacement-js is not run. - const result = _data_rr.replace( - _config_rr.regex, - _config_rr.replacementJs && !_config_rr.outputMatch - ? codeReplacement( - _file_rr, - _config_rr, - _data_rr, - fileNumbers(_config_rr, _number_rr, _total_rr) - ) - : _config_rr.replacement + const result = data.replace( + config.regex, + config.replacementJs && !config.outputMatch + ? codeReplacement(file, config, data, fileNumbers(config, number, total)) + : config.replacement ); - countFile(_config_rr, _data_rr, result); + countFile(config, data, result); - if (_config_rr.output || _config_rr.outputMatch) { + if (config.output || config.outputMatch) { // With --output-match the matches are printed by the replacement - if (_config_rr.output) { - chat('Output result from: ' + _file_rr); - print(result, _config_rr.contentDelimiter, _config_rr.contentDelimiterTail); + if (config.output) { + verbose('Output result from: ' + file); + print(result, config.contentDelimiter, config.contentDelimiterTail); } // The file is not changed, but it is still named if its content would change - if (_config_rr.dataIsPiped) { - _config_rr.statusInfo.pipe.updated = result !== _data_rr; - } else if (result !== _data_rr) { - updated(_file_rr, _config_rr); - } else if (_config_rr.nameAll) { - info(_file_rr); + if (config.dataIsPiped) { + config.statusInfo.pipe.updated = result !== data; + } else if (result !== data) { + markFileChanged(file, config); + } else if (config.nameAll) { + printName(file); } return; } // Nothing replaced = no need for writing file again - if (result === _data_rr) { - chat('Nothing changed in: ' + _file_rr); - if (_config_rr.nameAll) { - info(_file_rr); + if (result === data) { + verbose('Nothing changed in: ' + file); + if (config.nameAll) { + printName(file); } return; } // --simulate does everything but write, so the file is named as if it was changed - if (_config_rr.simulate) { - return updated(_file_rr, _config_rr); + if (config.simulate) { + return markFileChanged(file, config); } // Release the memory while storing files - _data_rr = ''; + data = ''; - chat('Write new content to: ' + _file_rr); + verbose('Write new content to: ' + file); // Write directly to the same file (if the process is killed all new and old data is lost) - if (_config_rr.voidBackup) { + if (config.voidBackup) { return fs.promises - .writeFile(_file_rr, result, _config_rr.encoding) - .then(() => updated(_file_rr, _config_rr), error); + .writeFile(file, result, config.encoding) + .then(() => markFileChanged(file, config), error); } //Make sure data is always on disk - const oriFile = path.resolve(_file_rr); + const oriFile = path.resolve(file); const salt = new Date().toISOString().replace(re.colon, '_').replace('Z', ''); const backupFile = oriFile + '.' + salt + '.backup'; - if (_config_rr.voidAsync) { + if (config.voidAsync) { try { fs.renameSync(oriFile, backupFile); - fs.writeFileSync(oriFile, result, _config_rr.encoding); - if (!_config_rr.keepBackup) { + fs.writeFileSync(oriFile, result, config.encoding); + if (!config.keepBackup) { fs.unlinkSync(backupFile); } } catch (e) { return error(e); } - return updated(_file_rr, _config_rr); + return markFileChanged(file, config); } return fs.promises .rename(oriFile, backupFile) - .then(() => fs.promises.writeFile(oriFile, result, _config_rr.encoding)) - .then(() => _config_rr.keepBackup || fs.promises.unlink(backupFile)) - .then(() => updated(_file_rr, _config_rr), error); + .then(() => fs.promises.writeFile(oriFile, result, config.encoding)) + .then(() => config.keepBackup || fs.promises.unlink(backupFile)) + .then(() => markFileChanged(file, config), error); } -// The content of the file was changed, or would be with --output and --output-match, so it is named -function updated(file, config) { +// The content of the file was changed, or would be with --simulate, --output and --output-match, so it is named +function markFileChanged(file, config) { config.statusInfo.files.updated.push(file); - info(file); + printName(file); } // --rename: the name of each file is the text that is searched and replaced, and the file is renamed to the result. // --rename-path: the same with the whole path, so the file can be moved. function renameFiles(files, config) { - // With --rename a / in the new name would move the file - const slashed = []; - // The new name of each file, before it is joined with its folder + const {plan, newNames} = planRenames(files, config); + + // With --output-match the matches are printed by the replacement, and the names go to stderr. + // The replacement is not used, so nothing would be renamed: no clash check and no renames in the status. + if (config.outputMatch) { + return plan.forEach(item => + item.new !== item.old + ? markFileChanged(item.old, config) + : config.nameAll && printName(item.old) + ); + } + + checkNameClash(plan, checkNameInvalid(plan, newNames, config), config); + + plan.forEach(item => { + if (item.new === item.old) { + verbose('Same name: ' + item.old); + return config.nameAll && nameRename(item, config); + } + // --simulate and --output do everything but rename + if (config.simulate || config.output) { + return renamed(item, config); + } + renameFile(item, config); + }); +} + +// The old and the new path of each file. The new name is also kept from before it is joined with its folder, +// as an empty name would join to the folder itself. +function planRenames(files, config) { const newNames = new Map(); const plan = files.map((file, index) => { config.statusInfo.files.searched.push(file); const text = config.renamePath ? file : path.basename(file); - chat('Work on the ' + (config.renamePath ? 'path' : 'name') + ' of: ' + file); + verbose('Work on the ' + (config.renamePath ? 'path' : 'name') + ' of: ' + file); const replacement = config.replacementJs && !config.outputMatch ? codeReplacement(file, config, text, fileNumbers(config, index + 1, files.length)) @@ -412,23 +428,21 @@ function renameFiles(files, config) { if (result === text) { return {old: file, new: file}; } - if (config.renamePath) { - const item = {old: file, new: result}; - newNames.set(item, path.basename(result)); - return item; - } - const item = {old: file, new: path.join(path.dirname(file), result)}; - newNames.set(item, result); - if (/[\\/]/.test(result)) { - slashed.push(item); - } + const item = { + old: file, + new: config.renamePath ? result : path.join(path.dirname(file), result), + }; + newNames.set(item, config.renamePath ? path.basename(result) : result); return item; }); - // Names that can not be made. Each one is a clash of its own, and is left out of the other checks. + return {plan, newNames}; +} + +// The names that can not be made, each with the reason. Each one is a clash of its own, and is left out of the other checks. +function checkNameInvalid(plan, newNames, config) { const invalid = []; - // A path with a control character is not welcome, also when the pattern would not change it plan.forEach(item => { - if (/[\x00-\x1f\x7f]/.test(item.old + item.new)) { + if (re.controlChar.test(item.old + item.new)) { invalid.push([ 'Names can not have control characters, like a line break or a tab.' + (config.replacementFile @@ -436,47 +450,34 @@ function renameFiles(files, config) { : ''), item, ]); - } else if (item.new !== item.old && /^[. ]*$|[. ]$/.test(newNames.get(item))) { - // Like an empty name, . or .., which are folders. Windows does not allow a space or a dot at the end. + } else if (item.new === item.old) { + return; + } else if (re.onlyDotsOrEndsWithDot.test(newNames.get(item))) { invalid.push([ 'A name must have more than dots and spaces, and can not end with a space or a dot.', item, ]); + } else if (!config.renamePath && re.slash.test(newNames.get(item))) { + // With --rename a / in the new name would move the file + invalid.push(['"/" is not allowed in names. Use --rename-path to move files.', item]); } }); - const renames = plan.filter(item => item.new !== item.old); - - // With --output-match the matches are printed by the replacement, and the names go to stderr. - // The replacement is not used, so nothing would be renamed: no clash check and no renames in the status. - if (config.outputMatch) { - return plan.forEach(item => - item.new !== item.old ? updated(item.old, config) : config.nameAll && info(item.old) - ); - } - - findClashes(plan, renames, slashed, invalid, config); + return invalid; +} - plan.forEach(item => { - if (item.new === item.old) { - chat('Same name: ' + item.old); - return config.nameAll && nameRename(item, config); - } - // --simulate and --output do everything but rename - if (config.simulate || config.output) { - config.statusInfo.files.updated.push(item.old); - config.statusInfo.renames.push(item); - return nameRename(item, config); - } - renameFile(item, config); - }); +// The file was renamed, or would be with --simulate and --output +function renamed(item, config) { + config.statusInfo.files.updated.push(item.old); + config.statusInfo.renames.push(item); + nameRename(item, config); } // With --simulate and --output the old name goes to stderr and the new one to stdout, so they come in pairs in a terminal function nameRename(item, config) { if (config.simulate || config.output) { - info(item.old, process.stderr); + printName(item.old, process.stderr); } - info(item.new, process.stdout); + printName(item.new, process.stdout); } // If planned renames clash the process is halted and no change is applied to any file. @@ -484,11 +485,11 @@ function nameRename(item, config) { // and the case is ignored, as `A.md` and `a.md` are the same file on a file system that is case-insensitive. const pathKey = file => path.resolve(file).toLowerCase(); -function findClashes(plan, renames, slashed, invalid, config) { +function checkNameClash(plan, invalid, config) { const byTarget = new Map(); - const left = new Set(invalid.map(([, item]) => item)); - renames.forEach(item => { - if (left.has(item)) { + const invalidItems = new Set(invalid.map(([, item]) => item)); + plan.forEach(item => { + if (item.new === item.old || invalidItems.has(item)) { return; } const target = pathKey(item.new); @@ -497,18 +498,12 @@ function findClashes(plan, renames, slashed, invalid, config) { // All files in the run, also the ones that keep their name const inRun = new Set(plan.map(item => pathKey(item.old))); const inRunExact = new Set(plan.map(item => path.resolve(item.old))); - const ontoRun = new Set(byTarget.keys()).intersection(inRun); // Each path on its own line, so long paths can be read const clash = (title, items) => title + items.map(item => '\n From: ' + item.old).join('') + '\n To: ' + items[0].new; - const clashes = [ - ...invalid.map(([title, item]) => clash(title, [item])), - ...slashed.map(item => - clash('"/" is not allowed in names. Use --rename-path to move files.', [item]) - ), - ]; + const clashes = invalid.map(([title, item]) => clash(title, [item])); byTarget.forEach((items, target) => { const [item] = items; // The new path is another file, also when it differs from the old one only in case (on a file system that is case-sensitive) @@ -516,10 +511,7 @@ function findClashes(plan, renames, slashed, invalid, config) { const self = pathKey(item.old) === target; if (1 < items.length) { clashes.push(clash('Renames to the same path are not allowed.', items)); - } else if ( - ontoRun.has(target) && - (!self || (other && inRunExact.has(path.resolve(item.new)))) - ) { + } else if (inRun.has(target) && (!self || (other && inRunExact.has(path.resolve(item.new))))) { // Not even --rename-force writes over a file in the run. A file can still change the case of its own name. clashes.push(clash('Renames clashing with other files in this run are not allowed.', items)); } else if (other && !config.renameForce) { @@ -553,7 +545,7 @@ function renameFile(item, config) { try { const folder = path.dirname(target); if (!fs.existsSync(folder)) { - chat('Make folder: ' + path.dirname(item.new)); + verbose('Make folder: ' + path.dirname(item.new)); fs.mkdirSync(folder, {recursive: true}); } // Checked again, as the files can have changed since the clashes were found @@ -561,63 +553,28 @@ function renameFile(item, config) { if (!config.renameForce) { return error(item.new + ' is there already, so ' + item.old + ' is not renamed'); } - chat('Write over: ' + item.new); + verbose('Write over: ' + item.new); } // With --keep-backup the old file is kept, so the file is copied if (config.keepBackup) { - chat('Copy: ' + item.old + ' to ' + item.new); + verbose('Copy: ' + item.old + ' to ' + item.new); fs.copyFileSync(item.old, target); } else { - chat('Rename: ' + item.old + ' to ' + item.new); + verbose('Rename: ' + item.old + ' to ' + item.new); fs.renameSync(item.old, target); } } catch (e) { return error(e); } - config.statusInfo.files.updated.push(item.old); - config.statusInfo.renames.push(item); - info(item.new); -} - -// The checks from --expect-* are done when the run is done. Piped data counts as one file. -function checkExpectations(config) { - const status = config.statusInfo; - if (!status.checks) { - return; - } - const piped = Boolean(config.dataIsPiped); - const found = piped ? 1 : status.files.searched.length; - const changed = piped ? Number(status.pipe.updated) : status.files.updated.length; - // Nothing is written with --simulate, --output and --output-match - const would = !piped && (config.simulate || config.output || config.outputMatch); - const files = count => (piped ? 'the piped data' : count + (1 === count ? ' file' : ' files')); - const change = count => - would ? 'would change' : 1 === count || piped ? 'was changed' : 'were changed'; - - if (config.expectFiles && !found) { - checkFailed('Expected files, but 0 files found'); - } - if ((config.expectDiff || config.expectDiffAll) && !changed) { - checkFailed('Expected a diff, but nothing ' + (would ? 'would change' : 'was changed')); - } else if (config.expectDiffAll && changed < found) { - const unchanged = found - changed; - checkFailed( - 'Expected all files to change, but ' + - files(unchanged) + - (would ? ' would not change' : 1 === unchanged ? ' was not changed' : ' were not changed') - ); - } - if (config.expectZeroDiff && changed) { - checkFailed('Expected no diff, but ' + files(changed) + ' ' + change(changed)); - } + renamed(item, config); } // How the piped data was used -function pipeType(config, piped) { +function pipeType(config) { if (config.globPipe) { return 'GLOBS'; } - if (piped) { + if (config.dataIsPiped) { return 'CONTENT'; } if (config.replacementJs) { @@ -629,82 +586,40 @@ function pipeType(config, piped) { return 'UNUSED'; } -function handlePipedData(config) { - // With files the piped data is only read when a flag asks for it, and then the flag uses it - if (config.includeGlob.length) { - return false; - } - - if (null !== config.pipedData && !config.pipedDataUsed) { - config.dataIsPiped = true; +// Piped data is the content to search when no files are given and no flag uses it for something else +function usePipeAsContent(config) { + config.dataIsPiped = + !config.includeGlob.length && null !== config.pipedData && !config.pipedDataUsed; + if (config.dataIsPiped) { // Piped data is printed, unless only the matches are config.output = !config.outputMatch; - return true; } - - return false; } -function getFinalPattern(pattern, conf: any) { +function regexPattern(pattern, conf: any) { if (conf.literal) { pattern = pattern.replace(re.regexSpecialChars, '\\$&'); } - /*if (config.patternFile) { - pattern = fs.readFileSync(pattern, 'utf8'); - pattern = new Function('return '+pattern)(); - }*/ - return pattern; } -function getFinalReplacement(replacement, conf: any) { - if (conf.replacementPipe) { - conf.pipedDataUsed = true; - if (null === conf.pipedData) { +// The replacement as text: from the pipe, from --replacement-file, or as it was typed +function readReplacement(config) { + let replacement = config.replacement; + if (config.replacementPipe) { + config.pipedDataUsed = true; + if (null === config.pipedData) { return panic('No data piped into replacement'); } - replacement = conf.pipedData; - } - - if (conf.outputMatch) { - if (conf.replacementFile) { - warn( - 'The replacement file is not read with --output-match, as only the matches are printed.' - ); - } - if (parseInt(process.versions.node) < 6) { - return panic('outputMatch is only supported in node 6+'); - } - - return function (match, ...rest) { - // The groups are followed by the offset and the full text, and by the named groups if the pattern has any. - // Without groups the full match is printed, otherwise the groups are printed together. - const groups = rest.slice(0, 'object' === typeof rest[rest.length - 1] ? -3 : -2); - print( - groups.length ? groups.join('') : match, - conf.contentDelimiter, - conf.contentDelimiterTail - ); - return ''; - }; - } - - // If captured groups then run dynamicly - //console.log(process); - if ( - conf.replacementJs && - re.capturedGroupRef.test(conf.replacement) && - parseInt(process.versions.node) < 6 - ) { - return panic('Captured groups for javascript replacement is only supported in node 6+'); + replacement = config.pipedData; } // --replacement-file: the replacement is the path to the file with the replacement. - // With --js the code gives the path, so the file is read when the code has run. - if (conf.replacementFile && !conf.replacementJs) { + // With --js the code gives the path, so the file is read when the code has run. With --output-match it is not used. + if (config.replacementFile && !config.replacementJs && !config.outputMatch) { try { - return readReplacementFile(replacement, conf); + return readReplacementFile(replacement, config); } catch (e) { return panic('The replacement file can not be read:', pathAndReason(replacement, e)); } @@ -713,6 +628,22 @@ function getFinalReplacement(replacement, conf: any) { return replacement; } +// --output-match prints each match: the full match, or the groups together when the pattern has groups +function matchPrinter(config) { + if (config.replacementFile) { + warn('The replacement file is not read with --output-match, as only the matches are printed.'); + } + return (...args) => { + const [match, ...groups] = splitMatch(args).groups; + print( + groups.length ? groups.join('') : match, + config.contentDelimiter, + config.contentDelimiterTail + ); + return ''; + }; +} + // The path and why it can not be read, like `nope.txt (ENOENT: no such file or directory)`. // A path that is not clear as it is printed is shown as JSON, with each char that can not be seen escaped. function pathAndReason(filepath, e) { @@ -729,9 +660,8 @@ function pathAndReason(filepath, e) { // Each replacement file is read once, also when the code of --js gives the same path for many matches function readReplacementFile(filepath, conf) { - conf.replacementFiles ??= new Map(); if (!conf.replacementFiles.has(filepath)) { - chat('Read the replacement file: ' + filepath); + verbose('Read the replacement file: ' + filepath); const content = fs.readFileSync(filepath, conf.encoding); conf.replacementFiles.set(filepath, conf.trimFile ? content.trim() : content); } @@ -745,7 +675,6 @@ function codeReplacement(file, config, text, numbers) { return code; } // A path that can not be read is an error once in the run, and its matches are left as they are - config.unreadableFiles ??= new Set(); return (...args) => { const filepath = String(code(...args)); if (!config.unreadableFiles.has(filepath)) { @@ -760,77 +689,27 @@ function codeReplacement(file, config, text, numbers) { }; } -/*function oneLinerFromFile(str){ - let lines = str.split("\n"); - if(lines.length===1){ - return str; - } - return lines.map(function (line) { - return line.trim(); - }).join(' '); -}*/ - function getFinalRegex(pattern, config) { const engine = 'default' === config.engine ? runtimeEngine : config.engine; - - let regex; - - let flags = getFlags(config); - - switch (engine) { - case runtimeEngine: - try { - regex = new RegExp(pattern, flags); - } catch (e) { - panic(e.message); - } - break; - case 'V8': - case 'JSC': - panic( - `When running via the ${runtime} runtime you can only use default or ${runtimeEngine} as engine` - ); - break; - case 'RE2': - try { - //const RE2 = require('re2'); - //regex = new RE2(pattern, flags); - regex = new RegExp(pattern, flags); - } catch (e) { - panic(e.message); - } - break; - default: - panic(`Engine ${config.engine} not supported`); - } - - return regex; -} - -function getFlags(config) { - let flags = ''; - - if (!config.voidGlobal) { - flags += 'g'; - } - - if (!config.voidIgnoreCase) { - flags += 'i'; - } - - if (!config.voidMultiline) { - flags += 'm'; - } - - if (config.dotAll) { - flags += 's'; + // RE2 is not in yet, so it uses the engine of the runtime + if (engine !== runtimeEngine && 'RE2' !== engine) { + panic( + 'V8' === engine || 'JSC' === engine + ? `When running via the ${runtime} runtime you can only use default or ${runtimeEngine} as engine` + : `Engine ${config.engine} not supported` + ); } - - if (config.unicode) { - flags += 'u'; + const flags = + (config.voidGlobal ? '' : 'g') + + (config.voidIgnoreCase ? '' : 'i') + + (config.voidMultiline ? '' : 'm') + + (config.dotAll ? 's' : '') + + (config.unicode ? 'u' : ''); + try { + return new RegExp(pattern, flags); + } catch (e) { + return panic(e.message); } - - return flags; } function readableSize(size) { @@ -852,7 +731,7 @@ function fileNumbers(config, number, total) { // When a file is done, it counts for $i if the pattern found something, and for $c if it was changed. // It takes one more search of the file, so it is only done when the code uses $i or $c. function countFile(config, text, result) { - if (!config.countFiles) { + if (!config.codeUses?.countFiles) { return; } if (-1 !== text.search(config.regex)) { @@ -863,15 +742,33 @@ function countFile(config, text, result) { } } -// The code uses a named group of the pattern, like rr_year for (?…) -function usesNamedGroups(config) { - return namedGroups(config.regex).some(name => +// What the code of --js names. It is found once, with a plain text search, so it can be fooled, like by `eval('$' + 'n')`. +function codeUses(code, regex) { + const numbers = new Set( + ['n', 't', 'i', 'c', 'm', 'M'].filter(name => + new RegExp('\\$' + name + '(?![A-Za-z])').test(code) + ) + ); + const groups = namedGroups(regex); + // A named group is used as rr_year, or as rr_year_ with a space at the end + const usesGroup = name => new RegExp( - // rr_year_ is the group with a space at the end, so the code uses the group then too '(^|[^\\p{ID_Continue}$])rr_' + name.replace(/\$/g, '\\$') + '_?(?![\\p{ID_Continue}$])', 'u' - ).test(config.replacementOri) - ); + ).test(code); + const matchNumbers = numbers.has('m') || numbers.has('M'); + return { + numbers, + groups, + // $i and $c need one more search of each file, so the files are only counted when the code uses them + countFiles: numbers.has('i') || numbers.has('c'), + // $i, $c, $m and $M count what came before + countsBefore: matchNumbers || numbers.has('i') || numbers.has('c'), + // The groups, the named groups, $m and $M are about the match, so the code runs once per match + perMatch: matchNumbers || re.capturedGroupRef.test(code) || groups.some(usesGroup), + stat: re.mctime.test(code), + size: re.byteOrSize.test(code), + }; } // The names of the named groups of a regex, like year for (?…). A name can have any letter, like año. @@ -885,9 +782,9 @@ function namedGroups(regex) { // Code that makes the numbers and their cousins, put in front of the code of --js. // $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, and the ones with _ at the end have a space at the end. // Only the numbers the code uses are made, as each one costs time when the code runs once per match. -function numbersCode(numbers, code) { +function numbersCode(numbers, used) { return Object.entries(numbers) - .filter(([name]) => new RegExp('\\$' + name + '(?![A-Za-z])').test(code)) + .filter(([name]) => used.has(name)) .map(([name, value]) => { const cousins = [`$${name} = ${value}`, `$${name}_ = '${value} '`]; for (let width = 2; width <= 10; width++) { @@ -905,13 +802,7 @@ function numbersCode(numbers, code) { function stackTraceLocations(stackTrace) { return String(stackTrace) .split('\n') - .map( - line => - // Without a function name the location comes right after at, like `at /dir/a (copy)/anon.js:1:59` - line.match(/^\s*at (?:file:\/\/)?((?:\/|[A-Za-z]:[\\/]).*?)(:\d+:\d+)\s*$/) || - // With a name the location is in parentheses at the end, like `at fn (/path/file.js:2:15)` - line.match(/^\s*at .*? \((?:file:\/\/)?(.+?)(:\d+:\d+)\)\s*$/) - ) + .map(line => line.match(re.stackLineWithoutName) || line.match(re.stackLineWithName)) .filter(Boolean) .map(([, file, position]) => ({file, position})); } @@ -932,7 +823,7 @@ function userCodeLocation(thrown) { return ''; } // A file is an absolute path, so the frames of code run with eval are skipped too - if (/^(\/|[A-Za-z]:[\\/])/.test(file) && !file.includes('node_modules')) { + if (re.absolutePath.test(file) && !file.includes('node_modules')) { return path.relative(process.cwd(), file) + position; } } @@ -940,34 +831,31 @@ function userCodeLocation(thrown) { return ''; } -// The text of what code threw. Code can throw anything, also a value that can not be made into text. -function thrownText(thrown) { - let text = ''; +// Reading a value from the code of the user can throw, so it is tried +function tryOr(read, fallback = '') { try { - text = String(thrown); - } catch (e) {} + return read() ?? fallback; + } catch (e) { + return fallback; + } +} + +// What the code threw, as text, like `Error: x` or `a Foo was thrown: {"a":1}`. Code can throw anything. +function thrownText(thrown) { + const text = tryOr(() => String(thrown)); if (text && '[object Object]' !== text) { return text; } - if ('object' !== typeof thrown || null === thrown) { - return 'an empty ' + typeof thrown + ' was thrown'; + // Any other value that is not an object gives text + if ('' === thrown) { + return 'an empty string was thrown'; } - // The name of the class, like an Error whose message can not be read, or a class of the user - let name = ''; - try { - name = String(thrown.constructor?.name ?? ''); - } catch (e) {} - let json = ''; - try { - json = JSON.stringify(thrown) ?? ''; - } catch (e) {} + // The class, like a class of the user, or an Error whose message can not be read + const name = tryOr(() => String(thrown.constructor.name)); const what = name && 'Object' !== name ? name : 'object'; - return cut( - (/^[aeiou]/i.test(what) ? 'an ' : 'a ') + - what + - ' was thrown' + - (json && '{}' !== json ? ': ' + json : '') - ); + const json = tryOr(() => JSON.stringify(thrown)); + const article = re.startsWithVowel.test(what) ? 'an' : 'a'; + return cut(`${article} ${what} was thrown` + (json && '{}' !== json ? ': ' + json : '')); } // The thrown value can hold a whole file, so only the start is told. It is cut by characters, so an emoji is not split. @@ -976,119 +864,115 @@ function cut(text, length = 300) { return chars.length > length ? chars.slice(0, length).join('') + '…' : text; } -function dynamicReplacement( - _file_rr, - _config_rr, - _data_rr, - _numbers_rr = {n: 1, t: 1, i: 1, c: 1} -) { - const _time_obj = now; - const _time = localTimeString(_time_obj); - const _pipe = _config_rr.pipedData, - _text = _data_rr, - _find = _config_rr.pattern, - // The code of the user is in a block of its own, so it can declare the same names with let and const. - // The value of the block is the value of its last statement, like before. The line break ends a comment at the end. - code_rr = - numbersCode(_numbers_rr, _config_rr.replacementOri) + '{' + _config_rr.replacementOri + '\n}', - _cwd = process.cwd(), - _now = _time, - _ = ' ', - _nl = '\n'; +// The arguments of a replace function: the match and the groups, then the offset and the text, and the named groups if the pattern has any +function splitMatch(args) { + const named = args[args.length - 1]; + const hasNamed = 'object' === typeof named && null !== named; + return {groups: args.slice(0, hasNamed ? -3 : -2), named: hasNamed ? named : {}}; +} + +// The function that runs the code of --js. It is made once, as the names are the same for every file. +let codeFunction = null; +function dynamicReplacement(file, config, text, numbers = {n: 1, t: 1, i: 1, c: 1}) { + const cwd = process.cwd(); + const time = localTimeString(config.startTime); // biome-ignore format: one variable on each line - let _file = '❌', - _file_rel = '❌', - _dirpath = '❌', - _dirpath_rel = '❌', - _dirname = '❌', - _filename = '❌', - _name = '❌', - _ext = '❌', - _mtime = '❌', - _ctime = '❌', - _mtime_obj = new Date(0), - _ctime_obj = new Date(0), - _bytes = -1, - _size = '❌', - dynamicContent = new Function( - 'require', - 'fs', - 'globs', - 'path', - - 'pipe', - 'pipe_', - - 'find', - 'find_', - 'text', - 'text_', - - 'file', - 'file_', - 'file_rel', - 'file_rel_', - - 'dirpath', - 'dirpath_', - 'dirpath_rel', - 'dirpath_rel_', - - 'dirname', - 'dirname_', - 'filename', - 'filename_', - 'name', - 'name_', - 'ext', - 'ext_', - 'cwd', - 'cwd_', - - 'now', - 'now_', - 'time_obj', - 'time', - 'time_', - 'mtime_obj', - 'mtime', - 'mtime_', - 'ctime_obj', - 'ctime', - 'ctime_', - - 'bytes', - 'bytes_', - 'size', - 'size_', - 'nl', - '_', - '__code_rr', - 'var path = require("path");' + - 'var __require_ = require;' + - 'var r = function(file){' + - 'var result = null;' + - 'try{' + - 'result = __require_(file);' + - '} catch (e){' + - 'var dir = /^[\\\/]/.test(file) ? "" : cwd;' + - 'result = __require_(path.resolve(dir, file));' + - '};' + - 'return result;' + - '};' + - 'require = r;' + - 'return eval(__code_rr);' - ); + const values = { + pipe: config.pipedData, + find: config.pattern, + text, + file: '❌', + file_rel: '❌', + dirpath: '❌', + dirpath_rel: '❌', + dirname: '❌', + filename: '❌', + name: '❌', + ext: '❌', + cwd, + now: time, + time, + mtime: '❌', + ctime: '❌', + bytes: -1, + size: '❌', + }; + let mtime_obj = new Date(0); + let ctime_obj = new Date(0); + + if (!config.dataIsPiped) { + const full = path.resolve(cwd, file); + const {dir, base, name, ext} = path.parse(full); + Object.assign(values, { + file: full, + file_rel: path.relative(cwd, full), + dirpath: dir, + dirpath_rel: path.relative(cwd, dir), + dirname: (full.match(re.folderName) || ' _')[1], + filename: base, + name, + ext, + }); + // From around 50 MB the size is read from the file instead of counted in the text + if (config.codeUses.stat || (config.codeUses.size && 50000000 < text.length)) { + const stats = fs.statSync(full); + mtime_obj = stats.mtime; + ctime_obj = stats.ctime; + values.bytes = stats.size; + values.mtime = localTimeString(mtime_obj); + values.ctime = localTimeString(ctime_obj); + } + } + if (config.codeUses.size && -1 === values.bytes) { + // The size in the encoding of the file, not of the text as utf8 + values.bytes = Buffer.byteLength(text, config.encoding); + } + if (-1 !== values.bytes) { + values.size = readableSize(values.bytes); + } + + // require, and r for short, also find a file from the cwd when the path does not start with ./ + const r = module => { + try { + return require(module); + } catch (e) { + return require(path.resolve(cwd, module)); + } + }; + // Each value but the modules, the date objects, nl and _ has a cousin with _ at the end, with a space at the end. + // Without piped data `pipe` is null, while `pipe_` is `❌ ` like the file variables. + const params = { + require: r, + r, + fs, + globs, + path, + time_obj: config.startTime, + mtime_obj, + ctime_obj, + nl: '\n', + _: ' ', + }; + for (const [name, value] of Object.entries(values)) { + params[name] = value; + params[name + '_'] = (value ?? '❌') + ' '; + } + codeFunction ??= new Function(...Object.keys(params), '__code_rr', 'return eval(__code_rr);'); + const args = Object.values(params); + + // The code of the user is in a block of its own, so it can declare the same names with let and const. + // The value of the block is the value of its last statement, like before. The line break ends a comment at the end. + const code = numbersCode(numbers, config.codeUses.numbers) + '{' + config.replacementOri + '\n}'; // When the code fails, only the message is told, and not the stack trace with all the code put in front of the code of the user - const runCode = (...args) => { + const runCode = variables => { try { - return dynamicContent(...args); + return codeFunction(...args, variables + code); } catch (e) { - const where = _config_rr.dataIsPiped ? 'the piped data' : _file_rr; + const where = config.dataIsPiped ? 'the piped data' : file; // With --verbose it also tells where the code failed, when that is in a file of the user, like a module loaded with r() - const at = _config_rr.verbose ? userCodeLocation(e) : ''; + const at = config.verbose ? userCodeLocation(e) : ''; return panic( 'The code of --js failed for ' + where + @@ -1099,201 +983,25 @@ function dynamicReplacement( } }; - const needsByteOrSize = re.byteOrSize.test(_config_rr.replacement); - const betterToReadfromFile = needsByteOrSize && 50000000 < _text.length; // around 50 Mb will lead to reading filezise from file instead of copying into buffer - - if (!_config_rr.dataIsPiped) { - _file = path.resolve(_cwd, _file_rr); - _file_rel = path.relative(_cwd, _file); - const pathInfo = path.parse(_file); - _dirpath = pathInfo.dir; - _dirpath_rel = path.relative(_cwd, _dirpath); - _dirname = (_file.match(re.folderName) || ' _')[1]; - _filename = pathInfo.base; - _name = pathInfo.name; - _ext = pathInfo.ext; - - if (betterToReadfromFile || re.mctime.test(_config_rr.replacement)) { - const fileStats = fs.statSync(_file); - _bytes = fileStats.size; - _size = readableSize(_bytes); - _mtime_obj = fileStats.mtime; - _ctime_obj = fileStats.ctime; - _mtime = localTimeString(_mtime_obj); - _ctime = localTimeString(_ctime_obj); - - //console.log('filesize: ', fileStats.size); - //console.log('dataSize: ', _bytes); - } - } - - if (needsByteOrSize && -1 === _bytes) { - // The size in the encoding of the file, not of the text as utf8 - _bytes = Buffer.byteLength(_text, _config_rr.encoding); - _size = readableSize(_bytes); - } - - // Run only once if no captured groups (replacement cant change). - // The result is given back from a function, so `$&` and `$$` in it are inserted as they are and not expanded. - // $m, $M and the named groups are about the match, so they also need the code to run once per match - if ( - !/\$\d/.test(_config_rr.replacement) && - !re.matchNumbers.test(_config_rr.replacement) && - !usesNamedGroups(_config_rr) - ) { - const result = runCode( - require, - fs, - globs, - path, - _pipe, - // Without piped data `pipe` is null, while `pipe_` is `❌ ` like the file variables - (_pipe ?? '❌') + _, - _find, - _find + _, - _text, - _text + _, - _file, - _file + _, - _file_rel, - _file_rel + _, - _dirpath, - _dirpath + _, - _dirpath_rel, - _dirpath_rel + _, - _dirname, - _dirname + _, - _filename, - _filename + _, - _name, - _name + _, - _ext, - _ext + _, - _cwd, - _cwd + _, - _now, - _now + _, - _time_obj, - _time, - _time + _, - _mtime_obj, - _mtime, - _mtime + _, - _ctime_obj, - _ctime, - _ctime + _, - _bytes, - _bytes + _, - _size, - _size + _, - _nl, - _, - code_rr - ); + // The value is given back from a function, so `$&` and `$$` in it are inserted as they are and not expanded + if (!config.codeUses.perMatch) { + const result = runCode(''); return () => result; } - // Capture groups used, so need to run once per match - let _match_rr = 0; - return function () { - const __pipe = _pipe, - __text = _text, - __find = _find, - __file = _file, - __file_rel = _file_rel, - __dirpath = _dirpath, - __dirpath_rel = _dirpath_rel, - __dirname = _dirname, - __filename = _filename, - __name = _name, - __ext = _ext, - __cwd = _cwd, - __now = _now, - __time_obj = _time_obj, - __time = _time, - __mtime_obj = _mtime_obj, - __mtime = _mtime, - __ctime_obj = _ctime_obj, - __ctime = _ctime, - __bytes = _bytes, - __size = _size, - __nl = _nl, - __ = _, - __code_rr = code_rr; - - _match_rr++; - _config_rr.numbers.matches++; - var capturedGroups = numbersCode( - {m: _match_rr, M: _config_rr.numbers.matches}, - _config_rr.replacementOri - ); - // The match and the groups are followed by the offset and the full text, and by the named groups if the pattern has any - const named = arguments[arguments.length - 1]; - const hasNamed = 'object' === typeof named && null !== named; - for (var i = 0; i < arguments.length - (hasNamed ? 3 : 2); i++) { - capturedGroups += 'var $' + i + '=' + JSON.stringify(arguments[i]) + '; '; - } + let match = 0; + return (...matchArgs) => { + match++; + config.numbers.matches++; + const {groups, named} = splitMatch(matchArgs); + let variables = numbersCode({m: match, M: config.numbers.matches}, config.codeUses.numbers); + groups.forEach((value, i) => { + variables += 'var $' + i + '=' + JSON.stringify(value) + '; '; + }); // A named group is a variable with rr_ in front, like rr_year for (?…), and rr_year_ has a space at the end - if (hasNamed) { - for (const [name, value] of Object.entries(named)) { - capturedGroups += `var rr_${name} = ${JSON.stringify(value)}, rr_${name}_ = rr_${name} + ' '; `; - } + for (const [name, value] of Object.entries(named)) { + variables += `var rr_${name} = ${JSON.stringify(value)}, rr_${name}_ = rr_${name} + ' '; `; } - - return runCode( - require, - fs, - globs, - path, - - __pipe, - (__pipe ?? '❌') + __, - - __find, - __find + __, - __text, - __text + __, - - __file, - __file + __, - __file_rel, - __file_rel + __, - - __dirpath, - __dirpath + __, - __dirpath_rel, - __dirpath_rel + __, - - __dirname, - __dirname + __, - __filename, - __filename + __, - __name, - __name + __, - __ext, - __ext + __, - __cwd, - __cwd + __, - - __now, - __now + _, - __time_obj, - __time, - __time + _, - __mtime_obj, - __mtime, - __mtime + _, - __ctime_obj, - __ctime, - __ctime + _, - - __bytes, - __bytes + __, - __size, - __size + __, - __nl, - __, - capturedGroups + __code_rr - ); + return runCode(variables); }; } @@ -1327,12 +1035,12 @@ function getFilePaths(conf) { } excludeRe - .map(el => getFinalRegex(getFinalPattern(el, conf), conf)) - .forEach(re => { - files = files.filter(file => !file.match(re)); + .map(pattern => getFinalRegex(regexPattern(pattern, conf), conf)) + .forEach(exclude => { + files = files.filter(file => !file.match(exclude)); }); - files = sortFiles(files, sort, conf); + files = sortFiles(files, sort); return conf.sortReverse ? files.reverse() : files; } @@ -1359,7 +1067,7 @@ const collator = new Intl.Collator('en', {numeric: true, sensitivity: 'base'}).c const natural = (a, b) => collator(a, b) || ascii(a, b); const ascii = (a, b) => (a < b ? -1 : a > b ? 1 : 0); -function sortFiles(files, sort, conf) { +function sortFiles(files, sort) { // One file has no order, so it is not sorted, and needs no stat if (files.length < 2 || 'none' === sort) { return files; @@ -1404,16 +1112,16 @@ function sortFiles(files, sort, conf) { return files.sort((a, b) => value(a) - value(b) || ascii(a, b)); } -// Is the path a file (true), something to glob (false), or a path that can not be looked at (null, with an error)? +// Is the path a file, something to glob, or a path that can not be looked at (null, with an error)? // A path that is not there, or has a file where a folder should be, is globbed like before. So is a glob, // like a long brace glob that is too long as a name. A plain path that can not be looked at, like a name // that is too long or a symlink loop, is an error, so it is not hidden as 0 files found. -function isFile(file) { +function pathKind(file) { try { - return Boolean(fs.statSync(file, {throwIfNoEntry: false})?.isFile()); + return fs.statSync(file, {throwIfNoEntry: false})?.isFile() ? 'file' : 'glob'; } catch (e) { if ('ENOENT' === e.code || 'ENOTDIR' === e.code || globs.isDynamicPattern(file)) { - return false; + return 'glob'; } error(e.message); return null; @@ -1441,10 +1149,10 @@ function findFiles(paths, globOptions, sort) { const files = []; const patterns = []; paths.forEach(file => { - const kind = isFile(file); - if (kind) { + const kind = pathKind(file); + if ('file' === kind) { files.push(file); - } else if (false === kind) { + } else if ('glob' === kind) { patterns.push(file); } }); diff --git a/src/output.ts b/src/output.ts index 6486632a..0d5f719b 100644 --- a/src/output.ts +++ b/src/output.ts @@ -1,8 +1,3 @@ -let font: any = {}; -font.red = font.green = font.gray = str => str; -// check for node version supporting chalk - if so overwrite `font` -//const font = import('chalk'); - let config: any = null; // The char code of the last character of the last item printed to each stream (10 is a line break). Names and content never share a stream. @@ -17,12 +12,6 @@ export const outputConfig = function (_config) { lastPrintedChar.clear(); }; -// Thrown when the run can not go on. The engine adds the status so far, and the CLI ends the process. -export class Panic extends Error { - panic = true; - status = null; -} - // A pipe the reader has closed, like `| head`, can not take more. That is fine: nobody reads the rest. // Bun throws when it is written to, and Node gives an error event, so both are caught. const closed = new Set(); @@ -85,7 +74,7 @@ export const printDone = function () { }; // The name of a changed file goes to stdout so the names can be piped -export const info = function (name, stream = null) { +export const printName = function (name, stream = null) { if (config.quiet || config.quietTotal) { return; } @@ -94,72 +83,24 @@ export const info = function (name, stream = null) { print(name, config.nameDelimiter, config.nameDelimiterTail, stream); }; -const note = function (msg) { - if (config.quiet || config.quietTotal) { - return; +export const printWarning = function (msg) { + if (!config.quiet && !config.quietTotal) { + // The second line break gives an empty line under the warning + write(' 🟡 ' + msg + '\n\n', process.stderr); } - write(font.gray(msg) + '\n', process.stderr); -}; - -export const warn = function (msg) { - config.statusInfo?.warnings.push(msg); - // The line break gives an empty line under the warning - note(' 🟡 ' + msg + '\n'); }; // --verbose was asked for, so --quiet does not hide it. --quiet-total wins and hides it too. -// The status report keeps the same lines in `engine.steps`, so they are there even when they are not printed. -export const chat = function (msg) { - config.statusInfo?.engine?.steps.push(msg); +export const printVerbose = function (msg) { if (config.verbose && !config.quietTotal) { - write(font.gray(msg) + '\n', process.stderr); + write(msg + '\n', process.stderr); } }; -// An error is told about, and goes into `errors` in the status report. A panic also says the run stops here. -const printError = function (text, stop) { +// A panic also says the run stops here +export const printError = function (text, stop) { if (!config.quietTotal) { // The line break gives an empty line under the error - write( - font.red(' ❌ ' + text + (stop ? '\n No further processing.' : '')) + '\n\n', - process.stderr - ); - } -}; - -// The run can not go on, so it stops here. -// The files in work are finished, and they can fail too. Only the first panic is told, the rest are only in the status. -export const panic = function (msg = '', data = '', displayHelp = false) { - const text = msg + (data ? ' ' + data : ''); - config.statusInfo?.errors.push(text); - if (!config.stopped) { - config.stopped = true; - if (displayHelp && !config.quietTotal) { - config.showHelp(); - } - printError(text, true); - } - throw new Panic(text); -}; - -// A check from --expect-* that is not met. It is an error, and is also in `checks`, so the exit code can tell it from the other errors. -export const checkFailed = function (text) { - config.statusInfo?.errors.push(text); - config.statusInfo?.checks.push(text); - printError(text, false); -}; - -// Something failed, and the run goes on. With --halt it is a panic. -export const error = function (msg, data = '') { - const text = msg + (data ? ' ' + data : ''); - config.statusInfo?.errors.push(text); - // After a panic only the status is told about the files that are finished - if (!config.stopped) { - printError(text, config.halt); - } - if (config.halt) { - config.stopped = true; - throw new Panic(text); + write(' ❌ ' + text + (stop ? '\n No further processing.' : '') + '\n\n', process.stderr); } - return false; }; diff --git a/src/shim.ts b/src/shim.ts deleted file mode 100644 index d6142fea..00000000 --- a/src/shim.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Set.prototype.intersection() came in Node 22, so it is added here for older runtimes - -const proto: any = Set.prototype; - -proto.intersection ??= function (other) { - return new Set([...this].filter(item => other.has(item))); -}; diff --git a/src/status.ts b/src/status.ts new file mode 100644 index 00000000..957862c7 --- /dev/null +++ b/src/status.ts @@ -0,0 +1,93 @@ +import {outputConfig, printError, printVerbose, printWarning} from './output.ts'; + +let config: any = null; + +// The config of the run, for the status and for what is printed +export const statusConfig = function (_config) { + config = _config; + outputConfig(_config); +}; + +// Thrown when the run can not go on. The engine adds the status so far, and the CLI ends the process. +export class Panic extends Error { + panic = true; + status = null; +} + +export const warn = function (msg) { + config.statusInfo?.warnings.push(msg); + printWarning(msg); +}; + +// The status report keeps the lines of --verbose in `engine.steps`, so they are there even when they are not printed +export const verbose = function (msg) { + config.statusInfo?.engine?.steps.push(msg); + printVerbose(msg); +}; + +// The run can not go on, so it stops here. +// The files in work are finished, and they can fail too. Only the first panic is told, the rest are only in the status. +export const panic = function (msg = '', data = '') { + const text = msg + (data ? ' ' + data : ''); + config.statusInfo?.errors.push(text); + if (!config.plzStop) { + config.plzStop = true; + printError(text, true); + } + throw new Panic(text); +}; + +// A check from --expect-* that is not met. It is an error, and is also in `checks`, so the exit code can tell it from the other errors. +const checkFailed = function (text) { + config.statusInfo?.errors.push(text); + config.statusInfo?.checks.push(text); + printError(text, false); +}; + +// Something failed, and the run goes on. With --halt it is a panic. +export const error = function (msg, data = '') { + const text = msg + (data ? ' ' + data : ''); + config.statusInfo?.errors.push(text); + // After a panic only the status is told about the files that are finished + if (!config.plzStop) { + printError(text, config.halt); + } + if (config.halt) { + config.plzStop = true; + throw new Panic(text); + } + return false; +}; + +// The checks from --expect-* are done when the run is done. Piped data counts as one file. +export function checkExpectations(config) { + const status = config.statusInfo; + if (!status.checks) { + return; + } + const piped = Boolean(config.dataIsPiped); + const found = piped ? 1 : status.files.searched.length; + const changed = piped ? Number(status.pipe.updated) : status.files.updated.length; + // Nothing is written with --simulate, --output and --output-match + const would = !piped && (config.simulate || config.output || config.outputMatch); + const files = count => (piped ? 'the piped data' : count + (1 === count ? ' file' : ' files')); + const change = count => + would ? 'would change' : 1 === count || piped ? 'was changed' : 'were changed'; + + if (config.expectFiles && !found) { + checkFailed('Expected files, but 0 files found'); + } + if ((config.expectDiff || config.expectDiffAll) && !changed) { + checkFailed('Expected a diff, but nothing ' + (would ? 'would change' : 'was changed')); + } else if (config.expectDiffAll && changed < found) { + const unchanged = found - changed; + checkFailed( + 'Expected all files to change, but ' + + files(unchanged) + + (would ? ' would not change' : 1 === unchanged ? ' was not changed' : ' were not changed') + ); + } + if (config.expectZeroDiff && changed) { + checkFailed('Expected no diff, but ' + files(changed) + ' ' + change(changed)); + } +}