From 9e1a10d2e1cb2dc3a41a5611a325635eafe6c31b Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 21:03:15 +1100 Subject: [PATCH] Add file and match numbers to --js --- README.md | 10 ++++- src/cli.ts | 4 +- src/engine.ts | 94 ++++++++++++++++++++++++++++++++++++-------- test/cli/run.sh | 4 ++ test/fix/791.test.js | 55 ++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 test/fix/791.test.js diff --git a/README.md b/README.md index a2f442ed..56099b4e 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,14 @@ archive/a.log archive/b.log ``` +Number the files with `$n` in the code. `$n00` has three digits, and `--sort` decides the order: + +```bash +> rexreplace '^.*$' '"photo-" + $n00 + ext' '*.jpg' --rename --js --sort modified +photo-001.jpg +photo-002.jpg +``` + See what would happen first with `--simulate`. Each old name is printed (to stderr) just before its new name (to stdout), and nothing is renamed. ```bash @@ -358,7 +366,7 @@ If planned renames clash, the process is halted and no change is applied to any | `-g` | **`--glob-pipe`** Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments. [boolean] | | | **`--null-in`** The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out. [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --js-pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. The numbers: `$n`: the number of the file, `$t`: the number of files, `$i`: the number of the file among the files with a match so far, `$c`: among the files changed so far, `$m`: the number of the match in the file, `$M`: across all files. Each has cousins with zeros in front (`$n0` has 2 digits, `$n00` 3, and so on to 10) and with a space at the end (`$n_`). All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Files with a path that matches this regular expression will be excluded from processing. Can be provided multiple times. [string] | | `-X` | **`--exclude-glob`** Files found with this glob will be excluded from being processed. Can be provided multiple times and will act as one group of globs, meaning you can use ! notation to negate. [string] | | `-h` | **`--help`** Display help. [boolean] | diff --git a/src/cli.ts b/src/cli.ts index a281c517..39fefacf 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -407,7 +407,9 @@ The following values defaults to \`❌\` if haystack does not originate from a f \`mtime\`: ISO inspired representation of the last local modification time of the current file, \`ctime\`: ISO representation of the local creation time of the current file. \`mtime_obj\`: date object representing \`mtime\`, -\`ctime_obj\`: date object representing \`ctime\`. +\`ctime_obj\`: date object representing \`ctime\`. + +The numbers: \`$n\`: the number of the file, \`$t\`: the number of files, \`$i\`: the number of the file among the files with a match so far, \`$c\`: among the files changed so far, \`$m\`: the number of the match in the file, \`$M\`: across all files. Each has cousins with zeros in front (\`$n0\` has 2 digits, \`$n00\` 3, and so on to 10) and with a space at the end (\`$n_\`). All variables, except from module, date objects, \`nl\` and \`_\`, has a corresponding variable name followed by \`_\` where the content has an extra space at the end (for easy concatenation). ` diff --git a/src/engine.ts b/src/engine.ts index 2d1e3bec..0dd09554 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -30,6 +30,9 @@ const re = { regexSpecialChars: /[\[\]{}()*+?.\/\\^$|]/g, byteOrSize: /bytes|size/, folderName: /[\\\/]+([^\\\/]+)[\\\/]+[^\\\/]+$/, + // The numbers of #791 that count what came before: $i, $c, $m and $M with their cousins, like $i00 or $M_ + runNumbers: /\$[icmM](?![A-Za-z])/, + matchNumbers: /\$[mM](?![A-Za-z])/, }; // Only the version ends up in the bundle @@ -100,6 +103,9 @@ function newStatus(config) { async function run(config) { validateEncoding(config.encoding); + // For the numbers of --js: the files with a match, the files changed and the matches so far + config.numbers = {matched: 0, changed: 0, matches: 0}; + if (config.replacementJs && config.outputMatch) { warn('The code is not run with --output-match, as only the matches are printed.'); } @@ -185,7 +191,7 @@ async function run(config) { } if (piped) { - return doReplacement('Piped data', config, config.pipedData); + return doReplacement('Piped data', config, config.pipedData, 1, 1); } config.files = getFilePaths(config); @@ -204,7 +210,9 @@ async function run(config) { config.output || config.outputMatch || null != config.sort || - config.sortReverse; + config.sortReverse || + // The numbers that count what came before are only the same on every run when the files come in order + (config.replacementJs && re.runNumbers.test(config.replacementOri)); const files = config.files.filter(filepath => fs.existsSync(filepath) ? true : error('File not found:', filepath) @@ -224,14 +232,15 @@ async function run(config) { let next = 0; const worker = async () => { while (next < files.length) { - await openFile(files[next++], config); + const index = next++; + await openFile(files[index], config, index + 1, files.length); } }; await Promise.all(Array.from({length: workers}, worker)); } -function openFile(file, config) { +function openFile(file, config, number, total) { config.statusInfo.files.searched.push(file); if (config.voidAsync) { chat('Open sync: ' + file); @@ -241,26 +250,38 @@ function openFile(file, config) { } catch (e) { return error(e); } - return doReplacement(file, config, data); + return doReplacement(file, config, data, number, total); } else { chat('Open async: ' + file); return fs.promises .readFile(file, config.encoding) - .then(data => doReplacement(file, config, data), error); + .then(data => doReplacement(file, config, data, number, total), error); } } // postfix argument names to limit the probabillity of user inputted javascript accidently using same values -function doReplacement(_file_rr: string, _config_rr: any, _data_rr: string) { +function doReplacement( + _file_rr: string, + _config_rr: any, + _data_rr: string, + _number_rr: number, + _total_rr: number +) { chat('Work on content from: ' + _file_rr); // Main regexp of the whole thing. With --output-match the replacement prints the matches, so the code of --replacement-js is not run. const result = _data_rr.replace( _config_rr.regex, _config_rr.replacementJs && !_config_rr.outputMatch - ? codeReplacement(_file_rr, _config_rr, _data_rr) + ? codeReplacement( + _file_rr, + _config_rr, + _data_rr, + fileNumbers(_config_rr, _number_rr, _total_rr) + ) : _config_rr.replacement ); + countFile(_config_rr, _data_rr, result); if (_config_rr.output || _config_rr.outputMatch) { // With --output-match the matches are printed by the replacement @@ -341,15 +362,16 @@ function updated(file, config) { function renameFiles(files, config) { // With --rename a / in the new name would move the file const slashed = []; - const plan = files.map(file => { + const plan = files.map((file, index) => { config.statusInfo.files.searched.push(file); const text = config.renamePath ? file : path.basename(file); chat('Work on the ' + (config.renamePath ? 'path' : 'name') + ' of: ' + file); const replacement = config.replacementJs && !config.outputMatch - ? codeReplacement(file, config, text) + ? codeReplacement(file, config, text, fileNumbers(config, index + 1, files.length)) : config.replacement; const result = text.replace(config.regex, replacement); + countFile(config, text, result); if (result === text) { return {old: file, new: file}; } @@ -636,8 +658,8 @@ function readReplacementFile(filepath, conf) { } // The code of --js. With --replacement-file its value is the path to the file with the replacement. -function codeReplacement(file, config, text) { - const code = dynamicReplacement(file, config, text); +function codeReplacement(file, config, text, numbers) { + const code = dynamicReplacement(file, config, text, numbers); if (!config.replacementFile) { return code; } @@ -741,13 +763,49 @@ function readableSize(size) { ); } -function dynamicReplacement(_file_rr, _config_rr, _data_rr) { +// The numbers of this file for --js: $n of $t files, and $i and $c, the files with a match and the files changed so far +function fileNumbers(config, number, total) { + return {n: number, t: total, i: config.numbers.matched + 1, c: config.numbers.changed + 1}; +} + +// When a file is done, it counts for $i if the pattern found something, and for $c if it was changed +function countFile(config, text, result) { + if (-1 !== text.search(config.regex)) { + config.numbers.matched++; + } + if (result !== text) { + config.numbers.changed++; + } +} + +// Code that makes the numbers and their cousins, put in front of the code of --js. +// $n0 is $n with zeros in front to 2 digits, $n00 to 3 and so on to 10, and the ones with _ at the end have a space at the end. +function numbersCode(numbers) { + return Object.entries(numbers) + .map(([name, value]) => { + const cousins = [`$${name} = ${value}`, `$${name}_ = '${value} '`]; + for (let width = 2; width <= 10; width++) { + const zeros = '0'.repeat(width - 1); + const padded = String(value).padStart(width, '0'); + cousins.push(`$${name}${zeros} = '${padded}'`, `$${name}${zeros}_ = '${padded} '`); + } + return 'var ' + cousins.join(', ') + '; '; + }) + .join(''); +} + +function dynamicReplacement( + _file_rr, + _config_rr, + _data_rr, + _numbers_rr = {n: 1, t: 1, i: 1, c: 1} +) { const _time_obj = now; const _time = localTimeString(_time_obj); const _pipe = _config_rr.pipedData, _text = _data_rr, _find = _config_rr.pattern, - code_rr = _config_rr.replacementOri, + code_rr = numbersCode(_numbers_rr) + _config_rr.replacementOri, _cwd = process.cwd(), _now = _time, _ = ' ', @@ -874,7 +932,8 @@ function dynamicReplacement(_file_rr, _config_rr, _data_rr) { // Run only once if no captured groups (replacement cant change). // The result is given back from a function, so `$&` and `$$` in it are inserted as they are and not expanded. - if (!/\$\d/.test(_config_rr.replacement)) { + // $m and $M are the number of the match, so they also need the code to run once per match + if (!/\$\d/.test(_config_rr.replacement) && !re.matchNumbers.test(_config_rr.replacement)) { const result = dynamicContent( require, fs, @@ -927,6 +986,7 @@ function dynamicReplacement(_file_rr, _config_rr, _data_rr) { return () => result; } // Capture groups used, so need to run once per match + let _match_rr = 0; return function () { const __pipe = _pipe, __text = _text, @@ -953,7 +1013,9 @@ function dynamicReplacement(_file_rr, _config_rr, _data_rr) { __ = _, __code_rr = code_rr; - var capturedGroups = ''; + _match_rr++; + _config_rr.numbers.matches++; + var capturedGroups = numbersCode({m: _match_rr, M: _config_rr.numbers.matches}); for (var i = 0; i < arguments.length - 2; i++) { capturedGroups += 'var $' + i + '=' + JSON.stringify(arguments[i]) + '; '; } diff --git a/test/cli/run.sh b/test/cli/run.sh index bda82a13..b26ad19c 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -183,6 +183,10 @@ reset echo 'baz' > rep.file assert "rexreplace 'foo' rep.file my.file -o --replacement-file --trim-file" 'bazbar' +# --js has the number of the file, $n, with cousins like €n00 for 3 digits +reset +assert "rexreplace 'foo' '€n00' my.file -o --js" '001bar' + # --literal-replacement (-l) uses the replacement as it is, so `$1` is just text reset assert "rexreplace '(foo)' '€1' my.file -o --literal-replacement" '$1bar' diff --git a/test/fix/791.test.js b/test/fix/791.test.js new file mode 100644 index 00000000..f6666fc9 --- /dev/null +++ b/test/fix/791.test.js @@ -0,0 +1,55 @@ +// https://github.com/mathiasrw/rexreplace/issues/791 +// The code of --js knew about the file, but not where it was in the run, so files could not be numbered. +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +const content = name => fs.readFileSync(name, 'utf8'); + +describe('#791 numbers for --js', () => { + test('$n is the number of the file and $t the number of files', async () => { + fs.writeFileSync('a.txt', 'x'); + fs.writeFileSync('b.txt', 'x'); + await rr('x', '$n + " of " + $t', {includeGlob: ['*.txt'], replacementJs: true}); + expect([content('a.txt'), content('b.txt')]).toEqual(['1 of 2', '2 of 2']); + }); + + test('have cousins with zeros in front, and with a space at the end', async () => { + const config = {pipedData: 'x', replacementJs: true}; + expect((await rr('x', '$n0 + $n00 + $n_ + $n00_', config)).out).toBe('010011 001 '); + expect((await rr('x', '$n000000000', config)).out).toBe('0000000001'); + }); + + test('$i counts the files with a match, and $c the files changed', async () => { + fs.writeFileSync('a.txt', 'x'); + fs.writeFileSync('b.txt', 'x'); + fs.writeFileSync('c.txt', 'x'); + // a.txt has a match, but is not changed + const code = 'name === "a" ? "x" : $i + "-" + $c'; + await rr('x', code, {includeGlob: ['*.txt'], replacementJs: true}); + expect([content('a.txt'), content('b.txt'), content('c.txt')]).toEqual(['x', '2-1', '3-2']); + }); + + test('$m is the number of the match in the file, and $M across the files', async () => { + fs.writeFileSync('a.txt', 'oo'); + fs.writeFileSync('b.txt', 'oo'); + await rr('o', '$m + "" + $M', {includeGlob: ['*.txt'], replacementJs: true}); + expect([content('a.txt'), content('b.txt')]).toEqual(['1122', '1324']); + }); + + test('are the same on every run, also when the files could be handled at the same time', async () => { + const names = Array.from({length: 30}, (_, i) => `file${i + 1}.txt`); + names.forEach(name => fs.writeFileSync(name, 'x')); + await rr('x', '$i', {includeGlob: ['*.txt'], replacementJs: true, voidAsync: false}); + expect(names.map(content)).toEqual(names.map((_, i) => String(i + 1))); + }); + + test('number the files to rename', async () => { + fs.writeFileSync('b.jpg', 'x'); + fs.writeFileSync('a.jpg', 'x'); + const config = {includeGlob: ['*.jpg'], rename: true, replacementJs: true}; + expect((await rr('^.*$', '"photo-" + $n00 + ext', config)).out).toBe( + 'photo-001.jpg\nphoto-002.jpg\n' + ); + }); +});