diff --git a/README.md b/README.md index 2df0357f..e6a7e315 100644 --- a/README.md +++ b/README.md @@ -196,9 +196,10 @@ Let both `"foo 'is' bar"` and `' foo "was" bar'` become `» foo ... bar «` inde | `-b` | **`--keep-backup`** Keep a backup file of the original content. [boolean] | | `-m` | **`--output-match`** Output each match on a new line. Will not replace any content but you still need to provide a dummy value (like `_`) as replacement parameter. If search pattern does not contain matching groups the full match will be outputted. If search pattern does contain matching groups only matching groups will be outputted (same line with no delimiter). [boolean] | | `-T` | **`--trim-pipe`** Trim piped data before processing. If piped data only consists of chars that can be trimmed (new line, space, tabs...) it will become an empty string. [boolean] | -| `-P` | **`--void-pipe`** Ignore any pipe data - even if stdin is open. Can not be combined with -R or -T. Can also be set via the environment variable RR_VOID_PIPE. [boolean] | +| `-p` | **`--pipe`** Read the piped data and make it available to --replacement-js as `pipe`. [boolean] | +| | **`--js-pipe`** Shorthand for --replacement-js --pipe (-jp). [boolean] | | `-R` | **`--replacement-pipe`** Replacement will be piped in. You still need to provide a dummy value (like `_`) as replacement parameter. [boolean] | -| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command (null if no piped data), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | +| `-j` | **`--replacement-js`**, **`--js`** Treat replacement as javascript source code. The statement from the last expression will become the replacement string. Purposefully implemented the most insecure way possible to remove _any_ incentive to consider running code from an untrusted part. The full match will be available as a javascript variable named $0 while each captured group will be available as $1, $2, $3, ... and so on. At some point, the $ char _will_ give you a headache when used from the command line, so use €0, €1, €2, €3... instead. If the javascript source code references to the full match or a captured group the code will run once per match. Otherwise, it will run once per file. The code has access to the following variables: `r` as an alias for `require` with both expanded to understand a relative path even if it is not starting with `./`, `fs` from node, `path` from node, `globs` from npm, `pipe`: the data piped into the command with --pipe (null if no piped data, and `pipe_` is `❌ `), `find`: pattern searched for (the needle), `text`: full text being searched i.e. file content or piped data (the haystack), `bytes`: total size of the haystack in bytes, `size`: human-friendly representation of the total size of the haystack, `time`: String representing the local time when the command was invoked, `time_obj`: date object representing `time`, `now`: alias for `time`, `cwd`: current process working dir, `nl`: a new-line char, `_`: a single space char (for easy string concatenation). The following values defaults to `❌` if haystack does not originate from a file: `file`: contains the full path of the active file being searched (including full filename), `file_rel`: contains `file` relative to current process working dir, `dirpath`: contains the full path without filename of the active file being searched, `dirpath_rel`: contains `dirpath` relative to current process working dir, `filename`: is the full filename of the active file being searched without path, `name`: filename of the active file being searched with no extension, `ext`: extension of the filename including leading dot, `mtime`: ISO inspired representation of the last local modification time of the current file, `ctime`: ISO representation of the local creation time of the current file. `mtime_obj`: date object representing `mtime`, `ctime_obj`: date object representing `ctime`. All variables, except from module, date objects, `nl` and `_`, has a corresponding variable name followed by `_` where the content has an extra space at the end (for easy concatenation). [boolean] | | `-x` | **`--exclude-re`** Exclude files with a path that matches this regular expression. Will follow same regex flags and setup as the main search. Can be used multiple times. [string] | | `-X` | **`--exclude-glob`** Exclude files found with this glob. Can be used multiple times. [string] | | `-h` | **`--help`** Display help. [boolean] | @@ -211,7 +212,7 @@ Let both `"foo 'is' bar"` and `' foo "was" bar'` become `» foo ... bar «` inde - Pattern defaults to global multiline case-insensitive search - Supports regex lookaheads in the pattern - Supports backreference to matching groups in the replacement -- Data to be treated can be piped in (Ignore piped data via `-P`) +- Data to be treated can be piped in - See the [release note](https://github.com/mathiasrw/rexreplace/releases) for a log of changes. Descriptions are given in latest patch version. ### Quirks @@ -224,13 +225,6 @@ Let both `"foo 'is' bar"` and `' foo "was" bar'` become `» foo ... bar «` inde - The `€` and `§` aliases are only for what you write as pattern, replacement (also with `--replacement-js`) and `--exclude-re`. File globs, `--exclude-glob` and piped data (both content to search and a replacement given with `--replacement-pipe`) are used exactly as they are, so a file named `price€.txt` is found without any flags. -- RexReplace reads from stdin whenever stdin is not a terminal, and waits for it to close. Some environments (CI runners, cron, `docker exec`, editors spawning processes) leave stdin open without ever writing to it, which makes the command hang. Add the `--void-pipe` flag (`-P`) to ignore piped data. Can also be set with the environment variable `RR_VOID_PIPE`: - -```bash -> rexreplace 'foo' 'bar' myfile.md -P - -> RR_VOID_PIPE=1 rexreplace 'foo' 'bar' myfile.md -``` ### Limitations diff --git a/package.json b/package.json index 38d8261d..efd55047 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "test-ci": "yarn test-format && yarn build && yarn test-cli && yarn test-js && yarn test-minify", "test-speed": "bash test/speed/run.sh", "rebase": "yarn is-git-clean && git fetch && git rebase origin/main", - "load-options": "# rr -h | rr 'Options:(.+)Examples:' _ -ms | rr '\\n {26,}|\\n\\n *' ' ' | rr \"'\" '`' | rr '^ (-.+?), (--[^ ]+) *' '`€1` | **`€2`** ' | rr '(^---- . ----).+?(## Good to know)' '€1 + nl + pipe + nl + nl + €2' readme.md -jsT", + "load-options": "# rr -h | rr 'Options:(.+)Examples:' _ -ms | rr '\\n {26,}|\\n\\n *' ' ' | rr \"'\" '`' | rr '^ (-.+?), (--[^ ]+) *' '`€1` | **`€2`** ' | rr '(^---- . ----).+?(## Good to know)' '€1 + nl + pipe + nl + nl + €2' readme.md -jspT", "test-format": "prettier . --list-different || (echo 'Please correct file formatting using `yarn format` and try again.' && exit 1)", "format": "prettier . --write", "is-git-clean": "(git diff --quiet --exit-code --cached && git diff --quiet --exit-code) || (echo Please commit or stash changes && exit 1)", diff --git a/src/cli.ts b/src/cli.ts index 46c7661a..499d8dd1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,7 +1,7 @@ // CLI interface for rexreplace import * as rexreplace from './engine.ts'; -import {outputConfig, die, warn} from './output.ts'; +import {outputConfig, die} from './output.ts'; // Make it visible when Bun is the runtime (like with rexreplace-bun) as the regex engine is then not the one from Node. // Added as semver build metadata (`+bun`) as a `-bun` would make it a prerelease. @@ -189,15 +189,12 @@ const yargs = require('yargs') '' ) - .boolean('P') - .alias('P', 'void-pipe') - .describe( - 'P', - `Ignore any pipe data - even if stdin is open. Can not be combined with -R or -T. ` + - `Can also be set via the environment variable RR_VOID_PIPE.` - ) - .conflicts('P', 'R') - .conflicts('P', 'T') + .boolean('p') + .alias('p', 'pipe') + .describe('p', `Read the piped data and make it available to --replacement-js as \`pipe\`.`) + + .boolean('js-pipe') + .describe('js-pipe', `Shorthand for --replacement-js --pipe (-jp).`) .boolean('j') .alias('j', 'js') @@ -216,7 +213,7 @@ The code has access to the following variables: \`fs\` from node, \`path\` from node, \`globs\` from npm, -\`pipe\`: the data piped into the command (null if no piped data), +\`pipe\`: the data piped into the command with --pipe (null if no piped data, and \`pipe_\` is \`❌ \`), \`find\`: pattern searched for (the needle), \`text\`: full text being searched i.e. file content or piped data (the haystack), \`bytes\`: total size of the haystack in bytes, @@ -335,7 +332,7 @@ K k l N n O -P p +P r S t @@ -394,6 +391,10 @@ function unescapeString(str = '') { } }); + if (config.jsPipe) { + config.replacementJs = config.pipe = true; + } + outputConfig(config); let pipeInUse = false; @@ -428,38 +429,34 @@ function unescapeString(str = '') { config.output = true; }*/ - if (process.env.RR_VOID_PIPE) { - const conflict = ' conflicts with the env. var RR_VOID_PIPE as no data will be piped.'; - config.replacementPipe && warn('The flag --replacement-pipe' + conflict); - config.trimPipe && warn('The flag --trim-pipe' + conflict); - config.voidPipe = true; + // Like grep, stdin is only read when piped data is expected. An open pipe that nobody writes to would make the run wait forever. + const pipeDataExpected = !config.includeGlob.length || config.replacementPipe || config.pipe; + + if (!pipeDataExpected || Boolean(process.stdin.isTTY)) { + return rexreplace.engine(config); } - if (config.voidPipe || Boolean(process.stdin.isTTY)) { - rexreplace.engine(config); - } else { - process.stdin.setEncoding(config.encoding); + process.stdin.setEncoding(config.encoding); - process.stdin.on('readable', () => { - let chunk = process.stdin.read(); + process.stdin.on('readable', () => { + let chunk = process.stdin.read(); - if (null !== chunk) { - pipeInUse = true; + if (null !== chunk) { + pipeInUse = true; + pipeData += chunk; + while ((chunk = process.stdin.read())) { pipeData += chunk; - while ((chunk = process.stdin.read())) { - pipeData += chunk; - } } - }); - - process.stdin.on('end', () => { - if (pipeInUse) { - if (yargs.argv.trimPipe) { - pipeData = pipeData.trim(); - } - config.pipedData = pipeData; + } + }); + + process.stdin.on('end', () => { + if (pipeInUse) { + if (yargs.argv.trimPipe) { + pipeData = pipeData.trim(); } - rexreplace.engine(config); - }); - } + config.pipedData = pipeData; + } + rexreplace.engine(config); + }); })(); diff --git a/src/engine.ts b/src/engine.ts index 6cb51667..f340cfa4 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -483,7 +483,8 @@ function dynamicReplacement(_file_rr, _config_rr, _data_rr) { globs, path, _pipe, - _pipe + _, + // Without piped data `pipe` is null, while `pipe_` is `❌ ` like the file variables + (_pipe ?? '❌') + _, _find, _find + _, _text, @@ -567,7 +568,7 @@ function dynamicReplacement(_file_rr, _config_rr, _data_rr) { path, __pipe, - __pipe + __, + (__pipe ?? '❌') + __, __find, __find + __, diff --git a/test/cli/run.sh b/test/cli/run.sh index 3454688c..b3ea0bdd 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -285,44 +285,26 @@ reset assert "printf x | rexreplace 'b' _ my.file -o --replacement-pipe" 'fooxar' -# -P +# --pipe reset -assert "printf x | rexreplace 'foo' 'pipe' my.file -o --replacement-js" 'xbar' -assert "printf x | rexreplace 'foo' 'pipe' my.file -o --replacement-js -P" 'nullbar' -assert "printf x | rexreplace 'foo' 'pipe' my.file -o --replacement-js --void-pipe" 'nullbar' -assert_failure "printf x | rexreplace 'b' _ my.file -P -R" -assert_failure "printf x | rexreplace 'b' _ my.file -P -T" +assert "printf x | rexreplace 'foo' 'pipe' my.file -o --replacement-js" 'nullbar' +assert "printf x | rexreplace 'foo' 'pipe' my.file -o --replacement-js --pipe" 'xbar' +assert "printf x | rexreplace 'foo' 'pipe' my.file -o -jp" 'xbar' +assert "printf x | rexreplace 'foo' 'pipe' my.file -o --js-pipe" 'xbar' +assert "printf ' x ' | rexreplace 'foo' 'pipe' my.file -o --js-pipe -T" 'xbar' # Warning when piped data is not used reset -assert "printf x | rexreplace 'o' 'x' my.file 2>&1 | grep 'Data is piped in'" ' 🟡 Data is piped in, but no flag tells what to do with it. It is not used. ' +assert "printf x | rexreplace 'o' 'x' my.file --pipe 2>&1 | grep 'Data is piped in'" ' 🟡 Data is piped in, but no flag tells what to do with it. It is not used. ' assert "cat my.file" 'fxxbar' -assert_failure "printf x | rexreplace 'o' 'x' my.file -q 2>&1 | grep 'Data is piped in'" -assert_failure "printf x | rexreplace 'o' 'x' my.file -P 2>&1 | grep 'Data is piped in'" +assert_failure "printf x | rexreplace 'o' 'x' my.file 2>&1 | grep 'Data is piped in'" +assert_failure "printf x | rexreplace 'o' 'x' my.file --pipe -q 2>&1 | grep 'Data is piped in'" assert_failure "printf x | rexreplace 'b' _ my.file -R 2>&1 | grep 'Data is piped in'" -assert_failure "printf x | rexreplace 'b' 'pipe' my.file -j 2>&1 | grep 'Data is piped in'" -assert_failure "printf '' | rexreplace 'o' 'x' my.file 2>&1 | grep 'Data is piped in'" -assert_failure "printf '\n' | rexreplace 'o' 'x' my.file 2>&1 | grep 'Data is piped in'" -assert_failure "printf '\r\n' | rexreplace 'o' 'x' my.file 2>&1 | grep 'Data is piped in'" - - -# RR_VOID_PIPE -reset -assert "printf x | RR_VOID_PIPE=1 rexreplace 'foo' 'pipe' my.file -o --replacement-js 2>&1" 'nullbar' -assert "printf x | RR_VOID_PIPE= rexreplace 'foo' 'pipe' my.file -o --replacement-js" 'xbar' -assert "printf x | RR_VOID_PIPE=1 rexreplace 'foo' 'pipe' my.file -o --replacement-js -T" 'nullbar' -assert "printf x | RR_VOID_PIPE=1 rexreplace 'foo' 'pipe' my.file -o --replacement-js -T 2>&1 >/dev/null" ' 🟡 The flag --trim-pipe conflicts with the env. var RR_VOID_PIPE as no data will be piped. ' -assert "printf x | RR_VOID_PIPE=1 rexreplace 'foo' 'pipe' my.file -o --replacement-js -T -q 2>&1" 'nullbar' -assert "printf x | RR_VOID_PIPE=1 rexreplace 'foo' 'pipe' my.file -o --replacement-js -T -Q 2>&1" 'nullbar' -assert_failure "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R" -assert "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R 2>&1 | grep -E 'RR_VOID_PIPE as|No data piped'" ' 🟡 The flag --replacement-pipe conflicts with the env. var RR_VOID_PIPE as no data will be piped. \n ❌ No data piped into replacement ' -assert "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R -T 2>&1 | grep -E 'RR_VOID_PIPE as|No data piped'" ' 🟡 The flag --replacement-pipe conflicts with the env. var RR_VOID_PIPE as no data will be piped. \n 🟡 The flag --trim-pipe conflicts with the env. var RR_VOID_PIPE as no data will be piped. \n ❌ No data piped into replacement ' -assert_failure "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R -q" -assert "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R -q 2>&1 | grep -v 'exited with code'" ' ❌ No data piped into replacement ' -assert_failure "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R -Q" -assert "printf x | RR_VOID_PIPE=1 rexreplace 'b' _ my.file -R -Q 2>&1 | grep -v 'exited with code'" '' -assert "cat my.file" 'foobar' +assert_failure "printf x | rexreplace 'b' 'pipe' my.file --js-pipe 2>&1 | grep 'Data is piped in'" +assert_failure "printf '' | rexreplace 'o' 'x' my.file --pipe 2>&1 | grep 'Data is piped in'" +assert_failure "printf '\n' | rexreplace 'o' 'x' my.file --pipe 2>&1 | grep 'Data is piped in'" +assert_failure "printf '\r\n' | rexreplace 'o' 'x' my.file --pipe 2>&1 | grep 'Data is piped in'" diff --git a/test/js/fix/736.test.js b/test/js/fix/736.test.js new file mode 100644 index 00000000..c8d003bc --- /dev/null +++ b/test/js/fix/736.test.js @@ -0,0 +1,24 @@ +// https://github.com/mathiasrw/rexreplace/issues/736 +// stdin was read every time it was not a terminal, so a pipe that was open but never written to made the run wait forever. +// The flags that ask for the piped data are tested in test/cli/run.sh. +import {describe, test, expect} from 'bun:test'; +import path from 'path'; + +const cli = path.resolve(import.meta.dir, '../../../src/cli.ts'); + +describe('#736 stdin is only read when the data is asked for', () => { + test('files are changed without waiting for a pipe that is never closed', async () => { + await Bun.write('my.file', 'foobar'); + const proc = Bun.spawn([process.execPath, cli, 'o', 'x', 'my.file'], { + cwd: process.cwd(), + stdin: 'pipe', + stderr: 'pipe', + }); + + const exit = await Promise.race([proc.exited, Bun.sleep(2000).then(() => 'still waiting')]); + proc.kill(); + + expect(exit).toBe(0); + expect(await Bun.file('my.file').text()).toBe('fxxbar'); + }); +}); diff --git a/test/js/replacement-js.test.js b/test/js/replacement-js.test.js index 51a4ea86..c5d862d4 100644 --- a/test/js/replacement-js.test.js +++ b/test/js/replacement-js.test.js @@ -128,10 +128,17 @@ describe('--replacement-js', () => { test('pipe is the piped data', async () => { expect(await js('o', 'pipe', 'fo')).toBe('ffo'); + expect(await js('o', 'pipe_', 'fo')).toBe('ffo '); fs.writeFileSync('my.file', 'foobar'); const config = {includeGlob: ['my.file'], replacementJs: true, output: true}; expect((await rr('foo', 'pipe', {...config, pipedData: 'boo'})).out).toBe('boobar'); - expect((await rr('foo', 'String(pipe)', config)).out).toBe('nullbar'); + }); + + test('pipe is null and pipe_ is ❌ without piped data', async () => { + fs.writeFileSync('my.file', 'foobar'); + const config = {includeGlob: ['my.file'], replacementJs: true, output: true}; + expect((await rr('foo', 'String(pipe) + pipe_', config)).out).toBe('null❌ bar'); + expect((await rr('(o)', 'pipe_ + €1', config)).out).toBe('f❌ o❌ obar'); }); test('file, file_rel, dirpath, dirpath_rel, dirname, filename, name and ext tell about the file', async () => { @@ -218,7 +225,6 @@ describe('--replacement-js', () => { test('most variables come in a version with a space after', async () => { const names = [ - 'pipe', 'find', 'text', 'file', diff --git a/test/js/void-pipe.test.js b/test/js/void-pipe.test.js deleted file mode 100644 index 5b6b94d6..00000000 --- a/test/js/void-pipe.test.js +++ /dev/null @@ -1,5 +0,0 @@ -import {describe, test} from 'bun:test'; - -describe('--void-pipe', () => { - test.todo('is done by src/cli.ts (not the engine) and is tested in test/cli/run.sh'); -});