Add --glob-pipe to pipe in the files to work on to close #711

src/cli.ts

@@ -155,10 +155,11 @@ const yargs = require('yargs')
 		'Sets --name-delimiter and --content-delimiter to the same value. Supports escape sequences like `\\t`, `\\n` and the `€` and `§` aliases.'
 	)
 
-	.boolean('null')
+	.boolean('null-out')
+	.alias('null-out', 'null')
 	.describe(
-		'null',
-		"Shorthand for --delimiter '\\0'. Useful for piping content with spaces or line breaks (like for `xargs -0`). The specific delimiter values win over --delimiter, and --delimiter wins over --null."
+		'null-out',
+		"Shorthand for --delimiter '\\0'. Useful for piping content with spaces or line breaks (like for `xargs -0`). The specific delimiter values win over --delimiter, and --delimiter wins over --null-out."
 	)
 
 	.boolean('name-delimiter-tail')
@@ -275,6 +276,22 @@ const yargs = require('yargs')
 	.boolean('js-pipe')
 	.describe('js-pipe', `Shorthand for --replacement-js --pipe (-jp).`)
 
+	.boolean('g')
+	.alias('g', 'glob-pipe')
+	.describe(
+		'g',
+		'Each line of the piped data is a path or a glob of the files to work on. They are used together with the ones given as arguments.'
+	)
+	// The piped data can only be one thing
+	.conflicts('g', ['R', 'p', 'js-pipe'])
+
+	.boolean('null-in')
+	.describe(
+		'null-in',
+		'The piped paths and globs of --glob-pipe are separated by null bytes, like from `find -print0` or --null-out.'
+	)
+	.implies('null-in', 'glob-pipe')
+
 	.boolean('j')
 	.alias('j', 'js')
 	.alias('j', 'replacement-js')
@@ -366,11 +383,6 @@ All variables, except from module, date objects, \`nl\` and \`_\`, has a corresp
         .describe('G', "filename/globas are filename(s) for files containing one filename/globs on each line to be search/replaced")
         .alias('G', 'globs-file')
 
-    .boolean('g')
-        .describe('g', "filename/globs will be piped in. If any filename/globs are given in command the piped data will be prepened")
-        .alias('g', 'glob-pipe')
-
-
     .boolean('j')
         .describe('j', "Pattern is javascript source that will return a string giving the pattern to use")
         .alias('j', 'pattern-js')
@@ -563,7 +575,8 @@ const run = () =>
 	);
 
 // Like grep, stdin is only read when piped data is expected. An open pipe that nobody writes to would make the run wait forever.
-const pipeDataExpected = !config.includeGlob.length || config.replacementPipe || config.pipe;
+const pipeDataExpected =
+	!config.includeGlob.length || config.replacementPipe || config.pipe || config.globPipe;
 
 if (!pipeDataExpected || Boolean(process.stdin.isTTY)) {
 	run();

src/engine.ts

@@ -77,14 +77,26 @@ function newStatus(config) {
 async function run(config) {
 	validateEncoding(config.encoding);
 
+	// With --glob-pipe each line of the piped data is a path or a glob, used together with the ones given as arguments
+	if (config.globPipe) {
+		if (config.replacementPipe || config.pipe) {
+			die(
+				'--glob-pipe can not be used with --replacement-pipe or --pipe, as the piped data can only be one thing'
+			);
+		}
+		config.pipedDataUsed = true;
+		const lines = (config.pipedData ?? '').split(config.nullIn ? '\0' : /\r?\n/);
+		config.includeGlob = config.includeGlob.concat(lines.filter(line => '' !== line));
+	}
+
 	config.pattern = getFinalPattern(config.pattern, config) || '';
 
 	config.replacement = getFinalReplacement(config.replacement, config) || '';
 
 	config.replacementOri = config.replacement;
 
 	// The most specific delimiter wins. Without one there is a line break after each item, unless it already ends with one.
-	const delimiter = config.delimiter ?? (config.null ? '\0' : null);
+	const delimiter = config.delimiter ?? (config.nullOut ? '\0' : null);
 	config.nameDelimiter = config.nameDelimiter ?? delimiter;
 	config.contentDelimiter = config.contentDelimiter ?? delimiter;
 
@@ -262,6 +274,9 @@ function updated(file, config) {
 
 // How the piped data was used
 function pipeType(config, piped) {
+	if (config.globPipe) {
+		return 'GLOBS';
+	}
 	if (piped) {
 		return 'CONTENT';
 	}

test/cli/run.sh

@@ -451,6 +451,36 @@ assert		 		"rexreplace 'b' 'x' my.file --content-delimiter '\x4' 2>&1 | grep -c
 assert		 		"rexreplace 'b' 'x' my.file --delimiter '\x4' 2>&1 | grep -c 'Invalid escape sequence in --delimiter'"    '1'
 assert		 		"cat my.file"    'foobar'
 
+# --glob-pipe: each line of the piped data is a path or a glob to work on (#711)
+reset
+assert		 		"printf 'my.file\nyour.file\n' | rexreplace 'b' 'x' --glob-pipe -A"    'my.file\nyour.file'
+reset
+assert		 		"printf '*.file' | rexreplace 'b' 'x' -g -A"    'my.file\nyour.file'
+# The names of the changed files go on to the next command, also when a name has a line break in it
+reset
+assert		 		"rexreplace 'oo' 'x' '*.file' --null-out | rexreplace 'x' 'y' --glob-pipe --null-in"    'my.file'
+assert		 		"cat my.file"    'fybar'
+# The piped data can only be one thing
+reset
+assert_raises		"printf 'my.file' | rexreplace 'b' 'x' -g -R"    1
+assert_raises		"printf 'my.file' | rexreplace 'b' 'x' -g --js-pipe"    1
+assert_raises		"printf 'my.file' | rexreplace 'b' 'x' --null-in"    1
+assert		 		"cat my.file"    'foobar'
+
+
+# Null bytes are kept: they come out where they went in. (Bytes that are not valid UTF-8 are only kept with -e latin1.)
+reset
+assert		 		"printf 'foo\0boo\0' | rexreplace 'o' 'x' | tr '\0' '|'"    'fxx|bxx|'
+assert		 		"printf 'foo\0boo\0' | rexreplace 'o' 'x' --output | tr '\0' '|'"    'fxx|bxx|'
+assert		 		"printf 'foo\0boo\0' | rexreplace 'o' '_' --output-match --null-out | tr '\0' '|'"    'o|o|o|o|'
+printf 'foo\0boo\0' > null.file
+assert		 		"rexreplace 'o' 'x' null.file"    'null.file'
+assert		 		"tr '\0' '|' < null.file"    'fxx|bxx|'
+printf 'foo\0boo\0' > null.file
+assert		 		"rexreplace 'o' 'x' null.file --output 2>/dev/null | tr '\0' '|'"    'fxx|bxx|'
+assert		 		"rexreplace 'o' '_' null.file --output-match --null-out 2>/dev/null | tr '\0' '|'"    'o|o|o|o|'
+
+
 # Globs are case-insensitive like the search, unless --void-ignore-case (#716)
 reset
 assert		 		"rexreplace 'oo' 'x' '*.FILE' -X 'YOUR*'"    'my.file'

test/js/fix/703.test.js

@@ -61,14 +61,16 @@ describe('#703 the names of affected files', () => {
 		expect((await rr('x', 'o', {includeGlob: [file]})).out).toBe(file + '\n');
 	});
 
-	describe('--null', () => {
-		test('ends each name with a NUL byte', async () => {
-			expect(await run({null: true})).toEqual({out: 'a.file\0c.file\0', err: '', exit: null});
+	describe('--null-out', () => {
+		test('ends each name with a null byte', async () => {
+			expect(await run({nullOut: true})).toEqual({out: 'a.file\0c.file\0', err: '', exit: null});
 		});
 
 		test('keeps a name with a line break in one piece', async () => {
 			await Bun.write('my\n.file', 'foo');
-			expect((await rr('o', 'x', {includeGlob: ['*.file'], null: true})).out).toBe('my\n.file\0');
+			expect((await rr('o', 'x', {includeGlob: ['*.file'], nullOut: true})).out).toBe(
+				'my\n.file\0'
+			);
 		});
 	});
 
@@ -301,7 +303,7 @@ describe('#703 the names of affected files', () => {
 		});
 	});
 
-	describe('--delimiter and --null', () => {
+	describe('--delimiter and --null-out', () => {
 		test('--delimiter sets both delimiters', async () => {
 			expect(await run({output: true, delimiter: ', '})).toEqual({
 				out: 'fxx, bar, fxx, ',
@@ -310,16 +312,18 @@ describe('#703 the names of affected files', () => {
 			});
 		});
 
-		test('--null is the same as --delimiter with a NUL byte', async () => {
-			expect(await run({output: true, null: true})).toEqual({
+		test('--null-out is the same as --delimiter with a null byte', async () => {
+			expect(await run({output: true, nullOut: true})).toEqual({
 				out: 'fxx\0bar\0fxx\0',
 				err: 'a.file\0c.file\0',
 				exit: null,
 			});
 		});
 
 		test('the most specific delimiter wins', async () => {
-			expect((await run({output: true, delimiter: ', ', null: true})).out).toBe('fxx, bar, fxx, ');
+			expect((await run({output: true, delimiter: ', ', nullOut: true})).out).toBe(
+				'fxx, bar, fxx, '
+			);
 			expect(
 				await run({
 					output: true,
@@ -328,7 +332,7 @@ describe('#703 the names of affected files', () => {
 					nameDelimiter: ';',
 				})
 			).toEqual({out: 'fxx|bar|fxx|', err: 'a.file;c.file;', exit: null});
-			expect(await run({output: true, null: true, nameDelimiter: ', '})).toEqual({
+			expect(await run({output: true, nullOut: true, nameDelimiter: ', '})).toEqual({
 				out: 'fxx\0bar\0fxx\0',
 				err: 'a.file, c.file, ',
 				exit: null,
@@ -354,8 +358,8 @@ describe('#703 the names of affected files', () => {
 			expect(await run({[flag]: true, [quiet]: true})).toEqual({out, err: '', exit: null});
 		});
 
-		test('follow --null, --name-delimiter and --full-path', async () => {
-			expect((await run({[flag]: true, null: true})).err).toBe('a.file\0c.file\0');
+		test('follow --null-out, --name-delimiter and --full-path', async () => {
+			expect((await run({[flag]: true, nullOut: true})).err).toBe('a.file\0c.file\0');
 			expect((await run({[flag]: true, nameDelimiter: ', '})).err).toBe('a.file, c.file, ');
 			expect((await run({[flag]: true, fullPath: true})).err).toBe(
 				path.resolve('a.file') + '\n' + path.resolve('c.file') + '\n'
@@ -382,7 +386,7 @@ describe('#703 the names of affected files', () => {
 		});
 
 		test.each([
-			['null', {null: true}, '\0', '\0'],
+			['null', {nullOut: true}, '\0', '\0'],
 			['name-delimiter', {nameDelimiter: ', ', nameDelimiterTail: false}, ', ', ''],
 		])('works with --%s', async (name, flags, between, last) => {
 			expect((await run({fullPath: true, ...flags})).out).toBe(

test/js/fix/711.test.js

@@ -0,0 +1,85 @@
+// https://github.com/mathiasrw/rexreplace/issues/711
+// A list of files from another command could only be used with `xargs` or `$(…)`.
+import {describe, test, expect} from 'bun:test';
+import fs from 'fs';
+import {rr, rrStatus} from '../helper.js';
+
+const content = name => fs.readFileSync(name, 'utf8');
+
+// Make each file with the content `foo`
+function makeFiles(...names) {
+	names.forEach(name => fs.writeFileSync(name, 'foo'));
+}
+
+describe('#711 --glob-pipe', () => {
+	test('each line of the piped data is a file to work on', async () => {
+		makeFiles('a.file', 'b.file', 'c.file');
+		const config = {pipedData: 'a.file\nb.file\n', globPipe: true};
+		expect(await rr('o', 'x', config)).toEqual({out: 'a.file\nb.file\n', err: '', exit: null});
+		expect(content('c.file')).toBe('foo');
+	});
+
+	test('a line can be a glob', async () => {
+		makeFiles('a.file', 'b.file', 'c.txt');
+		const result = await rr('o', 'x', {pipedData: '*.file', globPipe: true});
+		expect(result.out).toBe('a.file\nb.file\n');
+	});
+
+	test('a line is used together with the paths and globs given as arguments', async () => {
+		makeFiles('a.file', 'b.file', 'c.txt');
+		const config = {pipedData: 'a.file\n', includeGlob: ['*.txt'], globPipe: true};
+		expect((await rr('o', 'x', config)).out).toBe('a.file\nc.txt\n');
+	});
+
+	test('a line that is a file is used as it is, like an argument (#718)', async () => {
+		makeFiles('a[1].file', 'a1.file');
+		const result = await rr('o', 'x', {pipedData: 'a[1].file\n', globPipe: true});
+		expect(result.out).toBe('a[1].file\n');
+		expect(content('a1.file')).toBe('foo');
+	});
+
+	test('a line that starts with ! takes files away from the other globs', async () => {
+		makeFiles('a.file', 'b.file');
+		const result = await rr('o', 'x', {pipedData: '*.file\n!b.file\n', globPipe: true});
+		expect(result.out).toBe('a.file\n');
+	});
+
+	test('takes Windows line breaks, leaves out empty lines and keeps spaces in names', async () => {
+		makeFiles('a.file', 'my file.txt');
+		const config = {pipedData: '\r\na.file\r\n\r\nmy file.txt\r\n', globPipe: true};
+		expect((await rr('o', 'x', config)).out).toBe('a.file\nmy file.txt\n');
+	});
+
+	test('with --null-in the lines are separated by null bytes', async () => {
+		makeFiles('a.file', 'line\nbreak.file');
+		const config = {pipedData: 'a.file\0line\nbreak.file\0', globPipe: true, nullIn: true};
+		expect((await rr('o', 'x', config)).out).toBe('a.file\nline\nbreak.file\n');
+		expect(content('line\nbreak.file')).toBe('fxx');
+	});
+
+	test('with nothing piped in and no arguments there are no files', async () => {
+		expect(await rr('o', 'x', {pipedData: '', globPipe: true})).toEqual({
+			out: '',
+			err: '0 files found\n',
+			exit: 1,
+		});
+	});
+
+	test('is told about in the status report', async () => {
+		makeFiles('a.file');
+		const status = await rrStatus('o', 'x', {pipedData: 'a.file\n', globPipe: true});
+		expect(status.pipe).toEqual({bytes: 7, type: 'GLOBS', updated: false});
+		expect(status.files).toEqual({searched: ['a.file'], updated: ['a.file']});
+	});
+
+	test.each([
+		['--replacement-pipe', {replacementPipe: true}],
+		['--pipe', {pipe: true}],
+	])('can not be used with %s, as the piped data can only be one thing', async (name, flags) => {
+		makeFiles('a.file');
+		const result = await rr('o', 'x', {pipedData: 'a.file', globPipe: true, ...flags});
+		expect(result.exit).toBe(1);
+		expect(result.err).toContain('--glob-pipe can not be used with');
+		expect(content('a.file')).toBe('foo');
+	});
+});