Add --sort to choose the order of the files to close #792

README.md

@@ -341,6 +341,8 @@ If planned renames clash, the process is halted and no change is applied to any
 | `-B` | **`--void-backup`** Avoid temporary backing up file. Works async (independent of -A flag) and will speed up things but at one point data lives only in memory, and you will lose the content if the process is interrupted. [boolean]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | `-b` | **`--keep-backup`** Keep a backup file of the original content. With --rename the old file is kept. [boolean]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | `-S` | **`--simulate`** Do everything but write the files. The names of the files that would be changed are printed as usual, and no backup is made. [boolean] |
+|      | **`--sort`** Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (alphabetic), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order. [string] [choices: "natural", "ascii", "modified", "created", "size", "none"] |
+|      | **`--sort-reverse`** Turn the order of the files around. [boolean] |
 | `-F` | **`--expect-files`** Check that at least one file is found. The exit code is 1 if not. [boolean] |
 | `-z` | **`--expect-diff`** Check that at least one file is changed. The exit code is 1 if not. [boolean] |
 | `-Z` | **`--expect-zero-diff`** Check that no file is changed. The exit code is 1 if one is. [boolean] |
@@ -372,6 +374,7 @@ Files identified for potential processing are identified by:
 - Remove any file paths that matches any `-x` provided regex. Use `--full-path` to look for the full path.
 - All globs are case-insensitive, like the search, unless `--void-ignore-case` is set.
 - The lines piped in with `--glob-pipe` are input values too, just like the arguments.
+- The files are sorted with `--sort`: natural by default, so `file2` comes before `file10`, and case is ignored.
 
 ### Features
 
@@ -543,15 +546,14 @@ _.oO(What should "sed" have looked like by now?)_
 
 - Default to RegExmoji lax syntax
 - Support RegExmoji strict syntax
-- Add support to require hjson, jsonh, yaml, ini files directly
+- Add support to require hjson, jsonh, json5, yaml, ini files directly
 - Test-run with info outputted about what will happen (sets -t and does not change anything)
 - Let search and replace be within the names of the files (ask for overwriting. -Y = no questions)
 - Let search and replace be within the path of the files (ask for overwriting. -Y = no questions)
 - Let pattern and globs be piped
 - Let pipe be placed into search or replace.
 - Let Pattern, replacement, and globs come from a file
 - Let pattern and glob be javascript code returning a string as the result
-- Auto string search / replace if no regex magic is used (and verify that speed is better)
 - Flag for String-literal (no regex, no special chars, no escape chars) to avoid backslashes or remembering which characters needs to be escaped
 - Check if https://github.com/eugeneware/replacestream is good to rely on
 - Check if regex engine from spider monkey can be wrapped in something that does not need node

src/cli.ts

@@ -265,6 +265,21 @@ const yargs = require('yargs')
 	)
 	.alias('S', 'simulate')
 
+	.string('sort')
+	.describe(
+		'sort',
+		'Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (alphabetic), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order.'
+	)
+	// Any case, and true and yes are natural
+	.coerce('sort', value => {
+		const name = String(value).toLowerCase();
+		return 'true' === name || 'yes' === name ? 'natural' : name;
+	})
+	.choices('sort', ['natural', 'ascii', 'modified', 'created', 'size', 'none'])
+
+	.boolean('sort-reverse')
+	.describe('sort-reverse', 'Turn the order of the files around.')
+
 	// Checks for a CI step. When one is not met, the exit code is 1.
 	.boolean('F')
 	.describe('F', 'Check that at least one file is found. The exit code is 1 if not.')

src/engine.ts

@@ -197,9 +197,14 @@ async function run(config) {
 
 	chat(config.files.length + ' files found');
 
-	// One file needs no async flow, and with -o or -m one file at a time gives the output in order
+	// One file needs no async flow, and with -o, -m or --sort one file at a time gives the output in order
 	config.voidAsync =
-		config.voidAsync || 1 === config.files.length || config.output || config.outputMatch;
+		config.voidAsync ||
+		1 === config.files.length ||
+		config.output ||
+		config.outputMatch ||
+		null != config.sort ||
+		config.sortReverse;
 
 	const files = config.files.filter(filepath =>
 		fs.existsSync(filepath) ? true : error('File not found:', filepath)
@@ -1025,7 +1030,9 @@ function getFilePaths(conf) {
 	// All globs follow the search: case-insensitive, unless --void-ignore-case
 	const globOptions = {caseSensitiveMatch: Boolean(conf.voidIgnoreCase)};
 
-	let files = findFiles(includeGlob, globOptions);
+	const sort = sortName(conf.sort);
+
+	let files = findFiles(includeGlob, globOptions, sort);
 
 	// With --full-path each file is known by its full path from here on. That is what --exclude-re sees, and the name that is printed.
 	if (conf.fullPath) {
@@ -1044,12 +1051,64 @@ function getFilePaths(conf) {
 			files = files.filter(file => !file.match(re));
 		});
 
-	return files.sort();
+	files = sortFiles(files, sort, conf);
+	return conf.sortReverse ? files.reverse() : files;
+}
+
+const sorts = ['natural', 'ascii', 'modified', 'created', 'size', 'none'];
+
+// The value of --sort in any case, where `true` and `yes` are `natural`. Without --sort it is `natural`.
+function sortName(value) {
+	if (undefined === value || null === value) {
+		return 'natural';
+	}
+	const name = String(value).toLowerCase();
+	if ('true' === name || 'yes' === name) {
+		return 'natural';
+	}
+	if (!sorts.includes(name)) {
+		panic('--sort must be one of ' + sorts.join(', ') + ', not:', String(value));
+	}
+	return name;
+}
+
+const natural = new Intl.Collator(undefined, {numeric: true, sensitivity: 'base'}).compare;
+const ascii = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+
+function sortFiles(files, sort, conf) {
+	// One file has no order, so it is not sorted, and needs no stat
+	if (files.length < 2 || 'none' === sort) {
+		return files;
+	}
+	if ('natural' === sort) {
+		return files.sort(natural);
+	}
+	if ('ascii' === sort) {
+		return files.sort(ascii);
+	}
+
+	// By a number from each file, and when two are the same, by ascii
+	const stats = new Map(files.map(file => [file, fs.statSync(file)]));
+	let key = {
+		modified: stat => stat.mtimeMs,
+		created: stat => stat.birthtimeMs,
+		size: stat => stat.size,
+	}[sort];
+
+	// Not all file systems keep the time a file was made. Then Node gives 0.
+	if ('created' === sort && [...stats.values()].some(stat => !stat.birthtimeMs)) {
+		warn(
+			'The file system does not keep the time a file was made, so --sort created uses the modified time.'
+		);
+		key = stat => stat.mtimeMs;
+	}
+
+	return files.sort((a, b) => key(stats.get(a)) - key(stats.get(b)) || ascii(a, b));
 }
 
 // A path that is a file is used as it is, so a name with glob characters, like `a[1].md` or `my (1).md`, gives only that file.
 // The other paths are globbed together, so a pattern that starts with `!` can take files away from the other patterns.
-function findFiles(paths, globOptions) {
+function findFiles(paths, globOptions, sort) {
 	const files = [];
 	const patterns = [];
 	paths.forEach(file => {
@@ -1062,7 +1121,9 @@ function findFiles(paths, globOptions) {
 
 	// A file given by its name and also found by a glob is only handled once
 	const seen = new Set();
-	return files.concat(patterns.length ? globs.sync(patterns, globOptions) : []).filter(file => {
+	// A glob has no order of its own, so with --sort none its files come after the paths, sorted with natural
+	const found = patterns.length ? globs.sync(patterns, globOptions) : [];
+	return files.concat('none' === sort ? found.sort(natural) : found).filter(file => {
 		const fullPath = path.resolve(file);
 		if (seen.has(fullPath)) {
 			return false;

test/cli/run.sh

@@ -148,7 +148,7 @@ assert		 		"cat my.file"    "foo\nar"
 # Globs
 reset
 echo barfoo > my_file
-assert		 		"rexreplace o x 'my*le' -o"    "fxxbar\nbarfxx"
+assert		 		"rexreplace o x 'my*le' -o"    "barfxx\nfxxbar"
 rm my_file
 
 # A path to a file is used as it is, also with glob characters in its name
@@ -267,6 +267,11 @@ assert_raises		"rexreplace 'oo' 'x' my.file -z"    0
 assert_raises		"rexreplace 'oo' 'x' nope.file --expect-files"    1
 
 
+# --sort decides the order of the files, and --sort-reverse turns it around
+reset
+assert		 		"rexreplace 'b' 'x' my.file your.file --sort-reverse --name-delimiter ','"    'your.file,my.file,'
+
+
 # --full-path prints the full path of each name
 reset
 assert		 		"rexreplace 'b' 'x' my.file --full-path"    "$(pwd -P)/my.file"

test/cli/sort.test.js

@@ -0,0 +1,22 @@
+// With --sort the files are handled one at a time, so the names are printed in the order of the sort
+import {describe, test, expect} from 'bun:test';
+import fs from 'fs';
+import {$} from 'bun';
+import {cli, sh} from '../helper.js';
+
+describe('--sort', () => {
+	test('prints the names in the order of the sort, also with many files', async () => {
+		const names = Array.from({length: 40}, (_, i) => `file${i}.md`);
+		// The first name gets the largest file, so the order by size is the other way
+		names.forEach((name, i) => fs.writeFileSync(name, 'o'.repeat(40 - i)));
+		const result = await sh($`bun ${cli} o x '*.md' --sort size`);
+		expect(result.out).toBe([...names].reverse().join('\n') + '\n');
+	});
+
+	test('must have a value', async () => {
+		fs.writeFileSync('a.md', 'foo');
+		const result = await sh($`bun ${cli} o x a.md --sort`);
+		expect(result.exit).toBe(3);
+		expect(fs.readFileSync('a.md', 'utf8')).toBe('foo');
+	});
+});

test/engine/sort-reverse.test.js

@@ -0,0 +1,20 @@
+import {describe, test, expect} from 'bun:test';
+import fs from 'fs';
+import {rr} from '../helper.js';
+
+describe('--sort-reverse', () => {
+	test('turns the order around', async () => {
+		fs.writeFileSync('file10.md', 'foo');
+		fs.writeFileSync('file2.md', 'foo');
+		expect((await rr('o', 'x', {includeGlob: ['*.md'], sortReverse: true})).out).toBe(
+			'file10.md\nfile2.md\n'
+		);
+	});
+
+	test('works with the other sorts', async () => {
+		fs.writeFileSync('a.md', 'ooo');
+		fs.writeFileSync('b.md', 'o');
+		const config = {includeGlob: ['*.md'], sort: 'size', sortReverse: true};
+		expect((await rr('o', 'x', config)).out).toBe('a.md\nb.md\n');
+	});
+});

test/engine/sort.test.js

@@ -0,0 +1,86 @@
+import {describe, test, expect} from 'bun:test';
+import fs from 'fs';
+import {rr} from '../helper.js';
+
+describe('--sort', () => {
+	test('is natural by default: numbers as numbers, and case ignored', async () => {
+		fs.writeFileSync('file10.md', 'foo');
+		fs.writeFileSync('file2.md', 'foo');
+		fs.writeFileSync('B.md', 'foo');
+		fs.writeFileSync('a.md', 'foo');
+		expect((await rr('o', 'x', {includeGlob: ['*.md']})).out).toBe(
+			'a.md\nB.md\nfile2.md\nfile10.md\n'
+		);
+	});
+
+	test('ascii sorts by char code', async () => {
+		fs.writeFileSync('file10.md', 'foo');
+		fs.writeFileSync('file2.md', 'foo');
+		fs.writeFileSync('B.md', 'foo');
+		fs.writeFileSync('a.md', 'foo');
+		expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'ascii'})).out).toBe(
+			'B.md\na.md\nfile10.md\nfile2.md\n'
+		);
+	});
+
+	test.each([['NATURAL'], ['Natural'], ['true'], ['yes'], ['YES']])(
+		'%s is natural',
+		async value => {
+			fs.writeFileSync('file10.md', 'foo');
+			fs.writeFileSync('file2.md', 'foo');
+			expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: value})).out).toBe(
+				'file2.md\nfile10.md\n'
+			);
+		}
+	);
+
+	test('none keeps the paths in the order given, and then the files from globs, sorted natural', async () => {
+		fs.writeFileSync('c.md', 'foo');
+		fs.writeFileSync('a.md', 'foo');
+		fs.writeFileSync('file10.txt', 'foo');
+		fs.writeFileSync('file2.txt', 'foo');
+		const config = {includeGlob: ['c.md', '*.txt', 'a.md'], sort: 'none'};
+		expect((await rr('o', 'x', config)).out).toBe('c.md\na.md\nfile2.txt\nfile10.txt\n');
+	});
+
+	test('modified puts the oldest first, and ascii decides when they are the same', async () => {
+		fs.writeFileSync('a.md', 'foo');
+		fs.writeFileSync('b.md', 'foo');
+		fs.writeFileSync('c.md', 'foo');
+		fs.utimesSync('a.md', 3000, 3000);
+		fs.utimesSync('b.md', 1000, 1000);
+		fs.utimesSync('c.md', 1000, 1000);
+		expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'modified'})).out).toBe(
+			'b.md\nc.md\na.md\n'
+		);
+	});
+
+	test('created puts the oldest first, or uses the modified time where the file system has no time for it', async () => {
+		for (const name of ['c.md', 'b.md', 'a.md']) {
+			fs.writeFileSync(name, 'foo');
+			await Bun.sleep(20);
+		}
+		expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'created'})).out).toBe(
+			'c.md\nb.md\na.md\n'
+		);
+	});
+
+	test('size puts the smallest first, and ascii decides when they are the same', async () => {
+		fs.writeFileSync('a.md', 'ooo');
+		fs.writeFileSync('b.md', 'o');
+		fs.writeFileSync('c.md', 'o');
+		expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'size'})).out).toBe(
+			'b.md\nc.md\na.md\n'
+		);
+	});
+
+	test.each([['nope'], ['']])('%j stops the run, and tells the values', async value => {
+		fs.writeFileSync('a.md', 'foo');
+		const result = await rr('o', 'x', {includeGlob: ['a.md'], sort: value});
+		expect(result.err).toStartWith(
+			' ❌ --sort must be one of natural, ascii, modified, created, size, none'
+		);
+		expect(result.exit).toBe(3);
+		expect(fs.readFileSync('a.md', 'utf8')).toBe('foo');
+	});
+});

test/fix/716.test.js

@@ -10,7 +10,7 @@ describe('#716 the globs are case-insensitive like the search', () => {
 		fs.writeFileSync('a.file', 'foo');
 		fs.writeFileSync('B.FILE', 'foo');
 		expect(await rr('o', 'x', {includeGlob: ['*.file']})).toEqual({
-			out: 'B.FILE\na.file\n',
+			out: 'a.file\nB.FILE\n',
 			err: '',
 			exit: null,
 		});
@@ -32,6 +32,6 @@ describe('#716 the globs are case-insensitive like the search', () => {
 		expect((await rr('o', 'x', config)).out).toBe('a.file\n');
 		fs.writeFileSync('a.file', 'foo');
 		const excluded = {includeGlob: ['*.*'], excludeGlob: ['b.*'], voidIgnoreCase: true};
-		expect((await rr('o', 'x', excluded)).out).toBe('B.FILE\na.file\n');
+		expect((await rr('o', 'x', excluded)).out).toBe('a.file\nB.FILE\n');
 	});
 });

test/fix/718.test.js

@@ -51,7 +51,7 @@ describe('#718 together with globs', () => {
 	test('a path that is not a file is still a glob', async () => {
 		fs.writeFileSync('a[1].file', 'foo');
 		fs.writeFileSync('a1.file', 'foo');
-		expect((await rr('o', 'x', {includeGlob: ['*.file']})).out).toBe('a1.file\na[1].file\n');
+		expect((await rr('o', 'x', {includeGlob: ['*.file']})).out).toBe('a[1].file\na1.file\n');
 	});
 
 	test('a path to a folder is not a file, so it is still a glob', async () => {