diff --git a/README.md b/README.md index 957aacac..a2f442ed 100644 --- a/README.md +++ b/README.md @@ -341,6 +341,8 @@ If planned renames clash, the process is halted and no change is applied to any | `-B` | **`--void-backup`** Avoid temporary backing up file. Works async (independent of -A flag) and will speed up things but at one point data lives only in memory, and you will lose the content if the process is interrupted. [boolean] | | `-b` | **`--keep-backup`** Keep a backup file of the original content. With --rename the old file is kept. [boolean] | | `-S` | **`--simulate`** Do everything but write the files. The names of the files that would be changed are printed as usual, and no backup is made. [boolean] | +| | **`--sort`** Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (alphabetic), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order. [string] [choices: "natural", "ascii", "modified", "created", "size", "none"] | +| | **`--sort-reverse`** Turn the order of the files around. [boolean] | | `-F` | **`--expect-files`** Check that at least one file is found. The exit code is 1 if not. [boolean] | | `-z` | **`--expect-diff`** Check that at least one file is changed. The exit code is 1 if not. [boolean] | | `-Z` | **`--expect-zero-diff`** Check that no file is changed. The exit code is 1 if one is. [boolean] | @@ -372,6 +374,7 @@ Files identified for potential processing are identified by: - Remove any file paths that matches any `-x` provided regex. Use `--full-path` to look for the full path. - All globs are case-insensitive, like the search, unless `--void-ignore-case` is set. - The lines piped in with `--glob-pipe` are input values too, just like the arguments. +- The files are sorted with `--sort`: natural by default, so `file2` comes before `file10`, and case is ignored. ### Features @@ -543,7 +546,7 @@ _.oO(What should "sed" have looked like by now?)_ - Default to RegExmoji lax syntax - Support RegExmoji strict syntax -- Add support to require hjson, jsonh, yaml, ini files directly +- Add support to require hjson, jsonh, json5, yaml, ini files directly - Test-run with info outputted about what will happen (sets -t and does not change anything) - Let search and replace be within the names of the files (ask for overwriting. -Y = no questions) - Let search and replace be within the path of the files (ask for overwriting. -Y = no questions) @@ -551,7 +554,6 @@ _.oO(What should "sed" have looked like by now?)_ - Let pipe be placed into search or replace. - Let Pattern, replacement, and globs come from a file - Let pattern and glob be javascript code returning a string as the result -- Auto string search / replace if no regex magic is used (and verify that speed is better) - Flag for String-literal (no regex, no special chars, no escape chars) to avoid backslashes or remembering which characters needs to be escaped - Check if https://github.com/eugeneware/replacestream is good to rely on - Check if regex engine from spider monkey can be wrapped in something that does not need node diff --git a/src/cli.ts b/src/cli.ts index 25ef77b5..a281c517 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -265,6 +265,21 @@ const yargs = require('yargs') ) .alias('S', 'simulate') + .string('sort') + .describe( + 'sort', + 'Process the files one at a time, in a sorted order. The order is chosen from 6 values: `natural` (numbers compared as numbers, and case ignored), `ascii` (alphabetic), `modified`, `created`, `size`, or `none` (the order of the input: the files given first, and then the files found by globs). Files that sort the same (like the same size) are ordered by ascii sort. Without --sort the files are processed asynchronously, so the order of the output can not be guaranteed, but the processing is started in natural order.' + ) + // Any case, and true and yes are natural + .coerce('sort', value => { + const name = String(value).toLowerCase(); + return 'true' === name || 'yes' === name ? 'natural' : name; + }) + .choices('sort', ['natural', 'ascii', 'modified', 'created', 'size', 'none']) + + .boolean('sort-reverse') + .describe('sort-reverse', 'Turn the order of the files around.') + // Checks for a CI step. When one is not met, the exit code is 1. .boolean('F') .describe('F', 'Check that at least one file is found. The exit code is 1 if not.') diff --git a/src/engine.ts b/src/engine.ts index f879d67d..2d1e3bec 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -197,9 +197,14 @@ async function run(config) { chat(config.files.length + ' files found'); - // One file needs no async flow, and with -o or -m one file at a time gives the output in order + // One file needs no async flow, and with -o, -m or --sort one file at a time gives the output in order config.voidAsync = - config.voidAsync || 1 === config.files.length || config.output || config.outputMatch; + config.voidAsync || + 1 === config.files.length || + config.output || + config.outputMatch || + null != config.sort || + config.sortReverse; const files = config.files.filter(filepath => fs.existsSync(filepath) ? true : error('File not found:', filepath) @@ -1025,7 +1030,9 @@ function getFilePaths(conf) { // All globs follow the search: case-insensitive, unless --void-ignore-case const globOptions = {caseSensitiveMatch: Boolean(conf.voidIgnoreCase)}; - let files = findFiles(includeGlob, globOptions); + const sort = sortName(conf.sort); + + let files = findFiles(includeGlob, globOptions, sort); // With --full-path each file is known by its full path from here on. That is what --exclude-re sees, and the name that is printed. if (conf.fullPath) { @@ -1044,12 +1051,64 @@ function getFilePaths(conf) { files = files.filter(file => !file.match(re)); }); - return files.sort(); + files = sortFiles(files, sort, conf); + return conf.sortReverse ? files.reverse() : files; +} + +const sorts = ['natural', 'ascii', 'modified', 'created', 'size', 'none']; + +// The value of --sort in any case, where `true` and `yes` are `natural`. Without --sort it is `natural`. +function sortName(value) { + if (undefined === value || null === value) { + return 'natural'; + } + const name = String(value).toLowerCase(); + if ('true' === name || 'yes' === name) { + return 'natural'; + } + if (!sorts.includes(name)) { + panic('--sort must be one of ' + sorts.join(', ') + ', not:', String(value)); + } + return name; +} + +const natural = new Intl.Collator(undefined, {numeric: true, sensitivity: 'base'}).compare; +const ascii = (a, b) => (a < b ? -1 : a > b ? 1 : 0); + +function sortFiles(files, sort, conf) { + // One file has no order, so it is not sorted, and needs no stat + if (files.length < 2 || 'none' === sort) { + return files; + } + if ('natural' === sort) { + return files.sort(natural); + } + if ('ascii' === sort) { + return files.sort(ascii); + } + + // By a number from each file, and when two are the same, by ascii + const stats = new Map(files.map(file => [file, fs.statSync(file)])); + let key = { + modified: stat => stat.mtimeMs, + created: stat => stat.birthtimeMs, + size: stat => stat.size, + }[sort]; + + // Not all file systems keep the time a file was made. Then Node gives 0. + if ('created' === sort && [...stats.values()].some(stat => !stat.birthtimeMs)) { + warn( + 'The file system does not keep the time a file was made, so --sort created uses the modified time.' + ); + key = stat => stat.mtimeMs; + } + + return files.sort((a, b) => key(stats.get(a)) - key(stats.get(b)) || ascii(a, b)); } // A path that is a file is used as it is, so a name with glob characters, like `a[1].md` or `my (1).md`, gives only that file. // The other paths are globbed together, so a pattern that starts with `!` can take files away from the other patterns. -function findFiles(paths, globOptions) { +function findFiles(paths, globOptions, sort) { const files = []; const patterns = []; paths.forEach(file => { @@ -1062,7 +1121,9 @@ function findFiles(paths, globOptions) { // A file given by its name and also found by a glob is only handled once const seen = new Set(); - return files.concat(patterns.length ? globs.sync(patterns, globOptions) : []).filter(file => { + // A glob has no order of its own, so with --sort none its files come after the paths, sorted with natural + const found = patterns.length ? globs.sync(patterns, globOptions) : []; + return files.concat('none' === sort ? found.sort(natural) : found).filter(file => { const fullPath = path.resolve(file); if (seen.has(fullPath)) { return false; diff --git a/test/cli/run.sh b/test/cli/run.sh index 920beb57..bda82a13 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -148,7 +148,7 @@ assert "cat my.file" "foo\nar" # Globs reset echo barfoo > my_file -assert "rexreplace o x 'my*le' -o" "fxxbar\nbarfxx" +assert "rexreplace o x 'my*le' -o" "barfxx\nfxxbar" rm my_file # A path to a file is used as it is, also with glob characters in its name @@ -267,6 +267,11 @@ assert_raises "rexreplace 'oo' 'x' my.file -z" 0 assert_raises "rexreplace 'oo' 'x' nope.file --expect-files" 1 +# --sort decides the order of the files, and --sort-reverse turns it around +reset +assert "rexreplace 'b' 'x' my.file your.file --sort-reverse --name-delimiter ','" 'your.file,my.file,' + + # --full-path prints the full path of each name reset assert "rexreplace 'b' 'x' my.file --full-path" "$(pwd -P)/my.file" diff --git a/test/cli/sort.test.js b/test/cli/sort.test.js new file mode 100644 index 00000000..cfe0868d --- /dev/null +++ b/test/cli/sort.test.js @@ -0,0 +1,22 @@ +// With --sort the files are handled one at a time, so the names are printed in the order of the sort +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {$} from 'bun'; +import {cli, sh} from '../helper.js'; + +describe('--sort', () => { + test('prints the names in the order of the sort, also with many files', async () => { + const names = Array.from({length: 40}, (_, i) => `file${i}.md`); + // The first name gets the largest file, so the order by size is the other way + names.forEach((name, i) => fs.writeFileSync(name, 'o'.repeat(40 - i))); + const result = await sh($`bun ${cli} o x '*.md' --sort size`); + expect(result.out).toBe([...names].reverse().join('\n') + '\n'); + }); + + test('must have a value', async () => { + fs.writeFileSync('a.md', 'foo'); + const result = await sh($`bun ${cli} o x a.md --sort`); + expect(result.exit).toBe(3); + expect(fs.readFileSync('a.md', 'utf8')).toBe('foo'); + }); +}); diff --git a/test/engine/sort-reverse.test.js b/test/engine/sort-reverse.test.js new file mode 100644 index 00000000..330dc588 --- /dev/null +++ b/test/engine/sort-reverse.test.js @@ -0,0 +1,20 @@ +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('--sort-reverse', () => { + test('turns the order around', async () => { + fs.writeFileSync('file10.md', 'foo'); + fs.writeFileSync('file2.md', 'foo'); + expect((await rr('o', 'x', {includeGlob: ['*.md'], sortReverse: true})).out).toBe( + 'file10.md\nfile2.md\n' + ); + }); + + test('works with the other sorts', async () => { + fs.writeFileSync('a.md', 'ooo'); + fs.writeFileSync('b.md', 'o'); + const config = {includeGlob: ['*.md'], sort: 'size', sortReverse: true}; + expect((await rr('o', 'x', config)).out).toBe('a.md\nb.md\n'); + }); +}); diff --git a/test/engine/sort.test.js b/test/engine/sort.test.js new file mode 100644 index 00000000..a6ef1e26 --- /dev/null +++ b/test/engine/sort.test.js @@ -0,0 +1,86 @@ +import {describe, test, expect} from 'bun:test'; +import fs from 'fs'; +import {rr} from '../helper.js'; + +describe('--sort', () => { + test('is natural by default: numbers as numbers, and case ignored', async () => { + fs.writeFileSync('file10.md', 'foo'); + fs.writeFileSync('file2.md', 'foo'); + fs.writeFileSync('B.md', 'foo'); + fs.writeFileSync('a.md', 'foo'); + expect((await rr('o', 'x', {includeGlob: ['*.md']})).out).toBe( + 'a.md\nB.md\nfile2.md\nfile10.md\n' + ); + }); + + test('ascii sorts by char code', async () => { + fs.writeFileSync('file10.md', 'foo'); + fs.writeFileSync('file2.md', 'foo'); + fs.writeFileSync('B.md', 'foo'); + fs.writeFileSync('a.md', 'foo'); + expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'ascii'})).out).toBe( + 'B.md\na.md\nfile10.md\nfile2.md\n' + ); + }); + + test.each([['NATURAL'], ['Natural'], ['true'], ['yes'], ['YES']])( + '%s is natural', + async value => { + fs.writeFileSync('file10.md', 'foo'); + fs.writeFileSync('file2.md', 'foo'); + expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: value})).out).toBe( + 'file2.md\nfile10.md\n' + ); + } + ); + + test('none keeps the paths in the order given, and then the files from globs, sorted natural', async () => { + fs.writeFileSync('c.md', 'foo'); + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('file10.txt', 'foo'); + fs.writeFileSync('file2.txt', 'foo'); + const config = {includeGlob: ['c.md', '*.txt', 'a.md'], sort: 'none'}; + expect((await rr('o', 'x', config)).out).toBe('c.md\na.md\nfile2.txt\nfile10.txt\n'); + }); + + test('modified puts the oldest first, and ascii decides when they are the same', async () => { + fs.writeFileSync('a.md', 'foo'); + fs.writeFileSync('b.md', 'foo'); + fs.writeFileSync('c.md', 'foo'); + fs.utimesSync('a.md', 3000, 3000); + fs.utimesSync('b.md', 1000, 1000); + fs.utimesSync('c.md', 1000, 1000); + expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'modified'})).out).toBe( + 'b.md\nc.md\na.md\n' + ); + }); + + test('created puts the oldest first, or uses the modified time where the file system has no time for it', async () => { + for (const name of ['c.md', 'b.md', 'a.md']) { + fs.writeFileSync(name, 'foo'); + await Bun.sleep(20); + } + expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'created'})).out).toBe( + 'c.md\nb.md\na.md\n' + ); + }); + + test('size puts the smallest first, and ascii decides when they are the same', async () => { + fs.writeFileSync('a.md', 'ooo'); + fs.writeFileSync('b.md', 'o'); + fs.writeFileSync('c.md', 'o'); + expect((await rr('o', 'x', {includeGlob: ['*.md'], sort: 'size'})).out).toBe( + 'b.md\nc.md\na.md\n' + ); + }); + + test.each([['nope'], ['']])('%j stops the run, and tells the values', async value => { + fs.writeFileSync('a.md', 'foo'); + const result = await rr('o', 'x', {includeGlob: ['a.md'], sort: value}); + expect(result.err).toStartWith( + ' ❌ --sort must be one of natural, ascii, modified, created, size, none' + ); + expect(result.exit).toBe(3); + expect(fs.readFileSync('a.md', 'utf8')).toBe('foo'); + }); +}); diff --git a/test/fix/716.test.js b/test/fix/716.test.js index 1873dae4..57efa2d4 100644 --- a/test/fix/716.test.js +++ b/test/fix/716.test.js @@ -10,7 +10,7 @@ describe('#716 the globs are case-insensitive like the search', () => { fs.writeFileSync('a.file', 'foo'); fs.writeFileSync('B.FILE', 'foo'); expect(await rr('o', 'x', {includeGlob: ['*.file']})).toEqual({ - out: 'B.FILE\na.file\n', + out: 'a.file\nB.FILE\n', err: '', exit: null, }); @@ -32,6 +32,6 @@ describe('#716 the globs are case-insensitive like the search', () => { expect((await rr('o', 'x', config)).out).toBe('a.file\n'); fs.writeFileSync('a.file', 'foo'); const excluded = {includeGlob: ['*.*'], excludeGlob: ['b.*'], voidIgnoreCase: true}; - expect((await rr('o', 'x', excluded)).out).toBe('B.FILE\na.file\n'); + expect((await rr('o', 'x', excluded)).out).toBe('a.file\nB.FILE\n'); }); }); diff --git a/test/fix/718.test.js b/test/fix/718.test.js index c25d5147..e3237b12 100644 --- a/test/fix/718.test.js +++ b/test/fix/718.test.js @@ -51,7 +51,7 @@ describe('#718 together with globs', () => { test('a path that is not a file is still a glob', async () => { fs.writeFileSync('a[1].file', 'foo'); fs.writeFileSync('a1.file', 'foo'); - expect((await rr('o', 'x', {includeGlob: ['*.file']})).out).toBe('a1.file\na[1].file\n'); + expect((await rr('o', 'x', {includeGlob: ['*.file']})).out).toBe('a[1].file\na1.file\n'); }); test('a path to a folder is not a file, so it is still a glob', async () => {