From 824b64f0aebab7fcfe5e7d9aec9bab089fdfc792 Mon Sep 17 00:00:00 2001 From: Mathias Wulff Date: Fri, 23 Oct 2026 12:34:48 +1100 Subject: [PATCH] Test UTF-8 in all inputs and outputs to close #735 --- test/cli/run.sh | 11 ++++ test/js/utf8.test.js | 131 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 test/js/utf8.test.js diff --git a/test/cli/run.sh b/test/cli/run.sh index c72984fb..bba288cf 100644 --- a/test/cli/run.sh +++ b/test/cli/run.sh @@ -451,6 +451,17 @@ assert "rexreplace 'b' 'x' my.file --content-delimiter '\x4' 2>&1 | grep -c assert "rexreplace 'b' 'x' my.file --delimiter '\x4' 2>&1 | grep -c 'Invalid escape sequence in --delimiter'" '1' assert "cat my.file" 'foobar' +# UTF-8 goes in and out as it is (#735) +reset +assert "printf 'a🦖b' | rexreplace '🦖' '👩‍💻'" 'a👩‍💻b' +# Without --unicode . is half an emoji, so 'x.y' does not match an emoji between x and y +assert "printf 'x🦖y' | rexreplace 'x.y' '_'" 'x🦖y' +assert "printf 'x🦖y' | rexreplace 'x.y' '_' --unicode" '_' +echo foo > '🦖 中.file' +assert "rexreplace 'o' 'x' '🦖 中.file'" '🦖 中.file' +assert "cat '🦖 中.file'" 'fxx' + + # --glob-pipe: each line of the piped data is a path or a glob to work on (#711) reset assert "printf 'my.file\nyour.file\n' | rexreplace 'b' 'x' --glob-pipe -A" 'my.file\nyour.file' diff --git a/test/js/utf8.test.js b/test/js/utf8.test.js new file mode 100644 index 00000000..1675052e --- /dev/null +++ b/test/js/utf8.test.js @@ -0,0 +1,131 @@ +// https://github.com/mathiasrw/rexreplace/issues/735 +// Emoji and other UTF-8 in every way data comes in and goes out. +import {describe, test, expect} from 'bun:test'; +import path from 'path'; +import {rr} from './helper.js'; + +// rr() gives stdout as a JavaScript string. This is what real stdout (or a file) gets, so half an emoji shows up as `�`. +const asUtf8 = text => Buffer.from(text).toString(); + +const text = 'æ 中 🦖 👩‍💻\r\n'; + +describe('#735 UTF-8 in the pattern', () => { + test('is found as it is', async () => { + expect(asUtf8((await rr('🦖', 'x', {pipedData: text})).out)).toBe('æ 中 x 👩‍💻\r\n'); + }); + + test('. takes half an emoji without --unicode, which then shows as �', async () => { + const out = (await rr('.', 'x', {pipedData: '🦖a', voidGlobal: true})).out; + expect(asUtf8(out)).toBe('x�a'); + }); + + test('. takes the whole emoji with --unicode', async () => { + const config = {pipedData: '🦖a', voidGlobal: true, unicode: true}; + expect(asUtf8((await rr('.', 'x', config)).out)).toBe('xa'); + }); + + test('is plain text with --literal', async () => { + const config = {pipedData: 'a中.🦖b 中x🦖', literal: true}; + expect(asUtf8((await rr('中.🦖', 'x', config)).out)).toBe('axb 中x🦖'); + }); +}); + +describe('#735 UTF-8 in the replacement', () => { + test('is inserted as it is', async () => { + expect(asUtf8((await rr('a', '🦖', {pipedData: 'aba'})).out)).toBe('🦖b🦖'); + }); + + test('works with the € and § aliases', async () => { + expect(asUtf8((await rr('(🦖)', '€1€1', {pipedData: 'a🦖b'})).out)).toBe('a🦖🦖b'); + expect(asUtf8((await rr('§d', '🦖', {pipedData: 'a1b'})).out)).toBe('a🦖b'); + }); + + test('can come from --replacement-js, where an emoji has the length 2', async () => { + const config = {pipedData: 'a🦖', replacementJs: true}; + expect(asUtf8((await rr('🦖', '$0 + $0.length', config)).out)).toBe('a🦖2'); + }); +}); + +describe('#735 UTF-8 in the content of a file', () => { + test('is written as UTF-8 when changed in place', async () => { + await Bun.write('my.file', text); + await rr('🦖', '🦕', {includeGlob: ['my.file']}); + expect(Buffer.from(await Bun.file('my.file').arrayBuffer())).toEqual( + Buffer.from('æ 中 🦕 👩‍💻\r\n') + ); + }); + + test('is printed with --output', async () => { + await Bun.write('my.file', text); + const out = (await rr('🦖', '🦕', {includeGlob: ['my.file'], output: true})).out; + expect(asUtf8(out)).toBe('æ 中 🦕 👩‍💻\r\n'); + }); + + test('is printed with --output-match', async () => { + await Bun.write('my.file', text); + const config = {includeGlob: ['my.file'], outputMatch: true}; + expect(asUtf8((await rr('🦖|👩‍💻', '_', config)).out)).toBe('🦖\n👩‍💻\n'); + }); +}); + +describe('#735 UTF-8 in piped data', () => { + test('comes out as it went in, with \\r\\n', async () => { + expect(asUtf8((await rr('中', 'x', {pipedData: text})).out)).toBe('æ x 🦖 👩‍💻\r\n'); + }); + + test('can be the replacement with --replacement-pipe', async () => { + await Bun.write('my.file', 'aXb'); + await rr('X', '_', {includeGlob: ['my.file'], replacementPipe: true, pipedData: '🦖'}); + expect(await Bun.file('my.file').text()).toBe('a🦖b'); + }); +}); + +describe('#735 UTF-8 in file names', () => { + const name = '🦖 中.file'; + + test('a name is printed as it was given', async () => { + await Bun.write(name, 'foo'); + expect(asUtf8((await rr('o', 'x', {includeGlob: [name]})).out)).toBe(name + '\n'); + expect(await Bun.file(name).text()).toBe('fxx'); + }); + + test('a glob finds a name with emoji', async () => { + await Bun.write(name, 'foo'); + await Bun.write('other.file', 'foo'); + expect(asUtf8((await rr('o', 'x', {includeGlob: ['*🦖*']})).out)).toBe(name + '\n'); + }); + + test('a name is printed with its full path with --full-path', async () => { + await Bun.write(name, 'foo'); + const out = (await rr('o', 'x', {includeGlob: [name], fullPath: true})).out; + expect(asUtf8(out)).toBe(path.resolve(name) + '\n'); + }); +}); + +describe('#735 UTF-8 in the delimiters', () => { + async function makeFiles() { + await Bun.write('a.file', 'foo'); + await Bun.write('b.file', 'foo'); + return {includeGlob: ['*.file']}; + } + + test('--name-delimiter, also with no tail', async () => { + const config = {...(await makeFiles()), nameDelimiter: '🦖'}; + expect(asUtf8((await rr('o', 'x', config)).out)).toBe('a.file🦖b.file🦖'); + await makeFiles(); + const noTail = {...config, nameDelimiterTail: false}; + expect(asUtf8((await rr('o', 'x', noTail)).out)).toBe('a.file🦖b.file'); + }); + + test('--content-delimiter with a combined emoji', async () => { + const config = {...(await makeFiles()), output: true, contentDelimiter: '👩‍💻'}; + expect(asUtf8((await rr('o', 'x', config)).out)).toBe('fxx👩‍💻fxx👩‍💻'); + }); + + test('--delimiter sets both', async () => { + const config = {...(await makeFiles()), output: true, delimiter: '🦕'}; + const result = await rr('o', 'x', config); + expect(asUtf8(result.out)).toBe('fxx🦕fxx🦕'); + expect(asUtf8(result.err)).toBe('a.file🦕b.file🦕'); + }); +});