Step 11: alasql/esm, and the build without keepNames

build.js

@@ -20,6 +20,11 @@
 	                            plugin does what the rexreplace passes of build.sh did), inside the
 	                            UMD wrapper
 	  dist/alasql.min.js        the same, minified by esbuild before the wrapper is added
+	  dist/alasql.mjs           ES module for Node (alasql/esm): the same bundle in esbuild's esm
+	                            format, with require and __dirname made from import.meta.url at
+	                            the top, because the code keeps its lazy require() calls
+	  dist/alasql.browser.mjs   ES module for browsers and bundlers (alasql/esm, default
+	                            condition): the browser file's content, no Node import in it
 	  dist/alasql-worker.js     the worker, built the same way from the worker files
 	  dist/precompile/index.js  src/precompile/index.js as CommonJS, the dist file left external
 	  dist/alasql.d.ts, dist/alasql-echo.js, dist/alasql-prolog.js: copied
@@ -31,10 +36,12 @@
 	  output of an ES module exports {default: alasql}, and require('alasql') must give the function.
 	- The UMD wrapper for the browser files, by hand. esbuild has no UMD format. The wrapper gives
 	  the body a module object and returns module.exports.
-	- keepNames for the unminified files. When esbuild bundles, a name declared in a nested scope
-	  that shadows one of an enclosing scope gets a suffix (the class Table assigned to yy.Table
-	  beside var Table, the function extend inside var extend), which would change the .name of
-	  four functions and classes; keepNames keeps them. The minified file had mangled names before.
+	- No keepNames. It was on for the unminified files until the end of Phase 1, to keep the .name
+	  of functions whose identifier esbuild suffixes when a nested scope shadows an enclosing one;
+	  but its __name() helper, run at every creation of a named function, made SEARCH 40 percent
+	  slower on Node and 25 on Bun (the parity ledger, row 11). The two names it still protected,
+	  extend and cloneDeep, are function declarations in src/utils/helpers.js now, which esbuild
+	  does not suffix; the API snapshot of the parity check watches every other .name.
 	- __dirname is defined as '' in the browser build, which is what the only-for-browser shim in
 	  10start.js gives the code that shares its scope.
 	- PACKAGE_VERSION and BUILD_VERSION are replaced in the text of every source file, as build.sh
@@ -197,19 +204,27 @@ const sourcePlugin = (browser, version, build) => ({
 
 export async function bundle(
 	entry,
-	{browser = false, minify = false, external = EXTERNALS, version = '', build = ''} = {}
+	{
+		browser = false,
+		minify = false,
+		external = EXTERNALS,
+		version = '',
+		build = '',
+		format = 'cjs',
+		bannerJs = '',
+	} = {}
 ) {
 	const result = await esbuild.build({
 		entryPoints: [entry],
 		bundle: true,
 		write: false,
-		format: 'cjs',
+		format,
 		platform: browser ? 'browser' : 'node',
 		external,
 		minify,
-		keepNames: !minify,
 		legalComments: 'inline',
 		define: browser ? {__dirname: '""'} : undefined,
+		banner: bannerJs ? {js: bannerJs} : undefined,
 		plugins: [sourcePlugin(browser, version, build)],
 		logLevel: 'silent',
 	});
@@ -218,6 +233,17 @@ export async function bundle(
 	return result.outputFiles[0].text.replace(/^"use strict";\n/, '');
 }
 
+// The ES module files, reached as alasql/esm (package.json, exports). The Node one keeps the
+// require() calls of the externals and the __dirname that findAlaSQLPath returns, so it makes
+// both at the top with createRequire and import.meta.url; the browser one has neither, so a
+// bundler finds no Node import in it.
+export const NODE_ESM_BANNER =
+	"import {createRequire as __createRequire} from 'module';\n" +
+	"import {fileURLToPath as __fileURLToPath} from 'url';\n" +
+	"import {dirname as __dirnameOf} from 'path';\n" +
+	'const require = __createRequire(import.meta.url);\n' +
+	'const __dirname = __dirnameOf(__fileURLToPath(import.meta.url));\n';
+
 // esbuild's cjs output of an ES module exports {default: alasql}; the package exports the function
 export function defaultExport(code) {
 	return code + '\nmodule.exports = module.exports.default;\n';
@@ -284,6 +310,16 @@ export async function build() {
 		'alasql.min.js',
 		head + umd(defaultExport(await bundle(entry, {...opts, browser: true, minify: true})), true)
 	);
+	// ES modules are strict on their own, so no directive; esbuild's output ends with the export
+	write(
+		'alasql.mjs',
+		banner(version, buildId) +
+			(await bundle(entry, {...opts, format: 'esm', bannerJs: NODE_ESM_BANNER}))
+	);
+	write(
+		'alasql.browser.mjs',
+		banner(version, buildId) + (await bundle(entry, {...opts, browser: true, format: 'esm'}))
+	);
 
 	// The worker was never strict: src/worker/start.js has no directive and assigns an undeclared
 	// isArray

package.json

@@ -15,6 +15,11 @@
 			"browser": "./dist/alasql.min.js",
 			"default": "./dist/alasql.fs.js"
 		},
+		"./esm": {
+			"types": "./types/alasql.d.ts",
+			"node": "./dist/alasql.mjs",
+			"default": "./dist/alasql.browser.mjs"
+		},
 		"./precompile": "./dist/precompile/index.js"
 	},
 	"directories": {

src/utils/helpers.js

@@ -73,7 +73,7 @@ export var arrayIntersectDeep = function (a, b) {
 /**
   Deep clone objects
   */
-export var cloneDeep = function cloneDeep(obj) {
+export function cloneDeep(obj) {
 	if (null === obj || typeof obj !== 'object') {
 		return obj;
 	}
@@ -98,7 +98,7 @@ export var cloneDeep = function cloneDeep(obj) {
 		}
 	}
 	return temp;
-};
+}
 
 /**
   Compare two objects in deep
@@ -289,15 +289,15 @@ export var distinctArray = function (data) {
   @param {object} b
   @return {object}
   */
-export var extend = function extend(a, b) {
+export function extend(a, b) {
 	a = a || {};
 	for (var key in b) {
 		if (b.hasOwnProperty(key)) {
 			a[key] = b[key];
 		}
 	}
 	return a;
-};
+}
 
 /**
   Double quotes for SQL statements

test/parity/README.md

@@ -61,7 +61,7 @@ If the compare of two recordings of one build is not clean, fix the recorder or
 - `dialect-corpus.js`: runs the dialect cases through alasql under the recorder.
 - `compare.js`: aligns and compares two recordings, the API snapshots and the sizes.
 - `api.js`: `snapshot [dist]` and `compare a.json b.json`.
-- `smoke.js`: the browser global in a `vm` sandbox, the worker file, the CLI, the ESM import, `alasql.min.js` under node, other node binaries (`--node=`), and `--pack`: `npm pack`, install in a temporary folder, then `require`, `import` and the CLI.
+- `smoke.js`: the browser global in a `vm` sandbox, the worker file, the CLI, the ESM import, the two `alasql/esm` files (the Node one, and the browser one under node and through esbuild), `alasql.min.js` under node, other node binaries (`--node=`), and `--pack`: `npm pack`, install in a temporary folder, then `require`, `import` and the CLI.
 - `ledger.md`: bytes and gzip of every `dist` file per step of the conversion, the benchmark against the previous step, and whether two builds gave the same hashes.
 
 ## Not covered

test/parity/ledger.md

@@ -27,6 +27,7 @@ One row per step of `context/roadmap-phase-1.md` (from step 4 the Commit column
 | 9. the worker launcher, the worker head and tail, the finishing file; 97saveas.js, the legacy generator, its tools and four unused numbered files removed; no numbered file left; step 9 complete; under 90 percent: `engine/finish.js` 77 (34 lines, UMD tail and a stray brace out) | #2845 | build.js, esbuild | 929,626 / 142,125 | 925,188 / 141,480 | 524,543 / 115,441 | 4,676 / 2,210 (the launcher is bundled as a module now, not concatenated) | 4,915 / 1,762 | yes | end of the phase: 102 (Bun 101, Node 102) against 4.21.0 on npm, 3 runs, 172 cases, on a quiet machine (load 3, no other test process); it marked subquery +19 (Bun) and +8 (Node), json-path -7 and +16, memory -3 and +13, which a focused run of 7 put at json-path +14 and +9, subquery +13 and +4, core +4 and +8, memory -7 and -1 (Bun and Node). The subquery mark is one case, the JOIN baseline, at +49 to +52 percent on Bun and within 3 on Node: it is the same without keepNames (+51), it is at parity with 4.21.0 when timed alone through `alasql(sql, params)` on Bun, and across the step builds on Bun it reads -31 (1b), +22 (2), -2 (3), -28 (4a), +42 (4b), +44 (5), +52 (9), so it is a JavaScriptCore layout effect of the harness run, not a conversion cost. json-path at +14 on Bun and +9 on Node is the one dimension left above 10 on a runtime |
 | develop merged forward before step 10's Prettier commit: #2798 (044ab07a, `60createtable.js` and `94filestorage.js`); git carried both changes into `src/statements/createtable.js` and `src/storage/filestorage.js` without a conflict, and test1846.js joined the corpus | 35bd5a99 | build.js, esbuild | 929,770 / 142,135 | 925,332 / 141,491 | 524,616 / 115,473 | 4,679 / 2,214 | 4,915 / 1,762 | yes | not run; the reference moved to develop at 044ab07a, built with its own build.sh (`ref-044ab07a`): 0 differences, API identical apart from the 11 allowed; against the old 4.21.0 reference one result differs, the xlsx serial date of test815.js, which is the fixed clock running ahead by the statements test1846.js adds before it (README, "The reference"), not #2798 |
 | 10. Prettier once over the 84 converted modules (commit fac32eef, in `.git-blame-ignore-revs`), the conversion folders out of `.prettierignore`, `no-undef` over the converted modules, CI's formatting check widened to nested folders; step 10 complete | #2847 | build.js, esbuild | 930,140 / 142,209 | 925,702 / 141,564 | 524,616 / 115,472 | 4,679 / 2,214 | 4,915 / 1,762 | yes | test-perf-mini 100 against 4.21.0 on the tip, nothing marked; formatting changes no code, so the end-of-phase figure in row 9 stands. Parity on the tip against `ref-044ab07a`: 0 differences, API identical apart from the 11 allowed; double run 0 unstable |
+| 11. `alasql/esm`: `dist/alasql.mjs` 918,497 / 140,522 (Node, createRequire and __dirname from import.meta.url) and `dist/alasql.browser.mjs` 913,425 / 139,680 (no Node import); then `extend` and `cloneDeep` as function declarations and keepNames dropped, which had made SEARCH 40 percent slower on Node and 25 on Bun since step 1b; the release build | #2848 | build.js, esbuild, no keepNames | 918,768 / 140,210 | 914,325 / 139,568 | 524,607 / 115,453 | 4,672 / 2,207 | 4,789 / 1,728 | yes | end of the phase, on the release build: 99 (Bun 100, Node 99) against 4.21.0 on npm, 3 runs, 172 cases, quiet machine; it marked set-ops +21, coercion +21 and shapes +12 on Bun and mutation +13 on Node, which a focused run of 7 put at set-ops +4 and -2, shapes +1 and -4, coercion -12 and -1, mutation -2 and -10 (Bun and Node), with the single cases behind the marks flipping sign (UPDATE a parameter array +62 then -40 on Node). json-path is -1 and +2, the SEARCH case +3 and +1. The subquery JOIN baseline of row 9 reads -37 on Bun on this build, the same layout effect the other way |
 
 Notes
 

test/parity/run.js

@@ -133,7 +133,7 @@ function sizes() {
 	const dist = path.join(root, 'dist');
 	const result = {};
 	for (const f of fs.readdirSync(dist).sort()) {
-		if (!f.endsWith('.js')) continue;
+		if (!f.endsWith('.js') && !f.endsWith('.mjs')) continue;
 		const buf = fs.readFileSync(path.join(dist, f));
 		result[f] = {bytes: buf.length, gzip: zlib.gzipSync(buf, {level: 9}).length};
 	}

test/parity/smoke.js

@@ -10,11 +10,16 @@
 	           with the right script path.
 	3 cli      node bin/alasql-cli.js "SELECT VALUE 42" prints 42.
 	4 esm      import() of dist/alasql.fs.js gives the function as default export.
+	4b esm node file     import() of dist/alasql.mjs (alasql/esm under Node) gives the function
+	           and findAlaSQLPath returns a path (__dirname is made from import.meta.url).
+	4c esm browser file  dist/alasql.browser.mjs (alasql/esm elsewhere) runs a query in a child
+	           node, and no line of it imports module, fs, path or url.
+	4d esm browser bundle  esbuild bundles dist/alasql.browser.mjs for the browser without an error.
 	5 min      dist/alasql.min.js loads under node (the exports branch) and runs a query.
 	6 node     --node: dist/alasql.fs.js loads and runs on each given node binary (Node 16 is
 	           the floor to check).
-	7 pack     --pack: npm pack, install in a temporary folder, then require, import and the CLI
-	           from there. Slow and needs the network.
+	7 pack     --pack: npm pack, install in a temporary folder, then require, import, import of
+	           alasql/esm and the CLI from there. Slow and needs the network.
 */
 
 const fs = require('fs');
@@ -172,6 +177,49 @@ async function main() {
 		return 'default export is the function';
 	});
 
+	// alasql/esm: the Node file in this process, the browser file in a child so that its "browser
+	// version" warning stays out of the results, and the browser file through a bundler
+	await checkAsync('esm node file', async () => {
+		const m = await import(pathToFileURL(dist('alasql.mjs')).href);
+		if (typeof m.default !== 'function') throw new Error('default export is ' + typeof m.default);
+		query(m.default);
+		if (typeof m.default.utils.findAlaSQLPath() !== 'string') throw new Error('no path');
+		return 'default export is the function; findAlaSQLPath works';
+	});
+	check('esm browser file', () => {
+		const out = execFileSync(
+			process.execPath,
+			[
+				'--input-type=module',
+				'-e',
+				`import alasql from ${JSON.stringify(pathToFileURL(dist('alasql.browser.mjs')).href)};` +
+					"console.log(JSON.stringify(alasql('SELECT 1+1 AS v')))",
+			],
+			{encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore']}
+		).trim();
+		// the file prints its "browser version" warning first, under node
+		const last = out.split('\n').pop();
+		if (last !== '[{"v":2}]') throw new Error('import gave ' + last);
+		const text = fs.readFileSync(dist('alasql.browser.mjs'), 'utf8');
+		const nodeImport = text.match(/^import .* from ['"](module|fs|path|url)['"]/m);
+		if (nodeImport) throw new Error('has a Node import: ' + nodeImport[0]);
+		return 'select 1+1 under node; no Node import in the file';
+	});
+	await checkAsync('esm browser bundle', async () => {
+		const esbuild = require('esbuild');
+		const r = await esbuild.build({
+			entryPoints: [dist('alasql.browser.mjs')],
+			bundle: true,
+			platform: 'browser',
+			format: 'esm',
+			write: false,
+			logLevel: 'silent',
+		});
+		if (r.errors.length) throw new Error(r.errors[0].text);
+		const warnings = [...new Set(r.warnings.map(w => w.text.replace(/"[^"]*"/g, '"..."')))];
+		return `esbuild, platform browser: ${r.outputFiles[0].text.length.toLocaleString('en-US')} bytes; ${r.warnings.length} warnings${warnings.length ? ': ' + warnings.join('; ') : ''}`;
+	});
+
 	if (args.includes('--pack')) {
 		const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'alasql-parity-'));
 		check('npm pack + install', () => {
@@ -204,6 +252,16 @@ async function main() {
 				{cwd: tmp, encoding: 'utf8'}
 			).trim();
 			if (esm !== '[{"v":2}]') throw new Error('import gave ' + esm);
+			const sub = execFileSync(
+				'node',
+				[
+					'--input-type=module',
+					'-e',
+					"import alasql from 'alasql/esm'; console.log(JSON.stringify(alasql('SELECT 1+1 AS v')))",
+				],
+				{cwd: tmp, encoding: 'utf8'}
+			).trim();
+			if (sub !== '[{"v":2}]') throw new Error('import of alasql/esm gave ' + sub);
 			const cli = execFileSync(
 				path.join(tmp, 'node_modules', '.bin', 'alasql'),
 				['SELECT VALUE 42'],
@@ -214,7 +272,7 @@ async function main() {
 			).trim();
 			if (cli !== '42') throw new Error('cli printed ' + cli);
 			const size = fs.statSync(path.join(tmp, tgz)).size;
-			return `${tgz} ${size.toLocaleString('en-US')} bytes; require, import and cli ok`;
+			return `${tgz} ${size.toLocaleString('en-US')} bytes; require, import, alasql/esm and cli ok`;
 		});
 		fs.rmSync(tmp, {recursive: true, force: true});
 	}