From 2267f905524a4da879f4fcd09eb9eaf6dd3394c4 Mon Sep 17 00:00:00 2001 From: Kuba Sunderland-Ober Date: Thu, 1 Oct 2026 19:39:00 +0200 Subject: [PATCH 01/35] tbrun: exit 5 for a probe that never returned; --llvm, --compiler-options, --exe --- WIP.Harness.md | 38 ++++++- WIP.md | 3 +- docs/Documentation/Extending.md | 2 +- docs/Documentation/Tools.md | 40 ++++++- scripts/check_twin_parsers.mjs | 82 ++++++++++++++ scripts/lib/cli-cases.mjs | 10 ++ scripts/lib/tb-ide.mjs | 70 ++++++++---- scripts/lib/tb-launch.ps1 | 24 +++- scripts/lib/tb-probe.mjs | 138 +++++++++++++++++++++++ scripts/lib/tb-project.mjs | 4 +- scripts/tbrun.mjs | 189 +++++++++++++++++++++++++++++--- 11 files changed, 555 insertions(+), 45 deletions(-) create mode 100644 scripts/lib/tb-probe.mjs diff --git a/WIP.Harness.md b/WIP.Harness.md index e44c3b9c..95287210 100644 --- a/WIP.Harness.md +++ b/WIP.Harness.md @@ -548,6 +548,14 @@ Four smaller things it knows, each of which cost a run: remedies (rerun the second, isolate the probe for the first), and `check_examples` already isolated a sample on `tbbuild`'s 4. `tbrun` exits 3 for no output at all, and 2 for a compile that never settled, which `tbbuild` reports as 3. +- **`tbrun` exits 5 when the probe ended before it returned.** The quiet period cannot see + it: on BETA 995, `Err.Raise` with no handler in a `+llvm` procedure ends the run with + nothing in the console, and `tbrun` exited 0 with the output up to there. `End` does the + same, and so does an unhandled error in plain code, though that run took 30 s against + the others' 20 s, for a reason not looked into. `lib/tb-probe.mjs`'s `wrapProbe` moves the + attribute, blanked to spaces so diagnostics keep their positions, to a Sub appended to the + same module, so a Private probe Sub can still be called; that Sub prints a sentinel after + the call. `check_twin_parsers` has its fixtures. A reader of the console that is not `tbrun` should **compare the whole console before and after, not read on from an index**: new text can be appended to an entry that is still open. @@ -570,8 +578,8 @@ IDE escapes that continued text twice** ([BUGS-TO-REPORT.md](BUGS-TO-REPORT.md)) printing `&`, `<` or `>` after a `Debug.Print ...;` reads them back as `&`, `<` and `>`, which is also what the console shows. -It settles on a quiet period rather than a sentinel, so no probe has to print a marker the -script knows about. Distinct `--port` values let probes run concurrently, exactly as +It settles on the wrapper's sentinel, or else on a quiet period, so no probe has to print a +marker of its own. Distinct `--port` values let probes run concurrently, exactly as `tbbuild`'s do. **Two things make that safe, and both had to be built.** The workspace and `project.id` @@ -589,6 +597,32 @@ on an image allowlist, *and* windowless --- a new one that has a window is repor alone, since that cannot be told from a copy the user opened. `--no-reap` turns it off, and concurrent runs driving the same server should use it and sweep once at the end. +### Measuring LLVM + +`tbrun --llvm` sets `compiler.debugOptions` and `compiler.buildOptions` to `+llvm` in the +staged Settings; `--compiler-options` sets any other string. Measured on BETA 995 with no +per-procedure attributes, one probe in four variants of Settings: **`debugOptions` is what +the `[RunAfterBuild]` run is compiled with.** With `+llvm` there, `Debug.Assert`'s condition +was evaluated 0 times and a 200-million-step loop took 516 ms; with it only in +`buildOptions`, 1 time and 672 ms, as with neither. (`+llvm` alone is not +`+llvm +optimize`, which took the same loop to 62 ms as a procedure attribute.) + +**The licence is in the status bar's `compilerLicence`:** one of `COMMUNITY EDITION`, +`PERSONAL EDITION`, `PROFESSIONAL EDITION`, `ULTIMATE EDITION`, and `tB Licence: ...` +until the IDE knows (`ide/main.js`, BETA 995). The licence key is in HKCU, so a lane IDE +sees the user's. `tbrun` refuses an LLVM run on the first two, which compile no user code +with LLVM; it cannot be tested with a real Community licence here, only through a fault. + +**`--exe` runs the exe on a private desktop**, through `launchOnDesktop`, the part of +`launchIde` that calls `tb-launch.ps1`, which reports the program's exit code on a +second line (`exit `) once it ends. Nothing reaches the exe's standard output: the +launcher creates it with no inherited handles. So `TbRun.Out` writes UTF-8 to the file +`TBRUN_OUT` names, and to standard output only when there is none. Measured on BETA 995: +`App.IsInIDE` is True in a `[RunAfterBuild]` run and False in the exe, the exit code from +`ExitProcess 7` comes back as 7, a hung exe is ended at `--timeout` with nothing left +running, and **the exe never evaluates `Debug.Assert`, with or without LLVM**, as VB6 drops +`Debug` statements from a compiled program. + ### Building for win64 **`tbrun` and `tbbuild` take `--arch win32|win64`, and set it on every run, win32 diff --git a/WIP.md b/WIP.md index 9b896209..4013c457 100644 --- a/WIP.md +++ b/WIP.md @@ -140,7 +140,8 @@ node scripts/tbrun.mjs # what does it print - **It runs the IDE on a private Windows desktop**, so it cannot seize focus mid-sentence. Set `TBBUILD_SHOW=1` while working interactively and leave it unset for unattended runs --- a wedged IDE nobody can see is the failure that costs an afternoon. - **One project per IDE**, 8--11 seconds each and flat in project size. Reusing a live IDE for a second project wedges it, so the cold start is the unit of work, not overhead to optimise away. Concurrency is how to go faster: distinct `--port` values give distinct DevTools ports, user-data folders and desktops. - **Keep a probe that might crash the compiler in a project of its own.** twinBASIC runs the compiler in the same process as user code, so one bad probe can take the run down and cost the other thirty their answer. -- **`tbrun` takes an exported tree, not a `.twinproj`**, because it has to pin `project.buildPath` in its own staged copy --- a project still on the default template opens a native Save dialog that is invisible on the private desktop, and the build simply never happens while every health check says the IDE is fine. The probe is a module with a `[RunAfterBuild]` Sub, and must start with `Debug.Cls`. Its exit codes: 0 the probe ran and its output was captured, 1 the project has compile errors, 2 the harness failed or the build did after a clean compile, 3 no output, 4 the compiler crashed, as `tbbuild` reports it. +- **`tbrun` takes an exported tree, not a `.twinproj`**, because it has to pin `project.buildPath` in its own staged copy --- a project still on the default template opens a native Save dialog that is invisible on the private desktop, and the build simply never happens while every health check says the IDE is fine. The probe is a module with a `[RunAfterBuild]` Sub, and must start with `Debug.Cls`. Its exit codes: 0 the probe ran and its output was captured, 1 the project has compile errors, 2 the harness failed or the build did after a clean compile, 3 no output, 4 the compiler crashed, as `tbbuild` reports it, 5 the probe ended before it returned (`End`, or an error raised with no handler in LLVM-compiled code, which ends the run silently). +- **Measuring LLVM: `tbrun --llvm`** (or `--compiler-options ""`) compiles the whole probe with LLVM, by setting `compiler.debugOptions` --- what a `[RunAfterBuild]` run is compiled with; `compiler.buildOptions` alone does not reach it (BETA 995) --- and `compiler.buildOptions`, for the exe. It refuses a Community or Personal licence. `--exe` also runs the built exe on a private desktop; the exe runs `Sub Main` and prints with `TbRun.Out`, a module `tbrun` adds, because `Debug.Print` writes nothing in an exe. - **A census is evidence, not applicability.** The corpus not using an attribute somewhere does not mean the compiler refuses it there, and the reverse also holds. Only a probe settles that. - **The sweep is the probe for a whole attribute.** Before writing or changing an attribute's `Applicable to:` line, run `sweep_attributes.mjs --names ` (a minute or a few; the once-per-project ones, `RunAfterBuild` and `RunBeforeStartupObject`, take several); it tries every declaration site, not only the ones already claimed. **Always pass `--out` and `--dump-results`**: a run piped through `tail` keeps one line of a report that took ten minutes. A clean build there means the compiler accepts the attribute, not that it does anything, and an Enum member cannot be tested at all (see [WIP.Harness.md](WIP.Harness.md#sweeping-every-attribute-at-every-site)). - **End an IDE by its pid, never by image name.** `taskkill /IM twinBASIC.exe` ends every other run's IDE, another session's included, and the user's own. `tbbuild --keep` prints the pid for this reason. diff --git a/docs/Documentation/Extending.md b/docs/Documentation/Extending.md index 253d3d66..52c70d2e 100644 --- a/docs/Documentation/Extending.md +++ b/docs/Documentation/Extending.md @@ -669,7 +669,7 @@ The split exists so that an edit confined to `docs/` usually has to pay for `che Separating 1 from 2 is what stops a broken gate reading as a clean site, and it has to hold at the top level too. Node's own exit for an uncaught exception is 1, which reads as a finding. So a new tool calls `exitOnCrash()` from `lib/cli.mjs` before it does anything else; it makes an uncaught exception, or a rejection nothing awaits, print the error and exit 2, and it also catches a rejected top-level await. It installs when called, never on import, so a module that can also be imported calls it only when it is the entry point. `exitOnCrash(cleanup)` runs `cleanup(err)` first for a tool that holds something the exit would lose. A script with a `main()` can instead end with `main().catch((err) => { console.error(err); process.exit(2); })`, as `check_a11y.mjs` does; either way a crash must not leave Node's own exit 1. -The rule is not limited to gates. Every tool under `builder/`, `scripts/`, `book/`, `eval/` and `wisdom/` reports a finding as 1 where it has findings, and everything that stops it doing its job as 2. A tool with more outcomes to tell apart adds codes above 2 --- `tbbuild` and `tbrun` use 3 and 4, `addin_test` and `wisdom` use 3 --- and `impexp` keeps a table of its own, shared with its Python edition. **Each tool ends its `--help` text with an `Exit codes:` block, one line per code, and that block is the source of truth**: [Tools and Scripts](Tools) states the same codes once in each tool's entry, and a change to one goes into the other. +The rule is not limited to gates. Every tool under `builder/`, `scripts/`, `book/`, `eval/` and `wisdom/` reports a finding as 1 where it has findings, and everything that stops it doing its job as 2. A tool with more outcomes to tell apart adds codes above 2 --- `tbbuild` uses 3 and 4, `tbrun` 3 to 5, `addin_test` and `wisdom` use 3 --- and `impexp` keeps a table of its own, shared with its Python edition. **Each tool ends its `--help` text with an `Exit codes:` block, one line per code, and that block is the source of truth**: [Tools and Scripts](Tools) states the same codes once in each tool's entry, and a change to one goes into the other. **Say what a pass covered.** Nearly every gate in both wrappers does: `check_dot_fit.mjs` gives the diagram count, `pick_a11y_sample.mjs --check` the sample size and the number of construct families in use, `check_publish_policy.mjs` the probe counts on both sides, `check_code_regions.mjs` the number of files swept, the number whose code regions moved and the number of fences found, `check_a11y.mjs` the page × theme × viewport product it audited. A gate silent on success says nothing about whether it examined anything, which is the state a gate that has quietly stopped working also reports. diff --git a/docs/Documentation/Tools.md b/docs/Documentation/Tools.md index f5a20789..3aa5000e 100644 --- a/docs/Documentation/Tools.md +++ b/docs/Documentation/Tools.md @@ -845,6 +845,7 @@ Exit codes: **0** the project compiled without errors; **1** the project has err node scripts/tbrun.mjs [--port N] [--arch win32|win64] [--timeout S] [--quiet MS] [--json] [--raw] [--keep] [--no-reap] [--reap-images a,b] [--show|--hide] + [--llvm | --compiler-options S] [--exe] Builds a probe project and captures what it writes to the IDE's [Debug Console](../../tB/IDE/Project/DebugConsole). Where [`tbbuild.mjs`](#tbbuild) answers @@ -886,6 +887,18 @@ code generation is a failed run as well. Its error line is written before the pr statement, so the probe's `Debug.Cls` erases it, and the probe stops at the call. `tbrun` keeps what each clear erases, so it names that line and prints the output up to the call. +**A probe that ends before it returns is a failed run, exit 5**, with what it printed +printed all the same. `End` ends a probe that way, and so does an error raised with no +handler in a procedure compiled with LLVM, which ends the run without a report. In the +staged copy, `tbrun` moves the `[RunAfterBuild]` attribute to a Sub it adds to the same +module, which calls the probe's Sub and then prints a line of its own. That line is missing +when the probe did not return, and it is never printed. The attribute is replaced with +spaces, so the line and column numbers in a diagnostic are still the ones in your file. +`tbrun` warns when it cannot add the wrapper --- the Sub is in a class, takes parameters, or +is one of several marked --- and the check is then off. A probe that stays silent for longer +than `--quiet` while it works also ends the wait without that line, so raise `--quiet` for a +slow one. + **The capture is complete however much a probe prints**, so there is no reason to keep one short. `tbrun` reads the console's backing array rather than the pane, which is a virtualised list view holding only the rows that fit --- reading that instead returns the last ten or so @@ -905,14 +918,35 @@ they are 4, **vbArchWin32** and `x86`. console shows. The IDE does this, not the probe; `Debug.Print "A"; "&"`, in one statement, comes back as `A&`. +**`--llvm` compiles the whole probe with [LLVM](../../LLVM/)**, with no +`[CompilerOptions("+llvm")]` on each procedure. It sets the project's compiler options in the +staged copy: `compiler.debugOptions`, which the `[RunAfterBuild]` run is compiled with, and +`compiler.buildOptions`, which the exe is. `--compiler-options` sets both to any other +string, such as `"+llvm +optimize"`. A run that uses LLVM --- through either option, the +tree's own settings or a procedure's `[CompilerOptions]` --- is refused when the IDE shows a +Community or Personal licence. Neither of those compiles your code with LLVM, so the run would +measure the default compiler. + +**`--exe` also runs the exe the build wrote**, after the probe has run in the IDE. It +starts the exe on a private desktop, as it starts the IDE, so a message box the exe opens +appears on no desktop you use. It also ends the exe at `--timeout`. The exe runs its +`Sub Main`, not the `[RunAfterBuild]` Sub, so a probe for both gives the tree a `Main` that +calls the probe, and leaves out the template's own module with an empty `Main`. A built exe +writes nothing with `Debug.Print`, so the probe prints with `TbRun.Out`, from a module `tbrun` +adds to the staged copy. `TbRun.Out` writes to the Debug Console in the IDE, and to a file +`tbrun` reads in the exe. The exe's lines and its exit code follow the probe's output. + | Flag | Effect | |---|---| | `--port ` | DevTools port for the IDE. Default 9346. Distinct ports let probes run concurrently --- the staging directory and the project id are keyed to it, so two runs never share a workspace. A port another IDE holds is refused, as for `tbbuild`. | | `--arch ` | The target to build for, `win32` or `win64`. Default `win32`, set on every run, as for `tbbuild`. A `win64` probe runs as a 64-bit process. | | `--timeout ` | Give up waiting for console output. Default 120. | -| `--quiet ` | How long the console must stop changing before the output counts as complete. Default 2500. There is no sentinel string to match, so any probe works without telling the script anything. Raise it well above the default for a probe that drives an out-of-process server, which can take longer than that to start. | +| `--quiet ` | How long the console must stop changing before the output counts as complete, when the probe has not returned. Default 2500. Raise it well above the default for a probe that drives an out-of-process server, which can take longer than that to start. | +| `--llvm` | Compile the whole probe, and the exe, with LLVM. The same as `--compiler-options +llvm`. | +| `--compiler-options ` | The project's compiler options, for the run and the exe. | +| `--exe` | Also run the built exe, and print what it writes with `TbRun.Out` and its exit code. | | `--raw` | Keep the console's timestamp column, which is otherwise stripped. | -| `--json` | One object with the path of the built file, the target, the captured lines, the IDE pid and anything reaped. | +| `--json` | One object with the path of the built file, the target, the captured lines, whether the probe returned, the licence an LLVM run checked, the exe's run, the IDE pid and anything reaped. | | `--keep` | Leave the IDE running. Implies `--no-reap`, and leaves the IDE's registry entries for the probe as they are. | | `--no-reap` | Do not harvest automation servers the probe left behind. | | `--reap-images ` | Replace the harvested image list. Default is the Office suite. | @@ -947,7 +981,7 @@ behind. That includes the target the IDE remembers for each project, which a `wi writes. **A probe builds for the target `--arch` names**, whatever the IDE remembers, so a kept IDE switched to `win64` does not make later runs on the same port build 64-bit. -Exit codes: **0** the probe ran and its output was captured; **1** the project has compile errors (the diagnostics are printed); **2** a refused command line (a source folder that is missing or has no `Settings` file included), no IDE or compiler, an IDE that did not start, a compile that never settled, a build that failed after a clean compile, a probe that never ran or stopped at a procedure that failed code generation, or a crash; **3** no output: the console held none before the timeout, or the probe printed none after its last `Debug.Cls`; **4** the compiler crashed, or restarted twice, while compiling the project. +Exit codes: **0** the probe ran and its output was captured; **1** the project has compile errors (the diagnostics are printed); **2** a refused command line (a source folder that is missing or has no `Settings` file included), no IDE or compiler, an IDE that did not start, a compile that never settled, a build that failed after a clean compile, a probe that never ran or stopped at a procedure that failed code generation, an LLVM run on a Community or Personal licence, an `--exe` run with no exe built, or a crash; **3** no output: the console held none before the timeout, or the probe printed none after its last `Debug.Cls`; **4** the compiler crashed, or restarted twice, while compiling the project; **5** the probe ended before it returned, its output printed all the same. The exe's exit code under `--exe` is reported, not passed on. ### addin_test.mjs {: #addin-test } diff --git a/scripts/check_twin_parsers.mjs b/scripts/check_twin_parsers.mjs index 87c96ade..6a04551d 100644 --- a/scripts/check_twin_parsers.mjs +++ b/scripts/check_twin_parsers.mjs @@ -21,11 +21,15 @@ // under. // - parseTargets (scripts/lib/attributes-doc.mjs), which turns an // `Applicable to:` line into gen_attribute_probes.mjs's targets. +// - wrapProbe (scripts/lib/tb-probe.mjs), which moves a tbrun probe's +// [RunAfterBuild] to a wrapper; a Sub it wraps wrongly runs the wrong code, +// and one it misses loses tbrun's check that the probe returned. import { exitOnCrash, parseCli, printHelpAndExit, withUsageError } from "../lib/cli.mjs"; import { parseTargets } from "./lib/attributes-doc.mjs"; import { createProbes } from "./lib/gate-probes.mjs"; import { classify } from "./lib/tb-fences.mjs"; +import { SENTINEL, TBRUN_FILE, WRAPPER_SUB, wrapProbe } from "./lib/tb-probe.mjs"; import { parseTwin } from "./lib/twin-api.mjs"; import { MODIFIERS, declarationKind } from "./lib/twin-declarations.mjs"; @@ -129,4 +133,82 @@ for (const [app, want] of [ check(`parseTargets: ${app}`, show(got) === show(want), `got ${show(got)}`); } +// ---------------------------------------------------------------- wrapProbe + +// Each fixture is one file, Probe.twin; `want` is the Sub wrapped, or null for +// none. A wrapped file must keep every line where it was, and add exactly one +// [RunAfterBuild] -- the wrapper's -- inside the probe's module. +const probe = (body) => `Module Probe\n${body}\nEnd Module\n`; +for (const [what, text, want] of [ + ["a Public Sub", probe(" [RunAfterBuild]\n Public Sub Run()\n Debug.Cls\n End Sub"), "Run"], + [ + "a Private Sub, CRLF", + probe(" [RunAfterBuild]\n Private Sub Go()\n End Sub").replaceAll("\n", "\r\n"), + "Go", + ], + ["the Sub on the attribute's line", probe(" [RunAfterBuild] Sub Go\n End Sub"), "Go"], + [ + "a comment and another attribute between", + probe(' [RunAfterBuild]\n \' why\n [Description("x")]\n Sub Go()\n End Sub'), + "Go", + ], + ["a commented-out attribute", probe(" ' [RunAfterBuild]\n Sub Go()\n End Sub"), null], + ["a Sub with parameters", probe(" [RunAfterBuild]\n Sub Go(ByVal n As Long)\n End Sub"), null], + ["a Function", probe(" [RunAfterBuild]\n Function Go() As Long\n End Function"), null], + ["in a Class", "Class Probe\n [RunAfterBuild]\n Sub Go()\n End Sub\nEnd Class\n", null], + // The End Module that follows is another module's, where Go cannot be called. + [ + "in a Class before a Module", + "Class Probe\n [RunAfterBuild]\n Sub Go()\n End Sub\nEnd Class\nModule M\nEnd Module\n", + null, + ], + // The container is the last one before the attribute, not the last in the file. + [ + "in a Module before a Class", + `${probe(" [RunAfterBuild]\n Sub Go()\n End Sub")}Class C\nEnd Class\n`, + "Go", + ], + [ + "two of them", + probe(" [RunAfterBuild]\n Sub A()\n End Sub\n [RunAfterBuild]\n Sub B()\n End Sub"), + null, + ], + [ + "the second of two modules", + `${probe(" Sub A()\n End Sub")}Module Second\n [RunAfterBuild]\n Sub B()\n End Sub\nEnd Module\n`, + "B", + ], +]) { + const r = wrapProbe([{ name: "Probe.twin", text }]); + const out = r.files.find((f) => f.name === "Probe.twin")?.text; + const tbrun = r.files.some((f) => f.name === TBRUN_FILE); + let ok = (r.wrapped?.sub ?? null) === want && tbrun; + if (ok && want) { + const eol = text.includes("\r\n") ? "\r\n" : "\n"; + const [before, after] = [text.split(eol), out.split(eol)]; + const attrs = after.filter((l) => /\[RunAfterBuild\]/.test(l)).length; + const call = after.findIndex((l) => l.trim() === `Public Sub ${WRAPPER_SUB}()`); + // The wrapper is six lines, from its attribute to a blank line; without + // them, the file is the probe's with its attribute blanked. + const rest = after.filter((_, i) => i < call - 1 || i >= call + 5); + const kept = + rest.length === before.length && + before.every((l, i) => rest[i] === l || rest[i] === l.replace("[RunAfterBuild]", " ".repeat(15))); + const endModule = after.findIndex((l, i) => i > call && /^End Module/.test(l)); + ok = + kept && + attrs === 1 && + after[call + 1].trim() === want && + after[call + 2].trim() === `Debug.Print "${SENTINEL}"` && + endModule > call && + !after.slice(0, call).some((l) => l.includes(WRAPPER_SUB)); + } + check(`wrapProbe: ${what}`, ok, show({ wrapped: r.wrapped, why: r.why, tbrun, out })); +} +check( + "wrapProbe: a tree with its own Module TbRun gets no second one", + !wrapProbe([{ name: "Mine.twin", text: "Module TbRun\nEnd Module\n" }]).files.some((f) => f.name === TBRUN_FILE), + "", +); + process.exit(report()); diff --git a/scripts/lib/cli-cases.mjs b/scripts/lib/cli-cases.mjs index b3ab1608..75e03570 100644 --- a/scripts/lib/cli-cases.mjs +++ b/scripts/lib/cli-cases.mjs @@ -510,6 +510,16 @@ for (const [tool, first] of [ bad(tool, [first, "--show", "--hide"], thenUsage("--show and --hide cannot be given together", tool)); } bad("scripts/tbrun.mjs", ["no-such-dir", "--quiet=-1"], thenUsage(NOT_WHOLE("--quiet", -1), "scripts/tbrun.mjs")); +bad( + "scripts/tbrun.mjs", + ["no-such-dir", "--llvm", "--compiler-options", "+llvm"], + thenUsage("--llvm and --compiler-options cannot be given together", "scripts/tbrun.mjs"), +); +bad( + "scripts/tbrun.mjs", + ["no-such-dir", "--compiler-options="], + thenUsage("--compiler-options needs a non-empty value", "scripts/tbrun.mjs"), +); // sweep_attributes reads its values before it looks for an IDE, and follows the // message with its usage. diff --git a/scripts/lib/tb-ide.mjs b/scripts/lib/tb-ide.mjs index e831ada2..5dfe6a84 100644 --- a/scripts/lib/tb-ide.mjs +++ b/scripts/lib/tb-ide.mjs @@ -180,6 +180,40 @@ export async function launchIde({ exe, project, port, show = false, keep = false // anything could attach to it, because the launcher died with tbbuild and // took the job with it. With the launcher detached from Node instead, // PowerShell exited at once, without a pid and without a word on stderr. + try { + const { pid, launcher } = await launchOnDesktop({ + exe: exeWin, + arg: target, + desktop: `tbbuild-${port}`, + job: !keep, + env: fullEnv, + }); + return { pid, launcher }; + } catch (e) { + throw new Error( + `could not start the IDE on a private desktop:\n${e.message}\n(--show runs it on your own desktop instead)`, + ); + } +} + +/** + * Start a program on a private desktop, inside a kill-on-close job, through + * lib/tb-launch.ps1. launchIde starts the IDE this way, and tbrun's --exe the + * probe's exe, so that neither can show a window on the user's desktop. + * + * @param {object} o + * @param {string} o.exe the program, a Windows path + * @param {string} [o.arg] its one argument; none when empty + * @param {string} o.desktop the private desktop's name + * @param {boolean} [o.job] false for no job, for a program that is to outlive + * this Node process + * @param {object} o.env its whole environment + * @returns {Promise<{pid: number, launcher: import("node:child_process").ChildProcess, exited: Promise}>} + * `exited` settles with the program's exit code when it ends, or null when + * the launcher ended without one -- killed, or failed. Throws, with the + * launcher's error, when the program did not start. + */ +export async function launchOnDesktop({ exe, arg = "", desktop, job = true, env }) { const script = readFileSync(path.join(path.dirname(fileURLToPath(import.meta.url)), "tb-launch.ps1"), "utf8"); const ps = spawn( "powershell", @@ -187,34 +221,32 @@ export async function launchIde({ exe, project, port, show = false, keep = false { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, - env: { - ...fullEnv, - TBBUILD_EXE: exeWin, - TBBUILD_ARG: target, - TBBUILD_DESKTOP: `tbbuild-${port}`, - TBBUILD_JOB: keep ? "0" : "1", - }, + env: { ...env, TBBUILD_EXE: exe, TBBUILD_ARG: arg, TBBUILD_DESKTOP: desktop, TBBUILD_JOB: job ? "1" : "0" }, }, ); - let err = ""; + let err = "", + out = ""; ps.stderr.on("data", (d) => { err += d; }); + ps.stdout.on("data", (d) => { + out += d; + }); + // "close" rather than "exit": the exit line can still be in the pipe at "exit". + const ended = new Promise((res) => ps.on("close", res)); const pid = await new Promise((res) => { - let buf = ""; - ps.stdout.on("data", (d) => { - buf += d; - const m = /^\s*(\d+)\s*$/m.exec(buf); + ps.stdout.on("data", () => { + const m = /^\s*(\d+)\s*$/m.exec(out); if (m) res(Number(m[1])); }); - ps.on("exit", () => res(null)); + ended.then(() => res(null)); }); - if (!pid) { - throw new Error( - `could not start the IDE on a private desktop:\n${err.trim()}\n` + "(--show runs it on your own desktop instead)", - ); - } - return { pid, launcher: ps }; + if (!pid) throw new Error(err.trim()); + const exited = ended.then(() => { + const m = /^exit (-?\d+)\s*$/m.exec(out); + return m ? Number(m[1]) : null; + }); + return { pid, launcher: ps, exited }; } /** diff --git a/scripts/lib/tb-launch.ps1 b/scripts/lib/tb-launch.ps1 index c6a794a9..2e636f74 100644 --- a/scripts/lib/tb-launch.ps1 +++ b/scripts/lib/tb-launch.ps1 @@ -7,7 +7,7 @@ # for the same reason: there is no argument quoting to get wrong. # # TBBUILD_EXE the executable -# TBBUILD_ARG its single argument +# TBBUILD_ARG its single argument, or empty for none # TBBUILD_DESKTOP desktop name to create # TBBUILD_JOB "0" for no job -- a --keep IDE, which must outlive the run # @@ -120,6 +120,12 @@ public static class TbLaunch { [DllImport("kernel32.dll", SetLastError = true)] static extern bool TerminateProcess(IntPtr process, uint exitCode); + [DllImport("kernel32.dll", SetLastError = true)] + static extern uint WaitForSingleObject(IntPtr handle, uint ms); + + [DllImport("kernel32.dll", SetLastError = true)] + static extern bool GetExitCodeProcess(IntPtr process, out int exitCode); + // The error of the call just made, with its text. Nothing may come between // that call and this one. static Exception Failed(string what) { @@ -171,6 +177,15 @@ public static class TbLaunch { public static void Resume(PROCESS_INFORMATION pi) { ResumeThread(pi.hThread); } + + // Waits for the process to end, through the handle CreateProcess returned, + // which stays valid however soon it ends; returns its exit code. + public static int Wait(PROCESS_INFORMATION pi) { + WaitForSingleObject(pi.hProcess, 0xFFFFFFFF); + int code; + if (!GetExitCodeProcess(pi.hProcess, out code)) throw Failed("GetExitCodeProcess"); + return code; + } } '@ @@ -188,7 +203,8 @@ if ($useJob) { $job = [TbLaunch]::KillOnCloseJob() } # appends a trailing space -- parseCommandLine() in ide/main2.js reads that as an # empty second file argument and refuses the launch with "Bad command line # syntax." -$cmd = '"' + $exe + '" "' + $arg + '"' +$cmd = '"' + $exe + '"' +if ($arg) { $cmd += ' "' + $arg + '"' } # CREATE_SUSPENDED (0x4): the IDE goes into the job before it runs a single # instruction, so there is no moment in which it could start a child outside. @@ -204,4 +220,6 @@ if ($useJob) { Write-Output $pi.dwProcessId -try { (Get-Process -Id $pi.dwProcessId).WaitForExit() } catch { } +# The process's exit code, on a line of its own once it ends: tbrun's --exe +# reads it for the probe's exe. +Write-Output ("exit " + [TbLaunch]::Wait($pi)) diff --git a/scripts/lib/tb-probe.mjs b/scripts/lib/tb-probe.mjs new file mode 100644 index 00000000..9681e571 --- /dev/null +++ b/scripts/lib/tb-probe.mjs @@ -0,0 +1,138 @@ +// What tbrun adds to a probe's sources before it packs them. +// +// A probe is a module with a [RunAfterBuild] Sub. Two things are added to the +// staged copy, never to the caller's tree: +// +// * A wrapper. The attribute moves from the probe's Sub to a Sub appended to +// the same module, which calls the probe's Sub and then prints SENTINEL. A +// probe that stops before it returns -- End, or an error that ends the run +// without a word, as an unhandled one raised in LLVM-compiled code does on +// BETA 995 -- leaves no SENTINEL, which is how tbrun tells it from one that +// finished. Appended to the same module, so a Private probe Sub can be +// called. The attribute is blanked to spaces rather than deleted, so every +// line and column a diagnostic names is still the caller's. +// * The TbRun module, whose Out writes a line where the run can read it: the +// DEBUG CONSOLE in the IDE, and standard output in a built exe, where +// Debug.Print writes nothing. + +import { MODIFIERS } from "./twin-declarations.mjs"; + +export const SENTINEL = "[tbrun] the probe returned"; +export const WRAPPER_SUB = "tbrun_RunProbe"; +export const TBRUN_FILE = "TbRun.twin"; + +const ATTRIBUTE_RE = /^([ \t]*)\[RunAfterBuild\]/gim; +const SUB_RE = new RegExp( + `^[ \\t]*(?:(?:${MODIFIERS})[ \\t]+)*Sub[ \\t]+(\\w+)[ \\t]*(?:\\([ \\t]*\\))?[ \\t]*(?:'.*)?$`, + "i", +); +const SKIPPED_RE = /^[ \t]*(?:'.*|\[[^\]]*\][ \t]*)?$/; +const CONTAINER_RE = new RegExp( + `^[ \\t]*(?:(?:${MODIFIERS})[ \\t]+)*(Module|Class|CoClass|Interface)[ \\t]+(\\w+)`, + "gim", +); +const END_MODULE_RE = /^[ \t]*End[ \t]+Module\b/im; +const TBRUN_MODULE_RE = /^[ \t]*(?:(?:Public|Private)[ \t]+)?Module[ \t]+TbRun\b/im; + +/** + * Wrap a probe's [RunAfterBuild] Sub, and add the TbRun module. + * + * @param {{name: string, text: string}[]} files the tree's Sources/*.twin + * @returns {{files: {name: string, text: string}[], wrapped: {file: string, module: string, sub: string} | null, why?: string}} + * the files to write (changed or new), what was wrapped, and why nothing was + */ +export function wrapProbe(files) { + const out = []; + if (!files.some((f) => TBRUN_MODULE_RE.test(f.text))) out.push({ name: TBRUN_FILE, text: TBRUN_MODULE }); + const none = (why) => ({ files: out, wrapped: null, why }); + + const hits = files.flatMap((f) => + [...f.text.matchAll(ATTRIBUTE_RE)].map((m) => ({ f, at: m.index + m[1].length, indent: m[1] })), + ); + if (hits.length !== 1) { + return none( + hits.length ? `${hits.length} [RunAfterBuild] attributes` : "no [RunAfterBuild] attribute at the start of a line", + ); + } + const { f, at, indent } = hits[0]; + const end = at + "[RunAfterBuild]".length; + + // The Sub it marks: the rest of the attribute's line, or else the first line + // after it that is not blank, a comment or another attribute. + const lines = f.text.slice(end).split("\n"); + let sub = null; + for (const [i, raw] of lines.entries()) { + const line = raw.replace(/\r$/, ""); + if (SKIPPED_RE.test(line)) continue; + sub = SUB_RE.exec(line)?.[1] ?? null; + if (!sub) return none(`the line after [RunAfterBuild] is not a Sub without parameters: ${line.trim()}`); + lines.length = i + 1; + break; + } + if (!sub) return none("no Sub after [RunAfterBuild]"); + + const containers = [...f.text.slice(0, at).matchAll(CONTAINER_RE)]; + const container = containers.at(-1); + if (!container || container[1].toLowerCase() !== "module") { + return none( + `the [RunAfterBuild] Sub is not in a Module${container ? ` but in ${container[1]} ${container[2]}` : ""}`, + ); + } + const afterSub = end + lines.join("\n").length; + const closing = END_MODULE_RE.exec(f.text.slice(afterSub)); + if (!closing) return none(`no End Module after ${sub}`); + const insertAt = afterSub + closing.index; + + const eol = f.text.includes("\r\n") ? "\r\n" : "\n"; + const wrapper = [ + `${indent}[RunAfterBuild]`, + `${indent}Public Sub ${WRAPPER_SUB}()`, + `${indent} ${sub}`, + `${indent} Debug.Print "${SENTINEL}"`, + `${indent}End Sub`, + "", + "", + ].join(eol); + const text = + f.text.slice(0, at) + " ".repeat(end - at) + f.text.slice(end, insertAt) + wrapper + f.text.slice(insertAt); + out.push({ name: f.name, text }); + return { files: out, wrapped: { file: f.name, module: container[2], sub } }; +} + +// Debug.Print writes nothing in a built exe (docs/Features/Project-Configuration/ +// Project-Types.md), so there Out writes UTF-8: to the file TBRUN_OUT names, or +// else to standard output. tbrun's --exe names a file, because it starts the +// exe on a private desktop with no handles inherited, so no pipe reaches it. +const TBRUN_MODULE = `' Added by tbrun to the staged copy of a probe. TbRun.Out writes a line to the +' DEBUG CONSOLE in the IDE; in a built exe, as UTF-8, to the file the TBRUN_OUT +' environment variable names, or else to standard output. +Module TbRun + + Private Declare PtrSafe Function GetStdHandle Lib "kernel32" (ByVal nStdHandle As Long) As LongPtr + Private Declare PtrSafe Function WriteFile Lib "kernel32" (ByVal hFile As LongPtr, ByVal lpBuffer As LongPtr, ByVal nNumberOfBytesToWrite As Long, ByRef lpNumberOfBytesWritten As Long, ByVal lpOverlapped As LongPtr) As Long + Private Declare PtrSafe Function WideCharToMultiByte Lib "kernel32" (ByVal CodePage As Long, ByVal dwFlags As Long, ByVal lpWideCharStr As LongPtr, ByVal cchWideChar As Long, ByVal lpMultiByteStr As LongPtr, ByVal cbMultiByte As Long, ByVal lpDefaultChar As LongPtr, ByVal lpUsedDefaultChar As LongPtr) As Long + + Public Sub Out(ByVal Text As String) + If App.IsInIDE Then + Debug.Print Text + Exit Sub + End If + Text = Text & vbCrLf + Dim n As Long = WideCharToMultiByte(65001, 0, StrPtr(Text), Len(Text), 0, 0, 0, 0) + Dim bytes() As Byte + ReDim bytes(n - 1) + WideCharToMultiByte 65001, 0, StrPtr(Text), Len(Text), VarPtr(bytes(0)), n, 0, 0 + Dim file As String = Environ$("TBRUN_OUT") + If Len(file) = 0 Then + Dim written As Long + WriteFile GetStdHandle(-11), VarPtr(bytes(0)), n, written, 0 + Exit Sub + End If + Dim f As Integer = FreeFile + Open file For Binary Access Write As #f + Put #f, LOF(f) + 1, bytes + Close #f + End Sub + +End Module +`; diff --git a/scripts/lib/tb-project.mjs b/scripts/lib/tb-project.mjs index e9f193d8..8a5c6836 100644 --- a/scripts/lib/tb-project.mjs +++ b/scripts/lib/tb-project.mjs @@ -30,15 +30,17 @@ import { runCompiler } from "./tb-install.mjs"; * @param {string} o.compiler the compiler executable (tb-install's compilerExe) * @param {object | ((original: object) => object)} [o.settings] settings to * set in the copy, or a function from the tree's own settings to them + * @param {(stage: string) => void} [o.prepare] changes the copy before it is packed * @returns {{original: object, settings: object}} the tree's settings, and the copy's */ -export function stageProject({ src, stage, project, compiler, settings = {} }) { +export function stageProject({ src, stage, project, compiler, settings = {}, prepare }) { rmSync(stage, { recursive: true, force: true }); cpSync(src, stage, { recursive: true }); const file = path.join(stage, "Settings"); const original = JSON.parse(readFileSync(file, "utf8")); const staged = { ...original, ...(typeof settings === "function" ? settings(original) : settings) }; writeFileSync(file, JSON.stringify(staged, null, "\t"), "utf8"); + prepare?.(stage); // import's exit code does not say whether it worked -- 0 on the failures it // reports, 999 on a tree holding an embedded package -- so runCompiler reads diff --git a/scripts/tbrun.mjs b/scripts/tbrun.mjs index ef8edf92..fa3af974 100644 --- a/scripts/tbrun.mjs +++ b/scripts/tbrun.mjs @@ -16,6 +16,14 @@ // --reap-images comma-separated image names to harvest // (default: the Office suite -- see REAP_IMAGES) // --show / --hide as tbbuild's +// --llvm compile the whole project with LLVM: the same as +// --compiler-options +llvm +// --compiler-options the project's compiler options, for the run +// and for the exe (compiler.debugOptions and +// compiler.buildOptions); an option string with +llvm +// is refused on a Community or Personal licence +// --exe run the built exe as well, on a private desktop, and +// capture what it writes with TbRun.Out and its exit code // // Exit: 0 captured output, 1 the project has compile errors, 2 the harness // could not run (a refused command line included), a compile never settled, or @@ -24,7 +32,8 @@ // and a procedure the probe calls that fails it, since the probe stops at the // call -- 3 no output: the build produced none in the console before the // timeout, or the probe ran and printed none after its last Debug.Cls -- 4 the -// compiler crashed, or restarted twice, while compiling the project. +// compiler crashed, or restarted twice, while compiling the project -- 5 the +// probe ended before it returned, its output printed all the same. // // ---------------------------------------------------------------- why // @@ -80,9 +89,14 @@ // generation is reported before the probe's first statement runs. So the // script wraps the page's clearDebugConsole() before the build, keeps // what each clear erases, and looks there too (tb-ide's keepClears). -// 5. QUIET-PERIOD, NOT A MARKER. Waiting for a sentinel string means every -// probe has to print one and the script has to know it. Waiting for the -// console to stop changing works for any probe. +// 5. QUIET-PERIOD, AND A MARKER THE PROBE NEVER WRITES. The run is over when +// the console stops changing, which works for any probe. But a probe that +// stops early also stops changing the console: End does, and so does an +// error raised in LLVM-compiled code with no handler, which ends the run +// without a word (measured, BETA 995). So the staged copy calls the +// probe's Sub from a wrapper that prints a sentinel once it returns +// (lib/tb-probe.mjs). Its absence is exit 5; its arrival ends the wait +// without the quiet period. // 6. EVERY RUN OWNS ITS OWN WORKSPACE AND KILLS ONLY ITS OWN IDE. Both were // shared, and both broke concurrency in ways that looked like something // else. The staging directory was a fixed %TEMP%/tbrun/src, so a second @@ -100,7 +114,7 @@ // a blunt enough instrument to need the guard rails in reapOrphans(). import { execFileSync } from "node:child_process"; -import { existsSync, readFileSync, mkdirSync, statSync, readdirSync, rmSync } from "node:fs"; +import { existsSync, readFileSync, mkdirSync, statSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { @@ -123,6 +137,7 @@ import { compileOutcome, killTree, launchIde, + launchOnDesktop, setBuildTarget, shutdownIde, summaryLine, @@ -130,12 +145,13 @@ import { wantShow, } from "./lib/tb-ide.mjs"; import { keepClears, keptClears, readConsole } from "./lib/tb-ide-console.mjs"; +import { SENTINEL, wrapProbe } from "./lib/tb-probe.mjs"; import { laneProjectId, stageProject } from "./lib/tb-project.mjs"; import { finishTidy, startTidy } from "./lib/tb-registry.mjs"; exitOnCrash(); -const USAGE = `usage: node scripts/tbrun.mjs [--ide ] [--port N] [--arch win32|win64] [--timeout S] [--quiet MS] [--json] [--raw] [--keep] [--no-reap] [--reap-images a,b] [--show|--hide] [-h, --help] +const USAGE = `usage: node scripts/tbrun.mjs [--ide ] [--port N] [--arch win32|win64] [--timeout S] [--quiet MS] [--json] [--raw] [--keep] [--no-reap] [--reap-images a,b] [--show|--hide] [--llvm | --compiler-options S] [--exe] [-h, --help] Builds an exported twinBASIC source tree in the IDE, runs it, and prints what it writes to the DEBUG CONSOLE. @@ -153,6 +169,13 @@ writes to the DEBUG CONSOLE. --reap-images a,b comma-separated image names to harvest (default: the Office suite) --show, --hide as tbbuild's + --llvm compile the whole project with LLVM, as + --compiler-options +llvm + --compiler-options + the project's compiler options, for the run and the exe; + +llvm is refused on a Community or Personal licence + --exe also run the built exe on a private desktop, and print + what it writes with TbRun.Out and its exit code -h, --help print this text and exit Exit codes: @@ -161,10 +184,13 @@ Exit codes: 2 a refused command line (a source folder that is missing or has no Settings file included), no IDE or compiler, an IDE that did not start, a compile that never settled, a build that failed after a clean compile, a probe that never ran or - stopped at a procedure that failed code generation, or a crash + stopped at a procedure that failed code generation, an --llvm run on a + Community or Personal licence, an --exe run with no exe built, or a crash 3 no output: the console held none before the timeout, or the probe printed none after its last Debug.Cls - 4 the compiler crashed, or restarted twice, while compiling the project`; + 4 the compiler crashed, or restarted twice, while compiling the project + 5 the probe ended before it returned (End, or an error that ended the run); what + it printed is printed all the same`; const { values, positionals } = withUsageError( () => @@ -176,6 +202,9 @@ const { values, positionals } = withUsageError( quiet: { type: "string" }, ide: { type: "string" }, "reap-images": { type: "string" }, + "compiler-options": { type: "string" }, + llvm: { type: "boolean", default: false }, + exe: { type: "boolean", default: false }, json: { type: "boolean", default: false }, raw: { type: "boolean", default: false }, keep: { type: "boolean", default: false }, @@ -195,6 +224,7 @@ if (values.help) printHelpAndExit(USAGE); const { port, arch, timeoutMs, quietMs } = withUsageError( () => { refuseTogether(values, ["show", "hide"]); + refuseTogether(values, ["llvm", "compiler-options"]); return { port: numberOption(values.port ?? "9346", { option: "--port", integer: true, min: 1, max: 65535 }), arch: choiceOption(values.arch ?? TARGETS[0], { option: "--arch", choices: TARGETS }), @@ -288,21 +318,54 @@ if (!hasHook) { // ------------------------------------------------------------------- pack // A packing failure is the harness's, exit 2. +// The project's compiler options: compiler.debugOptions are what the +// [RunAfterBuild] run is compiled with, and compiler.buildOptions what the exe +// is -- measured on BETA 995, where +llvm in the build options alone left the +// run's Debug.Assert evaluated and its loop at the speed of the default. +const compilerOptions = values.llvm ? "+llvm" : values["compiler-options"]; let wasTemplate = false, - projectName = ""; + projectName = "", + usesLlvm = false, + wrap = null; try { const staged = stageProject({ src: srcDir, stage, project: projPath, compiler: COMPILER, - settings: { "project.buildPath": buildPath, "project.id": laneProjectId(0, port) }, + settings: { + "project.buildPath": buildPath, + "project.id": laneProjectId(0, port), + ...(compilerOptions === undefined + ? {} + : { "compiler.debugOptions": compilerOptions, "compiler.buildOptions": compilerOptions }), + }, + prepare: (dir) => { + const sources = path.join(dir, "Sources"); + const files = existsSync(sources) + ? readdirSync(sources) + .filter((f) => f.endsWith(".twin")) + .map((name) => ({ name, text: readFileSync(path.join(sources, name), "utf8") })) + : []; + wrap = wrapProbe(files); + for (const f of wrap.files) writeFileSync(path.join(sources, f.name), f.text, "utf8"); + }, }); wasTemplate = /\$\{/.test(staged.original["project.buildPath"] ?? ""); projectName = String(staged.settings["project.name"] ?? ""); + // Any LLVM in the run: the project's options, or a procedure's own. + usesLlvm = + /\+llvm\b/i.test( + `${staged.settings["compiler.debugOptions"] ?? ""} ${staged.settings["compiler.buildOptions"] ?? ""}`, + ) || /\[\s*CompilerOptions\s*\(\s*"[^"]*\+llvm/i.test(sourceText); } catch (e) { die(2, e.message); } +if (hasHook && !wrap.wrapped) { + console.error( + `warning: ${wrap.why} -- so a probe that ends before it returns cannot be told from one that finished.`, + ); +} // What the build wrote: the IDE expands the template, so the name is looked for // rather than assumed -- ${FileExtension} follows the build type -- and the @@ -386,6 +449,33 @@ if (outcome.counts[0] > 0) { failBuild(1, [...outcome.rows, summaryLine(outcome.counts)].join("\n")); } +// The licence, for a run with LLVM in it. A Community licence ignores the LLVM +// settings and a Personal one applies them only to the built-in packages +// (docs/LLVM/Getting-Started.md), so on either the run would measure the +// default compiler and say nothing. The status bar's compilerLicence holds one +// of four " EDITION" strings once the IDE knows (ide/main.js, BETA 995), +// and "tB Licence: ..." until then. +let licence = null; +if (usesLlvm) { + const until = Date.now() + 15 * 1000; + while (Date.now() < until) { + licence = await cdp + .evaluate("typeof compilerLicence === 'undefined' || !compilerLicence ? null : compilerLicence.innerText") + .catch(() => null); + if (/ EDITION$/.test(licence ?? "")) break; + licence = null; + await new Promise((r) => setTimeout(r, 250)); + } + if (!licence) failBuild(2, "tbrun: the IDE never showed its licence, so an LLVM run could not be checked for one."); + if (/^(?:COMMUNITY|PERSONAL) /.test(licence)) { + failBuild( + 2, + `tbrun: this IDE has a ${licence.toLowerCase()} licence, which does not compile user code with LLVM, ` + + "so the run would measure the default compiler. An LLVM run needs a Professional or Ultimate licence.", + ); + } +} + // --------------------------------------------- build the exe, read the console let captured = null, @@ -403,7 +493,7 @@ try { // (2) a real press/release pair; element.click() is ignored. await click(cdp, "buildIcon"); - // (5) settle on a quiet period rather than a sentinel. + // (5) settle on the sentinel, or else a quiet period. const started = Date.now(); let last = "", lastChange = Date.now(), @@ -421,7 +511,9 @@ try { if (now !== last) { last = now; lastChange = Date.now(); - if (strip(now).length) seen = true; + const lines = strip(now); + if (lines.length) seen = true; + if (wrap.wrapped && lines.at(-1) === SENTINEL) break; } else if (seen && Date.now() - lastChange > quietMs) break; } captured = strip(last); @@ -481,13 +573,21 @@ if (lost) { (shown.length ? shown.map((l) => ` ${l}`).join("\n") : " (nothing)"), ); } +// (5) The wrapper's sentinel says the probe returned. It is the wrapper's line, +// not the probe's, so it is never printed. +const returned = wrap.wrapped ? captured.at(-1) === SENTINEL : null; +if (returned) { + // Without --raw, shown is captured itself. + if (shown !== captured) shown.pop(); + captured.pop(); +} // A probe that ran leaves its Executing line among what its Debug.Cls erased, // so an empty console then means it printed nothing after its last clear, not // that it never ran. -if (!captured.length && started >= 0) { +if (!captured.length && started >= 0 && returned !== false) { die(3, `tbrun: the probe ran (${erased[started]}) but printed nothing after its last Debug.Cls.`); } -if (!captured.length) { +if (!captured.length && started < 0) { die( 3, "tbrun: the build produced no console output before the timeout.\n" + @@ -498,10 +598,30 @@ if (!captured.length) { ); } +const exeRun = values.exe ? await runExe() : null; + if (values.json) { - console.log(JSON.stringify({ exe: builtFile(), arch, lines: shown, idePid: ideRun?.pid ?? null, reaped }, null, 2)); + console.log( + JSON.stringify( + { exe: builtFile(), arch, lines: shown, returned, licence, exeRun, idePid: ideRun?.pid ?? null, reaped }, + null, + 2, + ), + ); } else { for (const l of shown) console.log(l); + if (exeRun) { + console.log(`--- exe: ${exeRun.timedOut ? "still running after --timeout, ended" : `exit ${exeRun.exitCode}`}`); + for (const l of exeRun.lines) console.log(l); + } +} +if (returned === false) { + die( + 5, + `tbrun: the probe ended before it returned${started >= 0 ? "" : ", and the run's start was not seen"}: ` + + "End, or an error that ended the run without a report -- as one raised with no handler in " + + "LLVM-compiled code does. Or it was still running, silent, after --quiet ms; raise --quiet for a slow probe.", + ); } // ------------------------------------------------------------------ helpers @@ -522,6 +642,45 @@ function strip(text, raw = null) { return (raw === null ? out : lines(raw)).slice(from, to); } +// --exe: the built exe, started as the IDE is, on a private desktop and inside +// a kill-on-close job, so a window it opens -- a MsgBox -- is on no desktop +// anyone uses, and nothing it starts outlives it. It inherits no handles, and +// Debug.Print writes nothing in an exe, so what it printed is what TbRun.Out +// appended to the file TBRUN_OUT names. +async function runExe() { + const file = builtFile(); + if (!file || !/\.exe$/i.test(file)) { + die(2, `tbrun: --exe, but the build wrote no exe to ${outDir}${file ? ` (it wrote ${path.basename(file)})` : ""}`); + } + const outFile = path.join(work, "exe-out.txt"); + rmSync(outFile, { force: true }); + let run; + try { + run = await launchOnDesktop({ + exe: file, + desktop: `tbrun-exe-${port}`, + env: { ...process.env, TBRUN_OUT: outFile }, + }); + } catch (e) { + die(2, `tbrun: could not start the exe on a private desktop: ${e.message}`); + } + let timer; + const timedOut = await Promise.race([ + run.exited.then(() => false), + new Promise((r) => { + timer = setTimeout(() => r(true), timeoutMs); + }), + ]); + clearTimeout(timer); + if (timedOut) { + killTree(run.pid); + run.launcher.kill(); + } + const exitCode = await run.exited; + const text = existsSync(outFile) ? readFileSync(outFile, "utf8") : ""; + return { file, exitCode: timedOut ? null : exitCode, timedOut, lines: strip(text) }; +} + // (6) End OUR IDE by pid, never by image name. The tree kill takes the probe exe // and anything it spawned with CreateProcess; what it cannot take is a COM // server, which is what reapOrphans is for. From 52f41811ebef87f0ea5fd59e6b1e32d77466d87e Mon Sep 17 00:00:00 2001 From: Kuba Sunderland-Ober Date: Thu, 1 Oct 2026 19:54:23 +0200 Subject: [PATCH 02/35] tbrun: exit 6 when the --exe run exits nonzero or outlives --timeout --- WIP.Harness.md | 8 ++++++-- WIP.md | 2 +- docs/Documentation/Extending.md | 2 +- docs/Documentation/Tools.md | 2 +- scripts/tbrun.mjs | 10 ++++++++-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/WIP.Harness.md b/WIP.Harness.md index 95287210..7942f66f 100644 --- a/WIP.Harness.md +++ b/WIP.Harness.md @@ -551,11 +551,15 @@ Four smaller things it knows, each of which cost a run: - **`tbrun` exits 5 when the probe ended before it returned.** The quiet period cannot see it: on BETA 995, `Err.Raise` with no handler in a `+llvm` procedure ends the run with nothing in the console, and `tbrun` exited 0 with the output up to there. `End` does the - same, and so does an unhandled error in plain code, though that run took 30 s against - the others' 20 s, for a reason not looked into. `lib/tb-probe.mjs`'s `wrapProbe` moves the + same, and so does an unhandled error in plain code, in the same time as a probe that + returns (20.7-21.9 s over three runs on BETA 995, against 18.6-20.4 s; one earlier 30 s + run came with a 25 s clean run beside it). `lib/tb-probe.mjs`'s `wrapProbe` moves the attribute, blanked to spaces so diagnostics keep their positions, to a Sub appended to the same module, so a Private probe Sub can still be called; that Sub prints a sentinel after the call. `check_twin_parsers` has its fixtures. +- **`tbrun --exe` exits 6 when the exe exited with a code other than 0**, or was still + running at `--timeout` and was ended. A run that would exit 5 exits 5 first, since the + IDE's run is the one `--exe` follows. A reader of the console that is not `tbrun` should **compare the whole console before and after, not read on from an index**: new text can be appended to an entry that is still open. diff --git a/WIP.md b/WIP.md index 4013c457..92f07a12 100644 --- a/WIP.md +++ b/WIP.md @@ -140,7 +140,7 @@ node scripts/tbrun.mjs # what does it print - **It runs the IDE on a private Windows desktop**, so it cannot seize focus mid-sentence. Set `TBBUILD_SHOW=1` while working interactively and leave it unset for unattended runs --- a wedged IDE nobody can see is the failure that costs an afternoon. - **One project per IDE**, 8--11 seconds each and flat in project size. Reusing a live IDE for a second project wedges it, so the cold start is the unit of work, not overhead to optimise away. Concurrency is how to go faster: distinct `--port` values give distinct DevTools ports, user-data folders and desktops. - **Keep a probe that might crash the compiler in a project of its own.** twinBASIC runs the compiler in the same process as user code, so one bad probe can take the run down and cost the other thirty their answer. -- **`tbrun` takes an exported tree, not a `.twinproj`**, because it has to pin `project.buildPath` in its own staged copy --- a project still on the default template opens a native Save dialog that is invisible on the private desktop, and the build simply never happens while every health check says the IDE is fine. The probe is a module with a `[RunAfterBuild]` Sub, and must start with `Debug.Cls`. Its exit codes: 0 the probe ran and its output was captured, 1 the project has compile errors, 2 the harness failed or the build did after a clean compile, 3 no output, 4 the compiler crashed, as `tbbuild` reports it, 5 the probe ended before it returned (`End`, or an error raised with no handler in LLVM-compiled code, which ends the run silently). +- **`tbrun` takes an exported tree, not a `.twinproj`**, because it has to pin `project.buildPath` in its own staged copy --- a project still on the default template opens a native Save dialog that is invisible on the private desktop, and the build simply never happens while every health check says the IDE is fine. The probe is a module with a `[RunAfterBuild]` Sub, and must start with `Debug.Cls`. Its exit codes: 0 the probe ran and its output was captured, 1 the project has compile errors, 2 the harness failed or the build did after a clean compile, 3 no output, 4 the compiler crashed, as `tbbuild` reports it, 5 the probe ended before it returned (`End`, or an error raised with no handler in LLVM-compiled code, which ends the run silently), 6 with `--exe`, the exe exited with a code other than 0 or was still running at `--timeout`. - **Measuring LLVM: `tbrun --llvm`** (or `--compiler-options ""`) compiles the whole probe with LLVM, by setting `compiler.debugOptions` --- what a `[RunAfterBuild]` run is compiled with; `compiler.buildOptions` alone does not reach it (BETA 995) --- and `compiler.buildOptions`, for the exe. It refuses a Community or Personal licence. `--exe` also runs the built exe on a private desktop; the exe runs `Sub Main` and prints with `TbRun.Out`, a module `tbrun` adds, because `Debug.Print` writes nothing in an exe. - **A census is evidence, not applicability.** The corpus not using an attribute somewhere does not mean the compiler refuses it there, and the reverse also holds. Only a probe settles that. - **The sweep is the probe for a whole attribute.** Before writing or changing an attribute's `Applicable to:` line, run `sweep_attributes.mjs --names ` (a minute or a few; the once-per-project ones, `RunAfterBuild` and `RunBeforeStartupObject`, take several); it tries every declaration site, not only the ones already claimed. **Always pass `--out` and `--dump-results`**: a run piped through `tail` keeps one line of a report that took ten minutes. A clean build there means the compiler accepts the attribute, not that it does anything, and an Enum member cannot be tested at all (see [WIP.Harness.md](WIP.Harness.md#sweeping-every-attribute-at-every-site)). diff --git a/docs/Documentation/Extending.md b/docs/Documentation/Extending.md index 52c70d2e..cc757073 100644 --- a/docs/Documentation/Extending.md +++ b/docs/Documentation/Extending.md @@ -669,7 +669,7 @@ The split exists so that an edit confined to `docs/` usually has to pay for `che Separating 1 from 2 is what stops a broken gate reading as a clean site, and it has to hold at the top level too. Node's own exit for an uncaught exception is 1, which reads as a finding. So a new tool calls `exitOnCrash()` from `lib/cli.mjs` before it does anything else; it makes an uncaught exception, or a rejection nothing awaits, print the error and exit 2, and it also catches a rejected top-level await. It installs when called, never on import, so a module that can also be imported calls it only when it is the entry point. `exitOnCrash(cleanup)` runs `cleanup(err)` first for a tool that holds something the exit would lose. A script with a `main()` can instead end with `main().catch((err) => { console.error(err); process.exit(2); })`, as `check_a11y.mjs` does; either way a crash must not leave Node's own exit 1. -The rule is not limited to gates. Every tool under `builder/`, `scripts/`, `book/`, `eval/` and `wisdom/` reports a finding as 1 where it has findings, and everything that stops it doing its job as 2. A tool with more outcomes to tell apart adds codes above 2 --- `tbbuild` uses 3 and 4, `tbrun` 3 to 5, `addin_test` and `wisdom` use 3 --- and `impexp` keeps a table of its own, shared with its Python edition. **Each tool ends its `--help` text with an `Exit codes:` block, one line per code, and that block is the source of truth**: [Tools and Scripts](Tools) states the same codes once in each tool's entry, and a change to one goes into the other. +The rule is not limited to gates. Every tool under `builder/`, `scripts/`, `book/`, `eval/` and `wisdom/` reports a finding as 1 where it has findings, and everything that stops it doing its job as 2. A tool with more outcomes to tell apart adds codes above 2 --- `tbbuild` uses 3 and 4, `tbrun` 3 to 6, `addin_test` and `wisdom` use 3 --- and `impexp` keeps a table of its own, shared with its Python edition. **Each tool ends its `--help` text with an `Exit codes:` block, one line per code, and that block is the source of truth**: [Tools and Scripts](Tools) states the same codes once in each tool's entry, and a change to one goes into the other. **Say what a pass covered.** Nearly every gate in both wrappers does: `check_dot_fit.mjs` gives the diagram count, `pick_a11y_sample.mjs --check` the sample size and the number of construct families in use, `check_publish_policy.mjs` the probe counts on both sides, `check_code_regions.mjs` the number of files swept, the number whose code regions moved and the number of fences found, `check_a11y.mjs` the page × theme × viewport product it audited. A gate silent on success says nothing about whether it examined anything, which is the state a gate that has quietly stopped working also reports. diff --git a/docs/Documentation/Tools.md b/docs/Documentation/Tools.md index 3aa5000e..739ce699 100644 --- a/docs/Documentation/Tools.md +++ b/docs/Documentation/Tools.md @@ -981,7 +981,7 @@ behind. That includes the target the IDE remembers for each project, which a `wi writes. **A probe builds for the target `--arch` names**, whatever the IDE remembers, so a kept IDE switched to `win64` does not make later runs on the same port build 64-bit. -Exit codes: **0** the probe ran and its output was captured; **1** the project has compile errors (the diagnostics are printed); **2** a refused command line (a source folder that is missing or has no `Settings` file included), no IDE or compiler, an IDE that did not start, a compile that never settled, a build that failed after a clean compile, a probe that never ran or stopped at a procedure that failed code generation, an LLVM run on a Community or Personal licence, an `--exe` run with no exe built, or a crash; **3** no output: the console held none before the timeout, or the probe printed none after its last `Debug.Cls`; **4** the compiler crashed, or restarted twice, while compiling the project; **5** the probe ended before it returned, its output printed all the same. The exe's exit code under `--exe` is reported, not passed on. +Exit codes: **0** the probe ran and its output was captured; **1** the project has compile errors (the diagnostics are printed); **2** a refused command line (a source folder that is missing or has no `Settings` file included), no IDE or compiler, an IDE that did not start, a compile that never settled, a build that failed after a clean compile, a probe that never ran or stopped at a procedure that failed code generation, an LLVM run on a Community or Personal licence, an `--exe` run with no exe built, or a crash; **3** no output: the console held none before the timeout, or the probe printed none after its last `Debug.Cls`; **4** the compiler crashed, or restarted twice, while compiling the project; **5** the probe ended before it returned, its output printed all the same; **6** under `--exe`, the exe exited with a code other than 0, or was still running after `--timeout` and was ended, its output and exit code printed all the same. A run that would exit 5 exits 5 whatever the exe did. ### addin_test.mjs {: #addin-test } diff --git a/scripts/tbrun.mjs b/scripts/tbrun.mjs index fa3af974..ebcc1aab 100644 --- a/scripts/tbrun.mjs +++ b/scripts/tbrun.mjs @@ -33,7 +33,9 @@ // call -- 3 no output: the build produced none in the console before the // timeout, or the probe ran and printed none after its last Debug.Cls -- 4 the // compiler crashed, or restarted twice, while compiling the project -- 5 the -// probe ended before it returned, its output printed all the same. +// probe ended before it returned, its output printed all the same -- 6 with +// --exe, the exe exited with a code other than 0, or was still running after +// --timeout; its output and exit code printed all the same. // // ---------------------------------------------------------------- why // @@ -190,7 +192,9 @@ Exit codes: after its last Debug.Cls 4 the compiler crashed, or restarted twice, while compiling the project 5 the probe ended before it returned (End, or an error that ended the run); what - it printed is printed all the same`; + it printed is printed all the same + 6 --exe: the exe exited with a code other than 0, or was still running after + --timeout and was ended; its output and exit code are printed all the same`; const { values, positionals } = withUsageError( () => @@ -623,6 +627,8 @@ if (returned === false) { "LLVM-compiled code does. Or it was still running, silent, after --quiet ms; raise --quiet for a slow probe.", ); } +if (exeRun?.timedOut) die(6, `tbrun: the exe was still running after --timeout, and was ended.`); +if (exeRun && exeRun.exitCode !== 0) die(6, `tbrun: the exe exited with code ${exeRun.exitCode}.`); // ------------------------------------------------------------------ helpers From a232c583f6c566d904105ec2bb062a86f31401a0 Mon Sep 17 00:00:00 2001 From: Kuba Sunderland-Ober Date: Fri, 2 Oct 2026 14:14:14 +0200 Subject: [PATCH 03/35] scripts: --build and --llvm for tbbuild and check_examples; buildProject reads past Debug.Cls --- WIP.ExamplesBuild.md | 56 ++++++++++ WIP.md | 6 +- docs/Documentation/Tools.md | 21 ++-- scripts/check_examples.mjs | 84 ++++++++++++++- scripts/lib/example-batches.mjs | 182 ++++++++++++++++++++++++++------ scripts/lib/tb-build.mjs | 47 ++++++++- scripts/lib/tb-ide.mjs | 77 +++++++++++++- scripts/lib/tb-project.mjs | 7 +- scripts/tbbuild.mjs | 128 +++++++++++++++++----- scripts/tbrun.mjs | 29 ++--- 10 files changed, 527 insertions(+), 110 deletions(-) diff --git a/WIP.ExamplesBuild.md b/WIP.ExamplesBuild.md index 71c50c52..95056a92 100644 --- a/WIP.ExamplesBuild.md +++ b/WIP.ExamplesBuild.md @@ -537,6 +537,62 @@ quiet about it deliberately, on the grounds that the filter is the caller's own says so as an advisory finding. **A narrowed run's results are not a full run's, and the tool has to be the thing that says which.** +### `--build` and `--llvm`: what a compile does not ask + +**A compile asks the front end; code generation runs in a build.** A sample can compile +clean and still fail the build, and under LLVM the IDE reports "a feature used in your code +is not yet supported with the LLVM compiler". `--build` presses Build on each project whose +compile has no errors, through `compileProject`'s `build` option, and `--llvm` (which implies +`--build`) writes `+llvm` into each batch's `compiler.buildOptions` and +`compiler.debugOptions`. A plain `--build` run is the control for an `--llvm` one: a sample +that fails only the second is one LLVM cannot generate code for. + +- **A failed build is a crash, for isolation.** `compileProject` returns code 5 and + `buildStaged` turns it into `{ crashed: true, buildFailed: true, named: }`. + Nothing in a build log names a sample, so `runBatch` halves the batch, and `together` + finds a set that fails only in combination. The notes and findings are worded by kind: + "fails the build", and a second line naming "the LLVM build" under `--llvm` and "the + build" otherwise, with the advice to record it in `BUGS-TO-REPORT.md` if it is the + compiler's fault. The lane's `llvm` field chooses the wording. +- **The canary does not stop a build.** `[EnforceWarnings(TB0005)]` keeps its `#Warning` a + warning whatever the project's settings say, and a warning does not stop the IDE building: + a probe with the canary module beside it builds and runs (tbrun, exit 0, BETA 995). +- **A project with compile errors is compiled and not built.** The run ends by counting + the samples that only ever sat in such a project: "N sample(s) in batches with errors were + compiled but not built". A sample counts as built if any project that held it, a smaller + one from isolating a larger included, was built. +- **`--llvm` needs a Professional or Ultimate licence.** `llvmLicence` in `tb-ide.mjs`, + shared with `tbrun`, reads the status bar's licence once the compile has settled, and + `compileProject` returns code 2 for a Community or Personal one. +- **A build that ends the compiler writes no failure line.** A native exception during an + LLVM build (`NATIVE EXCEPTION: ACCESS_VIOLATION`, then "restarting from MEMORY") leaves + the log without a success or a failure line, so `buildProject` treats the exception line + as a failure; before that it waited out its whole timeout. +- **A `[RunAfterBuild]` that calls `Debug.Cls` would erase the build log.** `buildProject` + wraps the page's `clearDebugConsole` (`keepClears`, as `tbrun` does) and reads what each + clear after its mark erased in front of the console, so Tools.md's own tbrun sample builds. +- **A failed build is believed when it repeats.** With four lanes, samples that build clean + alone fail with `[LINKER] FAILED to create type library`, a different one each run (BETA + 995: Inheritance.md in one full run, CEF's EnvironmentOptions.md in a narrowed one; both + clean with `--jobs 1`). `buildTwiceOnFailure` builds a failed batch again before halving, + and every part the halving builds the same way. The cause, a file the lanes share or + something else, was not found. +- **A project with errors says which.** `not built: b.twinproj has errors, the first ...` + prints when a compile has errors, and the run ends counting the samples never built. +- **A build adds three collision rules a compile does not have**, each measured on BETA 995 + as a false finding in the first full `--build` run: + - **`expect-error` samples are batched apart.** One such sample (Option.md, TB5079) left + the 129 samples beside it unbuilt, since a project with an error is not built. + - **A unit declaring its own `Sub Main` gets a project of its own, without `tbxMain`.** + Two Mains compile, but binding the startup object fails the build ("'Main' is + ambiguous"), as `tbxMain.twin`'s header says. `makeBatches`' `alone` picks the unit, + the batch carries `noMain`, and `stageBatch` leaves out `tbxMain.twin`. HelpFile, + PrevInstance, Project-Types and the two WinServicesLib groups failed this way. + - **A `[DllExport]` name counts among a sample's names.** Two samples exporting + `MyExportedFunction` (API-Declarations, Classes-and-Modules) compile together, and the + linker refuses them ("duplicate [DLLExport] functions detected"); as names, the batcher + keeps them apart. + ## Traps already paid for Each cost a run, either during the probing that produced this file or during the diff --git a/WIP.md b/WIP.md index 92f07a12..41f7a696 100644 --- a/WIP.md +++ b/WIP.md @@ -94,7 +94,7 @@ The rest of this file is the maintenance guide for updating existing pages or ad - `docs/Reference/Built-In/tbIDE/` — IDE Extensibility package (this is the **addin SDK**). The package is type-only — it ships **public interfaces + CoClasses** that an addin DLL binds to; every implementation behind them lives in the twinBASIC IDE itself. The user-facing surface is one entry-point factory (`tbCreateCompilerAddin`) plus 23 CoClasses grouped by role: the addin contract (`AddIn`), the root API (`Host`), the loaded `Project`, the editors collection (`Editor` / `CodeEditor` / `Editors`), the virtual file system (`FileSystem` / `FileSystemItem` / `Folder` / `File`), the in-IDE UI surface (`Toolbar` / `Toolbars` / `Button` / `ToolWindow` / `ToolWindows`), the HTML DOM inside a tool window (`HtmlElement` / `HtmlElements` / `HtmlElementProperty` / `HtmlElementProperties` / `HtmlEventProperty` / `HtmlEventProperties`), the `DebugConsole`, `KeyboardShortcuts`, `Themes`, and the single concrete user-instantiable helper class `AddinTimer`. Flat layout — one page per CoClass / Class plus the index landing. - `docs/Reference/Statements.md` — alphabetical index of language statements. - `docs/Reference/Procedures and Functions.md` — alphabetical index of procedures/functions. -- `docs/LLVM/` — the LLVM section: compiling with the LLVM back end. A top-level section between Features and Reference Section in the nav, at `nav_order: 6` (Features moved to 5 to make room). `index.md` is the landing page and `Getting-Started.md` the only page so far, with its screenshots under `Images/`. Everything it describes arrived in **BETA 984**; the local BETA 983 compiler restricts `+llvm` to standard-module procedures and to Professional/Ultimate, and has no LLVM project settings at all. Both of the page's samples are marked `check_build` and compile clean on 983 --- but the harness asks only the front end, which accepts `[CompilerOptions]` with every CPU flag in it; nothing runs LLVM code generation, so a clean run says nothing about the 984 behaviour the page describes. +- `docs/LLVM/` — the LLVM section: compiling with the LLVM back end. A top-level section between Features and Reference Section in the nav, at `nav_order: 6` (Features moved to 5 to make room). `index.md` is the landing page and `Getting-Started.md` the only page so far, with its screenshots under `Images/`. Everything it describes arrived in **BETA 984**. Its language claims are measured on BETA 995 with `tbrun --llvm` and `--exe`: the comma, space and comma-space spellings of `[CompilerOptions]` all turn LLVM and `+optimize` on, `[CompilerOptions("")]` turns it off under project-wide LLVM, the long CPU-flag sample runs, and an error is lost between procedures when either the callee or the caller is LLVM-compiled. Its samples are marked `check_build`, which compiles them; `examples.bat --llvm` also builds them with LLVM. **A new top-level section needs four things besides its folder**, and nothing checks the first and the third: a `nav_order` between its neighbours; an entry in `docs/_book.yml` for every page, in a part or in `left_out:` with a reason, which the build warns about under its `pdf:` summary when one is missing; a line in *Where content lives* in `docs/Documentation/Authoring.md`; and the page-count rise that `build.bat` writes to `builder/page-baseline.json`, committed with the pages. The Challenges and Videos sections are in `left_out:`, and so are the IDE pages that are still screenshots and labels; the IDE part names its pages one by one, so a new IDE page warns until it goes into the part or into `left_out:`. A link from the book to a left-out page opens the website, and the pass over `book.html` lists it as `OUT OF BOOK`. A section index lists its topics by hand and sets `has_toc: false`, or the template appends a second, automatic list of its children. - Footer rendering — [builder/template.mjs](builder/template.mjs)'s `renderFooterCustom()` renders the copyright line and, when `vba_attribution: true` is set in a page's frontmatter, an additional CC-BY-4.0 attribution line beneath it. @@ -136,7 +136,7 @@ node scripts/tbrun.mjs # what does it print ``` - **Give the executable backslashed paths, and `export` a full path to the project.** `export` prefixes `\\?\` to its project path, so a relative one, or one with forward slashes, reports `input twinproj file does not exist`; and a folder named with forward slashes cannot be created or even found, even when it exists. With backslashes `export` creates every missing level of its output folder. Redirect stdin (`\`), so the given `.twinproj` is untouched, and a plain `--build` is the control for an `--llvm` run. `--llvm` refuses a Community or Personal licence. - **It runs the IDE on a private Windows desktop**, so it cannot seize focus mid-sentence. Set `TBBUILD_SHOW=1` while working interactively and leave it unset for unattended runs --- a wedged IDE nobody can see is the failure that costs an afternoon. - **One project per IDE**, 8--11 seconds each and flat in project size. Reusing a live IDE for a second project wedges it, so the cold start is the unit of work, not overhead to optimise away. Concurrency is how to go faster: distinct `--port` values give distinct DevTools ports, user-data folders and desktops. - **Keep a probe that might crash the compiler in a project of its own.** twinBASIC runs the compiler in the same process as user code, so one bad probe can take the run down and cost the other thirty their answer. @@ -464,7 +464,7 @@ Why the report separates the wedged task from the merely blocked ones, and why - `test.bat` — the tests the *toolchain* has to pass: the publish-allowlist self-test (`scripts/check_publish_policy.mjs`), the gate-list check (`scripts/check_gate_lists.mjs`), the CI-workflow roster check (`scripts/check_ci_workflows.mjs`), the lint gate (`scripts/check_lint.mjs`), the site-search unit tests (`node --test test/search.test.mjs`), the markdown-plugin unit tests (`node --test test/render.test.mjs`), the date-formatter unit tests (`node --test test/strftime.test.mjs`), `check_examples.mjs`'s probes (`node --test test/example-batches.test.mjs`), the regex-safety gate (`scripts/check_regex_safety.mjs`), the code-region gate (`scripts/check_code_regions.mjs`), the page-count drift-guard probes (`scripts/check_page_baseline.mjs`), the book-coverage probes (`scripts/check_book_coverage.mjs`), the symbol-index probes (`scripts/check_symbol_index.mjs`), the twinBASIC-scanner probes (`scripts/check_twin_parsers.mjs`), the attribute-sweep probes (`scripts/check_attribute_sweep.mjs`), the command-line probes and cases (`scripts/check_cli.mjs`), the pdf-lib shim comparison (`scripts/check_pdf_shims_equiv.mjs`), the impexp parity check (`scripts/check_impexp_parity.mjs`), and the axe source-patch verification (`scripts/check_axe_patch_equiv.mjs`). ~23 s, of which the regex-safety gate is ~10 s and the impexp check ~4 s. See [What belongs in test.bat rather than check.bat](WIP.Build.md#what-belongs-in-testbat-rather-than-checkbat). - `book.bat` — renders the PDF from `docs\_site-pdf\book.html` via `node book\render-book.mjs` into `docs\_pdf\twinBASIC Book.pdf`. Run `build.bat` first to populate `_site-pdf/`; `book.bat` refuses a tree older than its sources rather than rendering the previous book (see [The book refuses a stale source tree](WIP.Build.md#the-book-refuses-a-stale-source-tree)). -- `examples.bat` — compiles the documentation's own twinBASIC code samples, every `tb` fence marked `check_build`, and reports the ones the compiler refuses against the line in the page they came from. Needs a twinBASIC install and Windows, so it is outside every gate and outside CI; ~120 s over the 1,136 samples marked. Two modes need no compiler at all: `--census` classifies every fence and says how many classifiable ones are still unmarked, and `--report ` groups a saved `--propose --json` survey by diagnostic, section and unresolved name. `--propose` itself does compile. See [Compiling the reference's own code samples](#compiling-the-references-own-code-samples) and [WIP.ExamplesBuild.md](WIP.ExamplesBuild.md). +- `examples.bat` — compiles the documentation's own twinBASIC code samples, every `tb` fence marked `check_build`, and reports the ones the compiler refuses against the line in the page they came from. Needs a twinBASIC install and Windows, so it is outside every gate and outside CI; ~120 s over the 1,136 samples marked. `--build` also builds each project that compiles clean, and `--llvm` builds it with LLVM (see [WIP.ExamplesBuild.md](WIP.ExamplesBuild.md)). Two modes need no compiler at all: `--census` classifies every fence and says how many classifiable ones are still unmarked, and `--report ` groups a saved `--propose --json` survey by diagnostic, section and unresolved name. `--propose` itself does compile. See [Compiling the reference's own code samples](#compiling-the-references-own-code-samples) and [WIP.ExamplesBuild.md](WIP.ExamplesBuild.md). - `addin-test.bat` — tests IDE add-ins by operating an IDE: every lane in `test/addin/lanes.mjs` builds the add-ins it tests into a private copy of the install, opens a project and checks what the add-in does. Outside every gate and outside CI for the same reasons as `examples.bat`; ~140 s for the ten lanes today: Samples 10 and 15, and the eight probe lanes behind Stage 2's answers in [WIP.HelpAddin.md](WIP.HelpAddin.md). Exit 0 every lane passed and the registry is as it was found, 1 a lane failed, 2 the harness failed, 3 the registry or a work folder was not put back. See [Driving the twinBASIC compiler](#driving-the-twinbasic-compiler) for its rules. - `ide-test.bat` --- the same runner for scenarios that operate the IDE itself rather than an add-in: every lane in `test/ide/lanes.mjs`, base port 9660. Same exit codes, same standing outside every gate and outside CI, same rules. diff --git a/docs/Documentation/Tools.md b/docs/Documentation/Tools.md index 739ce699..5b2f63de 100644 --- a/docs/Documentation/Tools.md +++ b/docs/Documentation/Tools.md @@ -810,8 +810,8 @@ Exit codes: **0** the trees match; **1** the trees differ; **2** a refused comma {: #tbbuild } node scripts/tbbuild.mjs [--ide ] [--port N] - [--arch win32|win64] [--timeout S] [--json] [--keep] - [--show|--hide] + [--arch win32|win64] [--timeout S] [--build | --llvm] + [--json] [--keep] [--show|--hide] Compiles a `.twinproj` and prints its diagnostics, with no IDE window to click through. This is how a claim the documentation makes about the language gets checked against the compiler rather than against memory: write a one-module project that uses the construct in the position you are asking about, run this, and read what comes back. Windows only, and no part of the site build. @@ -823,10 +823,14 @@ twinBASIC has no command-line build. The compiler executable's whole surface is | `--port ` | DevTools port. Default 9333. It also names the WebView2 user-data folder and the private desktop, which is what makes concurrent instances possible. A port another IDE already holds --- another run's, or another session's --- is refused after ten seconds, rather than attached to. | | `--arch ` | The target to compile for, `win32` or `win64`. Default `win32`. The diagnostics can differ between the two, because `#If Win64` and the size of `LongPtr` change what compiles. The target is set on every run, because the IDE opens a project in whatever target it last used for that project. Switching restarts the compiler, which then compiles the project again, so a switch adds a few seconds. When the target is not `win32`, or the IDE remembered another one for the project, the report starts with a `target:` line. | | `--timeout ` | Give up waiting for the compile to settle. Default 180. | -| `--json` | Emit one JSON object --- the target, counts, diagnostic rows, and the text of any alert the IDE opened, which is dismissed so the compile can go on --- instead of lines of text. | +| `--build` | After a compile with no errors, build the project, as the toolbar's Build button does, and print `built: ` after the summary line. The project is exported and packed again first (see below), so the given file is never changed. | +| `--llvm` | Build with LLVM: `--build`, with the project's compiler options set to `+llvm`. It is refused, with exit 2, on a Community or Personal licence, which would build with the default compiler and say nothing. Without it, `--build` is the control for an `--llvm` run. | +| `--json` | Emit one JSON object --- the target, counts, diagnostic rows, the text of any alert the IDE opened, which is dismissed so the compile can go on, and `built` (the file a build wrote, or null) and `buildLog` --- instead of lines of text. | | `--keep` | Leave the IDE running afterwards. The IDE's registry entries for the project are then left as they are, because the IDE is still writing them. | | `--show` / `--hide` | Put the IDE on your own desktop where you can watch it, or on a private one where it cannot take focus. Hidden is the default unless `TBBUILD_SHOW` is set to something other than `0`, `false` or `no`; the two flags override that for one invocation. | +**A compile does not generate code, and a build does.** The compiler's front end accepts a construct that its LLVM code generation refuses ("a feature used in your code is not yet supported with the LLVM compiler"), so a clean compile says nothing about an LLVM build. `--build` and `--llvm` press Build after the compile and read the build log. The default build path of a packed project opens a *Save* dialog that a private desktop hides, so either flag first exports the project into `%TEMP%\tbbuild\\src`, packs a copy with an explicit build path under `%TEMP%\tbbuild\\out` and a project id of its own, and opens that copy. A project with compile errors is not built: the report is the compile's, with exit 1. A build that fails, or that ends the compiler with a native exception, prints the build log on stdout and the failing line on stderr, and exits 5. A `[RunAfterBuild]` procedure runs during the build, and the log it would erase with `Debug.Cls` is kept and read all the same. + **It runs the IDE on a private Windows desktop, and that is not decoration.** The IDE calls `HostForceFocus()` from its own `window.onload`, so it takes the keyboard whatever window style it starts with --- `start /min` was tried and the window still came to the front. A process on another desktop has no foreground to take, and the compile does not care whether anything is on screen. Hidden by default has one real cost. A wedged IDE on a private desktop is invisible to the person debugging it, and the only way to see anything is to run it again visible. Export `TBBUILD_SHOW=1` for a session you are working through interactively, and leave it unset for unattended runs. **One IDE handles one project.** Loading a second project into a running IDE wedges it, so a fresh IDE per project is the design rather than a convenience. It costs roughly 8 to 11 seconds each on a development box and is flat in project size, because what is being paid for is IDE startup and not compilation. Concurrency is the way to make a batch of probes fast: distinct `--port` values give distinct DevTools ports, user-data folders and desktops, so instances do not collide. Keep a question that might crash the compiler in a project of its own, so the answer is attributable and one bad probe cannot cost the rest of the batch its run. @@ -837,7 +841,7 @@ twinBASIC has no command-line build. The compiler executable's whole surface is Six files under `scripts/lib/` belong to it and are never run directly. `tb-build.mjs` is `tbbuild` without its command line: `compileProject` opens a project in the IDE and returns its diagnostics as an array, which is how `check_examples.mjs` and `sweep_attributes.mjs` build many projects without starting a process for each. It never exits the process and never tidies the registry, so its caller owns both. `tb-ide.mjs` holds the mechanics `tb-build.mjs` and `tbrun.mjs` share: starting the IDE, attaching to it, waiting for the compile, and reading the diagnostics. `tb-ide-console.mjs` reads the IDE's DEBUG CONSOLE, which is where `tbrun` finds what a probe printed. `tb-registry.mjs` records and restores the registry entries described above, through .NET's registry API by way of PowerShell, because `reg.exe` mangles any path holding a character outside the console code page; [`check_tb_registry.mjs`](#check-tb-registry) is its self-test. `tb-cdp.mjs` is a minimal CDP client over Node's global `WebSocket`, raw rather than puppeteer because a pending `alert()` blocks the renderer and puppeteer's `connect()` handshake talks to the renderer --- so it hangs on precisely the state you need to recover from. Every call it makes has a time limit, so a blocked page ends a run with a message rather than holding it forever. `tb-launch.ps1` holds the Win32 calls Node cannot make without a native FFI addon: `CreateDesktop` and `CreateProcess` with `STARTUPINFO.lpDesktop` for the private desktop, and the job object described above. It is the only PowerShell file under `scripts/`, and it is not executed as a file: `tb-ide.mjs` reads the text and passes it through `-EncodedCommand`, so the execution policy never comes into it and nobody has to be told to bypass one. -Exit codes: **0** the project compiled without errors; **1** the project has errors; **2** a refused command line (a path that is not a `.twinproj` included), no IDE, an IDE that did not start or expose a debug port, or a crash; **3** the compile never settled: the IDE did not report the project open, or its diagnostics did not match its status bar; **4** the project crashes the compiler. +Exit codes: **0** the project compiled without errors; **1** the project has errors; **2** a refused command line (a path that is not a `.twinproj` included), no IDE, an IDE that did not start or expose a debug port, a project that could not be exported or packed, an `--llvm` run on a Community or Personal licence, or a crash; **3** the compile never settled: the IDE did not report the project open, or its diagnostics did not match its status bar; **4** the project crashes the compiler; **5** the build failed after a clean compile. ### tbrun.mjs {: #tbrun } @@ -1102,7 +1106,8 @@ Exit codes: **0** every assertion held, **1** an assertion failed, **2** the tes node scripts/check_examples.mjs [--only ] [--census] [--propose [--apply]] [--report ] [--jobs N] [--port N] [--batch N] - [--ide ] [--keep] [--verbose] [--json] + [--ide ] [--build | --llvm] [--keep] [--verbose] + [--json] Compiles the documentation's own code samples. A ` ```tb ` fence is something [`check_code_regions.mjs`](#check-code-regions) protects the *contents* of and nothing ever @@ -1145,6 +1150,8 @@ reports. | `--port ` | Base DevTools port. Default 9480; lane *n* uses base + *n*. | | `--batch ` | Upper bound on samples per generated project. Default 120. The batcher packs fewer than this when there are lanes to fill. | | `--ide ` | `twinBASIC.exe`. Default: `$TB_IDE`, else the newest `twinBASIC_IDE_BETA_` on the Desktop. | +| `--build` | Also build each project that compiles without errors. A project whose build fails is cut down, as a crash is, to the samples that fail it. A project whose compile has errors is not built: the run names its first error, and ends by counting its samples as "compiled but not built". | +| `--llvm` | Build with LLVM (implies `--build`): each project's compiler options are set to `+llvm`. It needs a Professional or Ultimate licence and exits 2 without one. A plain `--build` run is its control: a sample that fails only under `--llvm` is one LLVM cannot generate code for. | | `--keep` | Leave the generated projects on disk and print where. | | `--verbose` | Report warnings as well as errors. Only errors ever fail the run. | | `--json` | One object on stdout; every report line moves to stderr. | @@ -1176,6 +1183,8 @@ the batch crashes by itself. The samples it needs are then searched for as a set of them are reported. The cost is paid only on failure. The finding names the sample, or the set, and points at `BUGS-TO-REPORT.md`. +**A sample can compile and still fail the build.** The front end accepts constructs that code generation refuses, so `--build` presses Build on each project that compiled without errors, and `--llvm` does the same with LLVM switched on. Several lanes building at once fail builds that pass on their own (`[LINKER] FAILED to create type library`), so a project whose build fails is built once more, and only a second failure counts. `tbbuild` reports a failed build as exit 5 and names no sample, so the project is cut by halving, with the crash machinery and its costs, until one sample is left or the samples a failure needs together are found. The finding says "fails the build", and its second line says "the LLVM build" under `--llvm` and "the build" otherwise. The canary described below is a warning, so it does not stop a project from building. A build also refuses three things a compile accepts, so a run that builds batches around them: an `expect-error` sample is batched apart from the samples that should build; a sample, or a group, that declares its own `Sub Main` gets a project of its own without the template's `Main`, because two make the startup object ambiguous; and two samples that export one `[DllExport]` name are kept in different projects. + **A batch can report nothing when it should report something.** `tbbuild` does not wait for a build: it reads the IDE's own window, the status bar and the Problems panel for the project the IDE has open, once the compiler's status reads OPERATIONAL and has stopped changing. An IDE under load can be OPERATIONAL with an empty panel before it has published its diagnostics, and a batch read then reports every sample as compiling, which looks exactly like a batch with nothing wrong. So every batch carries a canary: a module holding a `#Warning` directive, whose warning (`TB0005`) is known. A read with no errors in it must report the canary, or it is not believed. The module carries `[EnforceWarnings(TB0005)]`, so a project setting that ignores the warning, or turns it into an error, does not change it. The warning is reported whatever else the batch holds: unterminated blocks, stray `End` statements, broken classes and many undefined names in other files do not hide it. It is a warning rather than an error so that a batch with nothing wrong still builds clean. A batch that crashes the compiler reports nothing at all and is isolated as a crash; its canary is never read. The canary proves only that the IDE published something, not that it published everything: a read that includes the canary but not a sample's later diagnostics would still pass that sample. So a read that holds real errors needs no canary --- the IDE was plainly not silent --- and is taken as read, whatever the canary did. Real errors here are errors in the batch's samples, and errors outside every sample that the template does not draw by itself; a template's own errors do not count, or a template that always draws one would switch the canary off for every batch built from it. A canary missing beside real errors has never been seen, and is printed as a note if it happens. A read with no errors and no canary is built once more, because a read that came too early says nothing about the batch. If it is silent again, the batch is split in half repeatedly, as for a crash, until each part reports the canary or errors of its own. A single unit --- one sample, or a group compiled as one program --- that is still silent stops the run with exit code 2 and its name, because its clean result cannot be trusted and it is not blamed for errors nobody saw. The template built with no samples, which is how the tool learns the errors a template draws by itself, follows the same rule: it needs its canary only if it has no errors, is read again when it is silent, and stops the run when it is silent twice. @@ -1206,7 +1215,7 @@ down, and holds the probes, which run before every run and in compiler beside it, and is shared with the two IDE-driving tools so the three cannot come to disagree about where an install is. -Exit codes: **0** every marked sample compiles, or none is marked (`--report` always, and `--propose` when it found only unmarked samples that fail, which is advisory); **1** a marked sample does not compile, a marker is misused, a template does not compile, or the compiler crashed on a project (the report names each); **2** the harness could not run: a refused command line, a failed self-test probe, no IDE or compiler, an unreadable `--report` file, a work folder it could not clear, or a crash. +Exit codes: **0** every marked sample compiles, or none is marked (`--report` always, and `--propose` when it found only unmarked samples that fail, which is advisory); **1** a marked sample does not compile, a marker is misused, a template does not compile, the compiler crashed on a project, or `--build` or `--llvm` found a sample that fails the build (the report names each); **2** the harness could not run: a refused command line, a failed self-test probe, no IDE or compiler, an unreadable `--report` file, a work folder it could not clear, an `--llvm` run on a Community or Personal licence, or a crash. ### gen_attribute_probes.mjs {: #gen-attribute-probes } diff --git a/scripts/check_examples.mjs b/scripts/check_examples.mjs index 9e8a3b67..241e990a 100644 --- a/scripts/check_examples.mjs +++ b/scripts/check_examples.mjs @@ -56,7 +56,9 @@ // `Sub Main` is not one of them, though it was once listed as one. The template // brings a Main, and a sample may bring its own beside it: two `Public Sub // Main`s in different modules compile (measured, BETA 983), which is how the -// WinServicesLib `Module Startup` samples build as written. +// WinServicesLib `Module Startup` samples build as written. That holds for a +// compile; a build refuses two Mains, and two [DllExport]s of one name, and a +// project with an error at all, so --build and --llvm batch around those three. // // And one that does not: a sample can take the compiler down. twinBASIC runs it // in-process with user code, and a two-line syntax skeleton in Attributes.md @@ -143,6 +145,8 @@ const { values } = withUsageError( verbose: { type: "boolean", default: false }, json: { type: "boolean", default: false }, keep: { type: "boolean", default: false }, + build: { type: "boolean", default: false }, + llvm: { type: "boolean", default: false }, show: { type: "boolean", default: false }, hide: { type: "boolean", default: false }, help: { type: "boolean", short: "h", default: false }, @@ -167,6 +171,10 @@ Compiles the documentation's own twinBASIC code samples, every tb fence marked --port base DevTools port (default 9480) --batch samples per generated project (default 120) --ide twinBASIC.exe (default: $TB_IDE, else the newest on the Desktop) + --build also build each project that compiles without errors; a project + whose build fails is cut down to the samples that fail it + --llvm build with LLVM (implies --build); needs a Professional or + Ultimate licence, and --build alone is its control --keep leave the generated projects on disk and say where --show, --hide as tbbuild's --verbose also print warnings, not only errors @@ -177,10 +185,11 @@ Exit codes: 0 every marked sample compiles, or none is marked; --report always, and --propose when it found only unmarked samples that fail (advisory) 1 a marked sample does not compile, a marker is misused, a template does not - compile, or the compiler crashed on a project; the report names each + compile, the compiler crashed on a project, or --build or --llvm found a sample + that fails the build; the report names each 2 the harness could not run: a refused command line, a failed self-test probe, no IDE or compiler, an unreadable --report file, a work folder it could not clear, - or a crash`; + an --llvm run on a Community or Personal licence, or a crash`; if (values.help) printHelpAndExit(USAGE); @@ -208,6 +217,18 @@ const MODE_REPORT = values.report ?? null; const APPLY = values.apply; const VERBOSE = values.verbose; const AS_JSON = values.json; +const LLVM = values.llvm; +// --llvm is a build, so the one switch the rest of the file reads is BUILD. +const BUILD = values.build || LLVM; +// The procedure a [DllExport] attribute is on, for a run that builds. +const DLL_EXPORT_NAME = /\[\s*DllExport\b[^\]]*\][\s_]*(?:(?:Public|Private|Friend)\s+)?(?:Function|Sub)\s+(\w+)/gi; + +// What a run that builds could say about each sample. A project whose compile has +// errors is not built (the IDE refuses one), so its samples are compiled and +// nothing more; a sample is counted as unbuilt only if no build of any project +// that held it, a smaller one from isolating a larger included, ever ran. +const builtIds = new Set(); +const unbuiltIds = new Set(); // A page's template, when its fence does not name one. Inferred from the path // because the package a sample needs is what the page is ABOUT -- stating @@ -362,7 +383,12 @@ function select(fences) { // fact written twice, and the pair could then disagree. fence.base = fence.keys.get("inherits") ?? null; if (fence.base && slot) slot = PROMOTE_TO_CLASS[slot] ?? slot; - fence.inferred = inferred; + // A [DllExport] name is the exe's, whatever module declares it, and a build + // refuses two ("[LINKER] FAILED duplicate [DLLExport] functions detected"), + // which a compile accepts. Counted among the names, it keeps two samples + // exporting one name out of one project, as a clash of names does. + const exported = BUILD ? [...fence.content.matchAll(DLL_EXPORT_NAME)].map((m) => `[DllExport] ${m[1]}`) : []; + fence.inferred = exported.length ? { ...inferred, names: [...(inferred.names ?? []), ...exported] } : inferred; fence.slot = slot; fence.slotStated = Boolean(stated); fence.project = fence.keys.get("project") ?? defaultProject(fence.rel); @@ -480,6 +506,8 @@ function stageBatch(batch, work) { for (const name of templateChain(batch.project)) { cpSync(path.join(TEMPLATES, name), dir, { recursive: true }); } + // The sample brings the project's Main (see makeBatches' `alone`). + if (batch.noMain) rmSync(path.join(dir, "Sources", "tbxMain.twin"), { force: true }); const settingsPath = path.join(dir, "Settings"); const settings = JSON.parse(readFileSync(settingsPath, "utf8")); @@ -492,6 +520,11 @@ function stageBatch(batch, work) { // invisible and unreachable -- so the build never happens while the WebView2 // renderer stays responsive and every health check says the IDE is fine. settings["project.buildPath"] = path.join(dir, `${name}.exe`).split("/").join("\\"); + // The run's compiler options and the exe's alike, as tbrun's --llvm sets them. + if (LLVM) { + settings["compiler.debugOptions"] = "+llvm"; + settings["compiler.buildOptions"] = "+llvm"; + } writeFileSync(settingsPath, JSON.stringify(settings, null, "\t") + "\n", "utf8"); const map = new Map(); @@ -552,12 +585,22 @@ async function buildStaged(staged, port) { ide: IDE, port, show: wantShow({ show: values.show, hide: values.hide }), + build: BUILD, + llvm: LLVM, }); if (r.code === 4) return { crashed: true, detail: r.message, named: crashedIn(r.crashFiles, staged.map) }; + // A build that failed after a clean compile is isolated as a crash is: nothing + // names a sample, so it is cut down by halving. `named` is empty for that. + if (r.code === 5) return { crashed: true, buildFailed: true, named: new Set(), detail: r.message }; if (r.code !== 0 && r.code !== 1) { throw new Error(`tbbuild exited ${r.code} on ${staged.proj}\n${r.message}`); } + if (BUILD) { + for (const entry of staged.map.values()) (r.code === 0 ? builtIds : unbuiltIds).add(entry.fence.id); + const first = r.rows.find((row) => row.startsWith("{ERROR}")); + if (r.code === 1) say(` not built: ${path.basename(staged.proj)} has errors, the first ${first}`); + } const result = { diagnostics: r.rows }; const perFence = new Map(); @@ -625,6 +668,7 @@ function laneOf(port, work) { }, finding: addFinding, note: say, + llvm: LLVM, }; } @@ -887,7 +931,32 @@ async function main() { process.exit(findings.some((f) => !f.advisory) ? 1 : 0); } - const batches = makeBatches(selected, { batchSize, jobs }); + // A project with an error is not built, and an `expect-error` sample is one on + // purpose; batched with the rest it would leave every sample beside it unbuilt. + // So a run that builds batches those samples apart, each with its page's + // hidden blocks, which the rest of the page keeps as well. + const expectsError = (f) => f.keys.has("expect-error"); + // A build binds the startup object, and fails on a template Main beside a + // sample's own ("'Main' is ambiguous"), which a compile accepts; so a run that + // builds gives such a unit a project of its own without the template's. + const DECLARES_MAIN = /^[ \t]*(?:(?:Public|Private|Friend)[ \t]+)?Sub[ \t]+Main[ \t]*\(/im; + const alone = BUILD ? (members) => members.some((f) => DECLARES_MAIN.test(f.content)) : null; + const errorPages = new Set(selected.filter(expectsError).map((f) => f.rel)); + const hiddenOfErrorPage = (f) => f.flags.has(HIDDEN_MARKER) && errorPages.has(f.rel); + const batches = BUILD + ? [ + ...makeBatches( + selected.filter((f) => !expectsError(f)), + { batchSize, jobs, alone }, + ), + ...(errorPages.size + ? makeBatches( + selected.filter((f) => expectsError(f) || hiddenOfErrorPage(f)), + { batchSize, jobs, alone }, + ) + : []), + ] + : makeBatches(selected, { batchSize, jobs }); const work = path.join(tmpdir(), "tbexamples", String(basePort)); try { rmSync(work, { recursive: true, force: true }); @@ -1068,6 +1137,11 @@ async function main() { `${real} finding(s), ${secs}s` + (real ? "" : " -- clean"), ); + // A project with errors is never built, so the build asked nothing of these. + if (BUILD) { + const unbuilt = samples.filter((f) => unbuiltIds.has(f.id) && !builtIds.has(f.id)).length; + if (unbuilt) say(`${unbuilt} sample(s) in batches with errors were compiled but not built`); + } } if (values.keep) say(`generated projects kept in ${work}`); else rmSync(work, { recursive: true, force: true }); diff --git a/scripts/lib/example-batches.mjs b/scripts/lib/example-batches.mjs index 3cbe2539..df4d0ccc 100644 --- a/scripts/lib/example-batches.mjs +++ b/scripts/lib/example-batches.mjs @@ -83,7 +83,13 @@ function unitKey(fence) { return fence.keys.get("projname") ? `@${fence.keys.get("projname")}` : `#${fence.id}`; } -export function makeBatches(fences, { batchSize = DEFAULT_BATCH, jobs = DEFAULT_JOBS } = {}) { +/** + * `alone`, given a unit's fences, says whether the unit is to be built in a + * project of its own with no template Main (`noMain` on the batch): a run that + * builds passes one for a unit declaring its own `Sub Main`, because a build, + * unlike a compile, fails on two. + */ +export function makeBatches(fences, { batchSize = DEFAULT_BATCH, jobs = DEFAULT_JOBS, alone = null } = {}) { // A `hidden` fence is not a unit of its own: it is the PAGE's context, and // it joins every project that holds a sample from that page. So a page can // carry the declarations its samples assume -- the class its prose describes @@ -156,7 +162,9 @@ export function makeBatches(fences, { batchSize = DEFAULT_BATCH, jobs = DEFAULT_ // author can actually reason about: what compiles is what they grouped, // plus the template. It costs one project per group, and groups are // written by hand, so there are never many. - if (key.startsWith("@")) { + // A unit `alone` picks gets one too, built without the template's Main. + const solo = !!alone?.(members); + if (key.startsWith("@") || solo) { const own = new Set(names); for (const rel of pages) for (const n of hiddenNamesFor(rel)) own.add(n); batches.push({ @@ -164,7 +172,8 @@ export function makeBatches(fences, { batchSize = DEFAULT_BATCH, jobs = DEFAULT_ fences: [...members], names: own, pages, - group: key.slice(1), + ...(key.startsWith("@") ? { group: key.slice(1) } : {}), + ...(solo ? { noMain: true } : {}), }); continue; } @@ -382,7 +391,11 @@ function ownRowsOf(project, lane) { ); } const rows = result.crashed - ? [`the ${project} template crashes the compiler with no samples in it`] + ? [ + result.buildFailed + ? `the ${project} template fails the build with no samples in it` + : `the ${project} template crashes the compiler with no samples in it`, + ] : [...(result.unattributed ?? []), ...(result.unreadable ?? [])]; if (rows.length) { lane.note( @@ -460,18 +473,26 @@ async function split(batch, lane, why) { * with nothing in `crashed` -- its crash needed several samples too -- and * whether a part crashed is what decides the next step. */ -async function isolateCrash(batch, named, lane) { - const where = `a compiler crash in ${batch.fences.length} sample(s) [${batch.project}]`; +async function isolateCrash(batch, named, lane, failed = false, detail = "") { + const where = `${failed ? "a failed build" : "a compiler crash"} in ${batch.fences.length} sample(s) [${batch.project}]`; let parts = takeOut(batch, named); if (parts) lane.note(` ${where}: building the sample it died parsing on its own`); else if ((parts = splitBatch(batch))) lane.note(` ${where}: splitting to find it`); else { const { rep, ids, rest } = leafOf(batch); - lane.finding( - rep, - "crashes the twinBASIC compiler" + rest, - "the compiler dies parsing this sample; record it in BUGS-TO-REPORT.md", - ); + if (failed) { + lane.finding( + rep, + "fails the build" + rest, + `${buildName(lane)} fails on this sample${detail ? ` (${detail})` : ""}; record it in BUGS-TO-REPORT.md if it is the compiler's fault`, + ); + } else { + lane.finding( + rep, + "crashes the twinBASIC compiler" + rest, + "the compiler dies parsing this sample; record it in BUGS-TO-REPORT.md", + ); + } // Named back to the caller, because a crashed sample produced no // diagnostics and would otherwise be counted as one that compiled -- the // same false-clean shape tbbuild's own crash check exists to close. @@ -480,10 +501,36 @@ async function isolateCrash(batch, named, lane) { const subs = []; for (const part of parts) subs.push(await runBatch(part, lane)); if (subs.some((s) => s.fromCrash)) return { ...merge(subs), fromCrash: true }; - lane.note(" neither part crashes on its own: looking for the samples it needs together"); - return { ...merge([...subs, await together(batch, ...parts, lane)]), fromCrash: true }; + lane.note( + ` neither part ${failed ? "fails the build" : "crashes"} on its own: looking for the samples it needs together`, + ); + return { ...merge([...subs, await together(batch, ...parts, lane, failed)]), fromCrash: true }; +} + +/** + * Build a batch, and build it once more if the build failed after a clean + * compile. + * + * Lanes building at once fail builds that pass alone: "[LINKER] FAILED to + * create type library" on samples that built clean with one lane (BETA 995, + * and BETA 983 under tbrun). A failure is believed only if the same batch fails + * twice running, and so is each part the halving that follows builds. + */ +async function buildTwiceOnFailure(batch, lane) { + const first = await lane.build(batch); + if (!first.crashed || !first.buildFailed) return first; + lane.note( + ` a failed build in ${batch.fences.length} sample(s) [${batch.project}] (${first.detail}): building it again`, + ); + return lane.build(batch); } +/** + * What the lane's build is called in a finding: LLVM is the point of an --llvm + * run, and the plain build is its control. + */ +const buildName = (lane) => (lane.llvm ? "the LLVM build" : "the build"); + /** * The samples a crash needs when it needs several: `a` and `b` each built * clean, and together they crash. @@ -498,7 +545,7 @@ async function isolateCrash(batch, named, lane) { * Every member built clean in `a` or `b`, so each has its own result. They are * blamed rather than passed, because together they take the compiler down. */ -async function together(batch, a, b, lane) { +async function together(batch, a, b, lane, failed = false) { const { hidden } = unitsOf(batch); const crashes = async (units) => !!(await lane.build(batchOf(batch, units, hidden))).crashed; const partners = async (fixed, pool) => { @@ -519,14 +566,24 @@ async function together(batch, a, b, lane) { .flat() .filter((f) => !f.flags.has(HIDDEN_MARKER) && !f.isResource); const [rep, ...others] = members; - lane.finding( - rep, - `crashes the twinBASIC compiler when built with ${others.length} other ` + - `sample(s), though none of the ${members.length} does on its own`, - `the others: ${others.map((f) => `${f.rel}:${f.line}`).join(", ")}` + - (found ? "" : "; no smaller set that crashes was found") + - " -- record it in BUGS-TO-REPORT.md", - ); + const theOthers = `the others: ${others.map((f) => `${f.rel}:${f.line}`).join(", ")}`; + if (failed) { + lane.finding( + rep, + `fails the build when built with ${others.length} other ` + + `sample(s), though none of the ${members.length} does on its own`, + theOthers + + (found ? "" : "; no smaller set that fails was found") + + `; ${buildName(lane)} fails on them together -- record it in BUGS-TO-REPORT.md if it is the compiler's fault`, + ); + } else { + lane.finding( + rep, + `crashes the twinBASIC compiler when built with ${others.length} other ` + + `sample(s), though none of the ${members.length} does on its own`, + theOthers + (found ? "" : "; no smaller set that crashes was found") + " -- record it in BUGS-TO-REPORT.md", + ); + } return { ...blank(), blamed: members.map((f) => f.id) }; } @@ -543,8 +600,8 @@ async function together(batch, a, b, lane) { * compiling while the run fails with a row naming no page. */ export async function runBatch(batch, lane) { - let result = await lane.build(batch); - if (result.crashed) return isolateCrash(batch, result.named, lane); + let result = await buildTwiceOnFailure(batch, lane); + if (result.crashed) return isolateCrash(batch, result.named, lane, !!result.buildFailed, result.detail); // The canary is needed only by a read that holds no error: that is the read // an IDE that published nothing returns, and it looks exactly like a clean @@ -558,8 +615,8 @@ export async function runBatch(batch, lane) { ` the canary did not report in ${batch.fences.length} sample(s) [${batch.project}] ` + `(${result.canaryProblem}), and nothing else did: building it again`, ); - result = await lane.build(batch); - if (result.crashed) return isolateCrash(batch, result.named, lane); + result = await buildTwiceOnFailure(batch, lane); + if (result.crashed) return isolateCrash(batch, result.named, lane, !!result.buildFailed, result.detail); if (result.canaryProblem && !(await heard(result, batch.project, lane))) { // Twice silent: either the IDE keeps reading early under this load, or a // sample in the batch hides the diagnostics of the rest. Halving finds @@ -868,6 +925,13 @@ export async function runProbes(say) { // definitions its tests need, and a page's hidden context left in the other // half takes away the declarations it exists to supply -- and the hidden // fences sit at the END of batch.fences, where a plain slice drops them. + // A unit `alone` picks is a project of its own, marked to leave out the + // template's Main; the rest batch as before. + const solo = makeBatches([fake("s1"), fake("s2"), fake("s3")], { alone: (m) => m.some((f) => f.id === "s2") }); + const soloShape = solo.map((b) => `${b.fences.map((f) => f.id)}${b.noMain ? "!" : ""}`).join(" "); + if (soloShape !== "s2! s1,s3" && soloShape !== "s1,s3 s2!") { + failures.push(`batching: a unit alone picks -> ${soloShape}, want s2 alone and marked`); + } const soleGroup = makeBatches([fake("g1", "g"), fake("g2", "g")])[0]; if (splitBatch(soleGroup) !== null) failures.push("split: a projname group was cut in half"); const mixed = makeBatches([ @@ -935,10 +999,19 @@ export async function runProbes(say) { lane.builds++; const c = crash(new Set(b.fences.map((f) => f.id))); return c - ? { crashed: true, named: new Set(c.named ?? []) } + ? { + crashed: true, + buildFailed: !!c.failed, + detail: c.failed ? "[BUILD] FAILED boom" : undefined, + named: new Set(c.named ?? []), + } : { perFence: new Map(), unreadable: [], unattributed: [] }; }, - finding: (fence) => lane.found.push(fence.id), + said: [], + finding: (fence, message, detail) => { + lane.found.push(fence.id); + lane.said.push([message, detail]); + }, note: () => {}, }; return lane; @@ -967,6 +1040,48 @@ export async function runProbes(say) { const cheap = fakeLane((ids) => when("c")(ids) && naming("c")(ids)); await runBatch(five, cheap); if (cheap.builds !== 3) failures.push(`isolation: a named crash took ${cheap.builds} builds, want 3`); + // A build that fails after a clean compile names no sample, so it is halved to + // the one that fails it, and reported in the build's words -- the LLVM build + // under --llvm, the build without it -- never as a compiler crash. + const failsBuild = (...need) => fakeLane((ids) => when(...need)(ids) && { failed: true }); + const lone = failsBuild("d"); + lone.llvm = true; + const loneResult = await runBatch(five, lone); + const loneSaid = lone.said[0] ?? []; + if (`${loneResult.crashed} / ${lone.found}` !== "d / d") { + failures.push(`failed build: a sample that fails alone -> ${loneResult.crashed} / ${lone.found}, want d / d`); + } + if (loneSaid[0] !== "fails the build" || !loneSaid[1]?.includes("the LLVM build fails on this sample")) { + failures.push(`failed build: an --llvm finding was worded ${JSON.stringify(loneSaid)}`); + } + if (!loneSaid[1]?.includes("([BUILD] FAILED boom)")) { + failures.push("failed build: the finding does not quote the build's failing line"); + } + if (!loneSaid[1]?.includes("BUGS-TO-REPORT.md if it is the compiler's fault")) { + failures.push("failed build: the finding does not say when to record it"); + } + const control = failsBuild("d"); + await runBatch(five, control); + if (!control.said[0]?.[1]?.startsWith("the build fails on this sample")) { + failures.push(`failed build: a plain build's finding was worded ${JSON.stringify(control.said[0])}`); + } + // A failure that does not repeat is no finding: the batch is built again. + let calls = 0; + const flaky = fakeLane(() => ++calls === 1 && { failed: true }); + const flakyResult = await runBatch(five, flaky); + if (flaky.found.length || flakyResult.crashed.length || flaky.builds !== 2) { + failures.push( + `failed build: one that passed when built again -> ${flaky.found} / ${flakyResult.crashed} in ${flaky.builds} builds`, + ); + } + const pair = failsBuild("b", "d"); + const pairResult = await runBatch(five, pair); + if ( + `${[...pairResult.blamed].sort()}` !== "b,d" || + !pair.said[0]?.[0].startsWith("fails the build when built with 1 other sample(s)") + ) { + failures.push(`failed build: two samples that fail only together -> ${JSON.stringify(pair.said[0])}`); + } // The canaries: what a batch's rows say about them, and what runBatch does when // they are not there. A batch with no rows at all is the one that matters, since // it is what an IDE that read its diagnostics early returns. @@ -1236,14 +1351,15 @@ export async function runProbes(say) { for (const f of failures) say(`FAIL probe: ${f}`); return false; } - // 10 line-map (5 slots x 2 bases) + 4 wrapper container + 7 batching - // + 5 splitting + 5 taking out + 3 crash report + 7 isolation + 9 concat + // 10 line-map (5 slots x 2 bases) + 4 wrapper container + 8 batching + // (1 of a unit built alone) + 5 splitting + 5 taking out + 3 crash report + 7 isolation + 9 concat // + 10 resource + 8 report + 6 markup + 15 canaries (5 of what a batch's rows say, // 7 of what runBatch does about a batch whose canary did not report, 3 of the - // template built with no samples). + // template built with no samples) + 7 of a failed build (wording, when to + // record it, one that passes when built again, a pair that fails only together). say( - `ok ${CLASSIFIER_PROBES.length + INFO_PROBES.length + 89} probes: ` + - "classifier, markup, line mapping, batching, splitting, crash isolation, canaries, concat, " + + `ok ${CLASSIFIER_PROBES.length + INFO_PROBES.length + 97} probes: ` + + "classifier, markup, line mapping, batching, splitting, crash isolation, failed builds, canaries, concat, " + "resources and the report", ); return true; diff --git a/scripts/lib/tb-build.mjs b/scripts/lib/tb-build.mjs index 8b1ce8b6..2bd8bad6 100644 --- a/scripts/lib/tb-build.mjs +++ b/scripts/lib/tb-build.mjs @@ -34,8 +34,10 @@ import { COMPILE_TIMEOUT, TARGETS, attachIde, + buildProject, compileOutcome, launchIde, + llvmLicence, setBuildTarget, shutdownIdeAsync, waitForCompile, @@ -50,14 +52,27 @@ import { * @param {number} [o.timeout] ms to wait for the compile (default COMPILE_TIMEOUT) * @param {boolean} [o.show] on the user's desktop instead of a private one * @param {boolean} [o.keep] leave the IDE running, and report its pid + * @param {boolean} [o.build] after a compile with no errors, build the project + * as the toolbar's Build button does. The project's settings decide what is + * built and how: the caller stages them (lib/tb-project.mjs), because a + * project opened in place has the template's build path, whose Save dialog + * a private desktop hides + * @param {boolean} [o.llvm] refuse a licence that does not compile user code + * with LLVM. It checks the licence and changes no setting; the project's + * own `compiler.buildOptions` is what asks for LLVM + * @param {number} [o.buildTimeout] ms to wait for the build (default buildProject's) * @returns {Promise<{ * code: number, message: string, rows: string[], counts: number[], dialogs: string[], * openedIn: string | null, arch: string, idePid: number | null, kept: boolean, crashFiles: string[], + * built: string | null, buildLog: string[], * }>} `code` is tbbuild's exit code: 0 clean, 1 the project has errors, 2 the IDE - * could not be started or attached, 3 the compile never settled, 4 the project - * crashes the compiler. `message` is what tbbuild prints on stderr for 2, 3 - * and 4 and is empty otherwise. `counts` is errors, warnings, hints, infos. - * `crashFiles` names, for 4, the files the compiler died parsing. + * could not be started or attached or the licence refuses LLVM, 3 the compile + * never settled, 4 the project crashes the compiler, 5 the build failed after + * a clean compile. `message` is what tbbuild prints on stderr for 2 to 5 and + * is empty otherwise: for 5 the failing line of the log. `counts` is errors, + * warnings, hints, infos. `crashFiles` names, for 4, the files the compiler + * died parsing. `built` is the file a successful build wrote, and `buildLog` + * the console from the build's first line on. */ export async function compileProject({ project, @@ -67,6 +82,9 @@ export async function compileProject({ timeout = COMPILE_TIMEOUT, show = false, keep = false, + build = false, + llvm = false, + buildTimeout, }) { let handle = null; let c = null; @@ -81,6 +99,8 @@ export async function compileProject({ idePid: handle?.pid ?? null, kept: keep, crashFiles: [], + built: null, + buildLog: [], ...fields, }); try { @@ -110,7 +130,24 @@ export async function compileProject({ } catch (e) { return result(2, { message: e.message }); } - return result(outcome.counts[0] > 0 ? 1 : 0, { rows: outcome.rows, counts: outcome.counts, openedIn }); + const found = { rows: outcome.rows, counts: outcome.counts, openedIn }; + + // The licence is read once the compile has settled, which is when the IDE + // knows it. A project that is to be built with LLVM on a licence that does + // not compile user code with LLVM would build with the default compiler and + // say nothing, so it is refused whatever the compile found. + if (llvm) { + const { refusal } = await llvmLicence(c); + if (refusal) return result(2, { ...found, message: refusal }); + } + if (outcome.counts[0] > 0) return result(1, found); + + // The IDE does not build a project it flags with errors, so a build is asked + // of a clean compile only. A warning does not stop one. + if (!build) return result(0, found); + const built = await buildProject(c, { timeout: buildTimeout }); + if (!built.ok) return result(5, { ...found, message: built.message, buildLog: built.log }); + return result(0, { ...found, built: built.file, buildLog: built.log }); } finally { // A close that threw must not skip ending the IDE, or mask what was thrown. try { diff --git a/scripts/lib/tb-ide.mjs b/scripts/lib/tb-ide.mjs index 5dfe6a84..5cb65670 100644 --- a/scripts/lib/tb-ide.mjs +++ b/scripts/lib/tb-ide.mjs @@ -15,7 +15,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { attach } from "./tb-cdp.mjs"; import { click } from "./tb-click.mjs"; -import { consoleMark, linesSince } from "./tb-ide-console.mjs"; +import { consoleMark, keepClears, keptClears, linesSince, readConsole } from "./tb-ide-console.mjs"; export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); @@ -806,7 +806,68 @@ export async function awaitNewCompiler(c, before, { why = "restarting it", timeo // ...", "[BUILD] failed" and "[LINKER] compilation (codegen) error ...". const BUILD_START = "[BUILD] Starting..."; const BUILD_OK = /^\[LINKER\] SUCCESS created output file '(.+)'$/; -export const BUILD_FAILED = /^\[(?:LINKER|BUILD)\] (?:FAILED|ERROR|failed)\b|^\[LINKER\] compilation \(codegen\) error/; +// The last is LLVM's refusal of a procedure, which has no bracketed prefix: +// "LLVM compilation error in 'Probe.Stopper': Unable to compile due to use of +// datatype that is not yet supported for LLVM compilation" (BETA 983). +export const BUILD_FAILED = + /^\[(?:LINKER|BUILD)\] (?:FAILED|ERROR|failed)\b|^\[LINKER\] compilation \(codegen\) error|^LLVM compilation error in /; +// A compiler that crashes while building -- measured under LLVM, BETA 995 -- +// writes "