From c564f2d5f0e46dd90b535874875a594c3a666e2c Mon Sep 17 00:00:00 2001 From: guangtao Date: Sun, 1 Nov 2026 23:34:33 +0800 Subject: [PATCH 1/2] fix(render): reclaim compiler allocation domains before returning --- packages/render/README.org | 8 ++++++++ packages/render/src/compiler-process.ts | 21 +++++++++++++++++++++ packages/render/src/index.ts | 21 +++++++++++++++------ packages/render/test/renderer.test.ts | 10 ++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 packages/render/src/compiler-process.ts diff --git a/packages/render/README.org b/packages/render/README.org index 6cd1083..8afd59a 100644 --- a/packages/render/README.org +++ b/packages/render/README.org @@ -273,3 +273,11 @@ bun add @deterministic-engineering/render@git+https://github.com/hardenedlinux/d #+end_src Bun owns Git fetching and the lockfile. A Git install needs no committed package archive. Release tags are created only after the package/consumer checks pass. +* Compiler allocation lifetime +=compileModel= owns one ephemeral compiler process and its internal SDK worker. +The result is accepted only after the process closes its worker, disconnects +IPC and exits naturally. No process exit call, signal kill, fallback renderer +or consumer-maintained pool is used. This process boundary releases terminated +Bun worker allocation domains that otherwise remain in the long-lived host. +Concurrent callers remain independent; the Library imposes no CPU-count clamp. +The host still owns cancellation, batch scheduling and browser lifetime. diff --git a/packages/render/src/compiler-process.ts b/packages/render/src/compiler-process.ts new file mode 100644 index 0000000..fa872db --- /dev/null +++ b/packages/render/src/compiler-process.ts @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 hardenedlinux Linux Team and Contributors +// +// SPDX-License-Identifier: LGPL-2.1-or-later + +import {Worker} from 'node:worker_threads'; +import type {ModelRequest,ModelResponse} from './worker'; +/** The process owns the compiler worker's native allocation domain. Bun may + * retain terminated worker heaps until process exit; wait for natural exit. */ +process.on('message',async(request:ModelRequest)=>{ + let worker:Worker|undefined; + let response:ModelResponse; + try{ + worker=new Worker(new URL('./worker.ts',import.meta.url),{workerData:request}); + response=await new Promise((resolve,reject)=>{ + worker!.once('message',resolve);worker!.once('error',reject); + worker!.once('exit',code=>reject(new Error('Compiler worker exited before result: '+code))); + }); + }catch(error:unknown){response={error:error instanceof Error?error.message:String(error)};} + finally{if(worker)await worker.terminate();} + process.send!(response,error=>{if(error)process.exitCode=1;process.disconnect?.();}); +}); diff --git a/packages/render/src/index.ts b/packages/render/src/index.ts index f624503..6d59f95 100644 --- a/packages/render/src/index.ts +++ b/packages/render/src/index.ts @@ -9,7 +9,8 @@ export {composeTaste,reviewTaste,minimumTextSize,focalBudget,type Taste,type Tas export {composeHeading,type HeadingSpec} from './heading'; export {composeModelView,graphFromModel,modelDiagram} from './model-view'; import {isDeepStrictEqual} from 'node:util'; -import {Worker} from 'node:worker_threads'; +import {fork} from 'node:child_process'; +import {fileURLToPath} from 'node:url'; import type {CompileOptions} from '@terrastruct/d2'; import {relativeSource} from './source/catalog'; import {readD2Figure,type FigureResources} from './d2-figure'; @@ -30,11 +31,19 @@ export async function compileModel(sources:Readonly>,input relativeSource(inputPath); if(!Object.hasOwn(sources,inputPath))throw new Error('Missing model source'); for(const[file,text]of Object.entries(sources)){relativeSource(file);if(typeof text!=='string')throw new Error('Invalid model source');} - const worker=new Worker(new URL('./worker.ts',import.meta.url),{workerData:{fs:{...sources},inputPath,options} satisfies import('./worker').ModelRequest}); - try{return await new Promise((resolve,reject)=>{ - worker.once('error',reject);worker.once('exit',code=>reject(new Error(`Model worker exited before result: ${code}`))); - worker.once('message',(message:import('./worker').ModelResponse)=>'error'in message?reject(new Error(message.error)):resolve(message.capture)); - });}finally{await worker.terminate();} + const child=fork(fileURLToPath(new URL('./compiler-process.ts',import.meta.url)),[],{execPath:process.execPath,stdio:['ignore','inherit','inherit','ipc']}); + return await new Promise((resolve,reject)=>{ + let response:import('./worker').ModelResponse|undefined; + child.once('error',reject); + child.on('message',(value:import('./worker').ModelResponse)=>{if(response){response={error:'Duplicate compiler result'};}else response=value;}); + child.once('exit',(code,signal)=>{ + if(code!==0||signal)return reject(new Error('Compiler process exit: '+code+'/'+signal)); + if(!response)return reject(new Error('Compiler process exited without result')); + if('error' in response)return reject(new Error(response.error)); + resolve(response.capture); + }); + child.send({fs:{...sources},inputPath,options} satisfies import('./worker').ModelRequest,error=>{if(error)reject(error);}); + }); } export async function compileFigureSource(sources:Readonly>,inputPath:string,resources:FigureResources={},options:CompileOptions={}):Promise { const captured={...sources}; diff --git a/packages/render/test/renderer.test.ts b/packages/render/test/renderer.test.ts index a5dd345..7767d87 100644 --- a/packages/render/test/renderer.test.ts +++ b/packages/render/test/renderer.test.ts @@ -17,3 +17,13 @@ test('failed syntax/import and absent/outside source refuse; independent compila await expect(compileModel({'../outside.d2':'a'},'../outside.d2')).rejects.toThrow('repository-relative'); expect((await compileModel({'model.d2':'a -> b'},'model.d2')).model.relations).toHaveLength(1); },30000); + +// Each call resolves only after its isolated allocation domain exits naturally. +// Keeping each call a separate case preserves the existing test watchdog. +const compilerParentRSS=process.memoryUsage().rss; +test.each(Array.from({length:24},(_,index)=>index))('compiler allocation ownership is reclaimed after call %i',async()=>{ + const capture=await compileModel({'repeat.d2':'a: Producer\nb: Consumer\na -> b: binds'},'repeat.d2',{layout:'elk'}); + expect(capture.model.subjects).toHaveLength(2); + expect(capture.model.relations).toHaveLength(1); + expect(process.memoryUsage().rss-compilerParentRSS).toBeLessThan(256*1024*1024); +}); From 97663d4c6dc456e78e35113b9c47349a321b2b36 Mon Sep 17 00:00:00 2001 From: guangtao Date: Mon, 2 Nov 2026 13:18:57 +0800 Subject: [PATCH 2/2] Close compiler lifetime on IPC failures and prepare patch release --- README.org | 15 +++-- .../10.146-taste-selection-and-reuse.org | 37 +++++++++++ package.json | 2 +- packages/render/README.org | 15 ++++- packages/render/src/compiler-process.ts | 16 ++--- packages/render/src/index.ts | 13 ++-- .../render/test/compiler-lifecycle.test.ts | 62 +++++++++++++++++++ 7 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 packages/render/test/compiler-lifecycle.test.ts diff --git a/README.org b/README.org index 71e6839..86f6049 100644 --- a/README.org +++ b/README.org @@ -137,17 +137,17 @@ Workflow: [[file:docs/90-99-views/90-architecture-views/90.133-capture-preparati ** Development and verification Use the declared Bun version from [[file:.github/workflows/render.yml][render CI]] and its pinned -package/browser dependencies. Run in =packages/render=: +package/browser dependencies. Run from the repository root: #+begin_src sh bun install --frozen-lockfile bunx playwright install chromium bun run typecheck -bun test ./test +bun run test bun run test:export #+end_src -Run the package-owned cases from =packages/render=: +Run the package-owned cases from the same repository root: #+begin_src sh bun run test:examples @@ -160,7 +160,7 @@ Candidate outputs and source/browser receipts are written to =.cache/render/=. Tests include unknown declarations, model/source mutation, invalid relations and ports, hidden Questions, unsupported primitives and failed image loading. The Linux/macOS workflow runs the same entries with the existing resource -bounds. [[https://github.com/hardenedlinux/deterministic-engineering/pull/2/checks][PR checks]] +bounds. [[https://github.com/hardenedlinux/deterministic-engineering/actions/workflows/render.yml][Render CI checks]] are authoritative for the actual tested commit. The [[file:docs/90-99-views/90-architecture-views/90.129-functional-architecture-rendering.svg][render architecture]] and @@ -169,6 +169,13 @@ are distinct current diagrams. Their D2 sources are maintained alongside the SVG and rendered through the measured TS scene API. The active SVGs and PNGs come from the same font-settled scene; the old raw-D2 rendering path has been removed. +The repository-root =package.json= is the Render version and dependency source. +Downstreams install the reviewed Git revision with Bun; Release attachments are +optional distribution. Each D2 compilation owns an ephemeral process and worker; +the Promise settles only after native process close, including IPC failure paths. +No compiler pool or fallback renderer is introduced. See the +[[file:docs/10-19-design/10-architecture/10.146-taste-selection-and-reuse.org][compiler lifecycle and release audit]] for acceptance and downstream recovery. + * Scheme package entrypoints The canonical dependency/prelude declaration is [[file:gerbil.pkg][gerbil.pkg]]. The compiler is Gerbil; package commands own dependency resolution, checkout, manifests, diff --git a/docs/10-19-design/10-architecture/10.146-taste-selection-and-reuse.org b/docs/10-19-design/10-architecture/10.146-taste-selection-and-reuse.org index a958891..75226d2 100644 --- a/docs/10-19-design/10-architecture/10.146-taste-selection-and-reuse.org +++ b/docs/10-19-design/10-architecture/10.146-taste-selection-and-reuse.org @@ -138,3 +138,40 @@ README therefore displays the same renderer's PNG and retains an SVG export link The display test binds the actual PNG receipt, observed subjects/relations, measured readability and captured source hashes; checking only the D2 theme selection did not prevent the original display/content regression. + + +* Compiler allocation lifetime, package release and downstream admission + +PR #5 is rebased onto the Git-installable root package from main. Its change is +a compiler allocation boundary, not a second build system: D2 still owns parsing +and model observations; the existing SDK Worker still owns compilation. A short +native child process owns that worker so process termination can return retained +Bun worker allocations instead of accumulating them in the publishing host. +The host accepts a result only after native process close and zero exit, with +exactly one response. It disconnects IPC on send errors and waits for close before +rejecting. No process-exit call, signal-kill fallback, compiler pool or new public +factory/configuration is introduced. Worker teardown precedes sending the result. + +The IPC channel must remain referenced until teardown and send acknowledgement. +An attempted once-only listener locally caused premature exit without a result; +that failure was retained and the listener remains active until explicit disconnect. +Native lifecycle controls exercise spawn/IPC failures, missing and duplicate results, +nonzero exit and a real successful D2 result. The existing 24 sequential compilations +retain the 256-MiB parent RSS guard. These controls qualify this runtime and workload; +they do not prove arbitrary host heaps or all process-cancellation policies. + +The root =package.json= remains the sole Render version declaration. Release tags, +package archives and notices derive from it. After both platform jobs and REUSE +pass on the exact head, squash PR #5, delete its branch and tag the merged source. +A standard Bun package archive is an optional Release artifact, not Cedar's input. +Cedar selects the merged Git commit in its existing manifest and Bun lockfile. + +Cedar must re-render its captured SDK/D2 sources using the new installed package, +then pass the existing full tests and dependency/worker/artifact recovery controls. +Its job-local initial Bun cache is empty. Recovery removes only node_modules, +reinstalls from the same frozen lock and starts a new process to execute the real +D2 worker and verify existing exports. Missing HTML, changed captured sources or +mixed artifacts must reject; re-rendering cannot stand in for recovery acceptance. +Neither vendor, manual receipts, an additional publisher nor a Release URL is used. +The upstream and downstream CI results qualify their respective revisions separately; +a release tag alone grants neither model publication nor Aitia review authority. diff --git a/package.json b/package.json index e0e6bb7..0f1e637 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "playwright": "1.64.1", "typescript": "7.0.2" }, - "version": "1.0.1", + "version": "1.0.2", "exports": { ".": "./packages/render/src/index.ts", "./opensysml": "./packages/render/src/opensysml.ts", diff --git a/packages/render/README.org b/packages/render/README.org index 8afd59a..3f0c9bb 100644 --- a/packages/render/README.org +++ b/packages/render/README.org @@ -185,7 +185,7 @@ after scene scaling; =minimumTextSize= is the caller's optional CSS-pixel thresh not a global readability score. Font/annotation clipping and detected line overlap refuse; there is no hidden font shrinking or alternate-layout retry. -The source-driven cases live in =packages/render/examples= and run from the parent package with its shared dependency installation. +The source-driven cases live in =packages/render/examples= and run from the repository root with its shared dependency installation. See [[../../docs/10-19-design/10-architecture/10.144-functional-themes-layouts-and-package-examples.org][10.144 implementation contract]] and the separate architecture/Workflow variants. * Declared vector surfaces @@ -276,8 +276,19 @@ archive. Release tags are created only after the package/consumer checks pass. * Compiler allocation lifetime =compileModel= owns one ephemeral compiler process and its internal SDK worker. The result is accepted only after the process closes its worker, disconnects -IPC and exits naturally. No process exit call, signal kill, fallback renderer +IPC and exits naturally. The parent waits for native =close= before either +resolution or rejection; spawn/IPC errors are retained until that boundary. No process exit call, signal kill, fallback renderer or consumer-maintained pool is used. This process boundary releases terminated Bun worker allocation domains that otherwise remain in the long-lived host. Concurrent callers remain independent; the Library imposes no CPU-count clamp. The host still owns cancellation, batch scheduling and browser lifetime. + + +Lifecycle controls observe real native child handles. They cover successful D2 +compilation, synchronous IPC failure, native disconnected IPC, spawn failure, +missing/duplicate results and nonzero exit. The existing 24-call parent RSS guard +remains 256 MiB; caller scheduling and the two-worker browser CI are unchanged. +IPC fault fixtures test protocol admission only; they are not alternate compilers. +The child keeps IPC referenced through worker termination and send acknowledgement, +then disconnects. Removing the last message listener before that point can make +Bun exit before delivering the result, and must not be used as a cleanup shortcut. diff --git a/packages/render/src/compiler-process.ts b/packages/render/src/compiler-process.ts index fa872db..552c096 100644 --- a/packages/render/src/compiler-process.ts +++ b/packages/render/src/compiler-process.ts @@ -6,16 +6,16 @@ import {Worker} from 'node:worker_threads'; import type {ModelRequest,ModelResponse} from './worker'; /** The process owns the compiler worker's native allocation domain. Bun may * retain terminated worker heaps until process exit; wait for natural exit. */ +// Keep IPC referenced until the worker is joined and the send callback completes. process.on('message',async(request:ModelRequest)=>{ - let worker:Worker|undefined; let response:ModelResponse; try{ - worker=new Worker(new URL('./worker.ts',import.meta.url),{workerData:request}); - response=await new Promise((resolve,reject)=>{ - worker!.once('message',resolve);worker!.once('error',reject); - worker!.once('exit',code=>reject(new Error('Compiler worker exited before result: '+code))); - }); + const worker=new Worker(new URL('./worker.ts',import.meta.url),{workerData:request}); + try{response=await new Promise((resolve,reject)=>{ + worker.once('message',resolve);worker.once('error',reject); + worker.once('exit',code=>reject(new Error('Compiler worker exited before result: '+code))); + });}finally{await worker.terminate();} }catch(error:unknown){response={error:error instanceof Error?error.message:String(error)};} - finally{if(worker)await worker.terminate();} - process.send!(response,error=>{if(error)process.exitCode=1;process.disconnect?.();}); + if(!process.connected){process.exitCode=1;return;} + process.send!(response,error=>{if(error)process.exitCode=1;if(process.connected)process.disconnect?.();}); }); diff --git a/packages/render/src/index.ts b/packages/render/src/index.ts index 6d59f95..df4ef29 100644 --- a/packages/render/src/index.ts +++ b/packages/render/src/index.ts @@ -10,7 +10,6 @@ export {composeHeading,type HeadingSpec} from './heading'; export {composeModelView,graphFromModel,modelDiagram} from './model-view'; import {isDeepStrictEqual} from 'node:util'; import {fork} from 'node:child_process'; -import {fileURLToPath} from 'node:url'; import type {CompileOptions} from '@terrastruct/d2'; import {relativeSource} from './source/catalog'; import {readD2Figure,type FigureResources} from './d2-figure'; @@ -31,18 +30,22 @@ export async function compileModel(sources:Readonly>,input relativeSource(inputPath); if(!Object.hasOwn(sources,inputPath))throw new Error('Missing model source'); for(const[file,text]of Object.entries(sources)){relativeSource(file);if(typeof text!=='string')throw new Error('Invalid model source');} - const child=fork(fileURLToPath(new URL('./compiler-process.ts',import.meta.url)),[],{execPath:process.execPath,stdio:['ignore','inherit','inherit','ipc']}); + const child=fork(new URL('./compiler-process.ts',import.meta.url),[],{execPath:process.execPath,stdio:['ignore','inherit','inherit','ipc']}); return await new Promise((resolve,reject)=>{ let response:import('./worker').ModelResponse|undefined; - child.once('error',reject); + let failure:Error|undefined; + const fail=(error:Error)=>{failure??=error;if(child.connected)child.disconnect();}; + child.on('error',fail); child.on('message',(value:import('./worker').ModelResponse)=>{if(response){response={error:'Duplicate compiler result'};}else response=value;}); - child.once('exit',(code,signal)=>{ + child.once('close',(code,signal)=>{ + if(failure)return reject(failure); if(code!==0||signal)return reject(new Error('Compiler process exit: '+code+'/'+signal)); if(!response)return reject(new Error('Compiler process exited without result')); if('error' in response)return reject(new Error(response.error)); resolve(response.capture); }); - child.send({fs:{...sources},inputPath,options} satisfies import('./worker').ModelRequest,error=>{if(error)reject(error);}); + try{child.send({fs:{...sources},inputPath,options} satisfies import('./worker').ModelRequest,error=>{if(error)fail(error);});} + catch(error:unknown){fail(error instanceof Error?error:new Error(String(error)));} }); } export async function compileFigureSource(sources:Readonly>,inputPath:string,resources:FigureResources={},options:CompileOptions={}):Promise { diff --git a/packages/render/test/compiler-lifecycle.test.ts b/packages/render/test/compiler-lifecycle.test.ts new file mode 100644 index 0000000..3e9bffd --- /dev/null +++ b/packages/render/test/compiler-lifecycle.test.ts @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2026 hardenedlinux Linux Team and Contributors +// SPDX-License-Identifier: LGPL-2.1-or-later + +import {test,expect,spyOn} from 'bun:test'; +import * as processes from 'node:child_process'; +import {mkdtemp,writeFile,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import {compileModel} from '../src'; +const fork=processes.fork; +const compile=()=>compileModel({'case.d2':'a -> b'},'case.d2'); + +// Observe the existing native boundary, without a production factory or test hook. +function observe(create:(...args:Parameters)=>processes.ChildProcess=(...args)=>fork(...args)){ + const children:processes.ChildProcess[]=[],closed=new Set(); + const spy=spyOn(processes,'fork').mockImplementation((path:string|URL,args?:readonly string[]|processes.ForkOptions,options?:processes.ForkOptions)=>{ + const child=Array.isArray(args)||args===undefined?create(path,args,options):create(path,undefined,args as processes.ForkOptions);children.push(child);child.once('close',()=>closed.add(child));return child; + }); + return{children,closed,restore:()=>spy.mockRestore()}; +} +function reclaimed(observed:ReturnType){ + expect(observed.children.length).toBe(1);expect(observed.closed.size).toBe(1); + expect(observed.children[0]!.connected).toBe(false); +} + +test('successful native D2 capture is returned only after its process closes',async()=>{ + const observed=observe(); + try{expect((await compile()).model.relations.length).toBe(1);reclaimed(observed);expect(observed.children[0]!.exitCode).toBe(0);} + finally{observed.restore();} +},30000); + +test('synchronous IPC failure disconnects and awaits the actual child before rejection',async()=>{ + const observed=observe((...args)=>{const child=fork(...args);spyOn(child,'send').mockImplementation(()=>{throw new Error('Synthetic IPC serialization failure');});return child;}); + try{await expect(compile()).rejects.toThrow('Synthetic IPC serialization failure');reclaimed(observed);} + finally{observed.restore();} +},30000); + +test('native disconnected IPC rejects only after the child closes',async()=>{ + const observed=observe((...args)=>{const child=fork(...args);child.disconnect();return child;}); + try{await expect(compile()).rejects.toThrow();reclaimed(observed);} + finally{observed.restore();} +},30000); + +test('native spawn failure is rejected without claiming compiler output',async()=>{ + const directory=await mkdtemp(join(tmpdir(),'render-no-executable-')); + const observed=observe((path,args,options)=>fork(path,args,{...options,execPath:join(directory,'missing-bun')})); + try{await expect(compile()).rejects.toThrow();reclaimed(observed);} + finally{observed.restore();await rm(directory,{recursive:true,force:true});} +},30000); + +// These are IPC/exit fault fixtures, not alternate compilers or test runners. +for(const fault of [ + {name:'missing result',body:'process.disconnect();',message:'without result'}, + {name:'nonzero exit',body:'process.exitCode=7;process.disconnect();',message:'Compiler process exit: 7'}, + {name:'duplicate result',body:'process.send({error:"first"},()=>process.send({error:"second"},()=>process.disconnect()));',message:'Duplicate compiler result'}, +])test('native '+fault.name+' cannot be accepted as a compiler capture',async()=>{ + const directory=await mkdtemp(join(tmpdir(),'render-ipc-fault-')),file=join(directory,'fixture.ts'); + await writeFile(file,'process.once("message",()=>{'+fault.body+'});'); + const observed=observe((_,args,options)=>fork(file,args,options)); + try{await expect(compile()).rejects.toThrow(fault.message);reclaimed(observed);} + finally{observed.restore();await rm(directory,{recursive:true,force:true});} +},30000);