diff --git a/.github/workflows/reuse-quality.yml b/.github/workflows/reuse-quality.yml index c6b8217..d72f00d 100644 --- a/.github/workflows/reuse-quality.yml +++ b/.github/workflows/reuse-quality.yml @@ -15,13 +15,16 @@ jobs: node-version-file: .nvmrc cache: 'npm' - # google-chrome-stable is preinstalled on the runner, gifsicle is not - name: Install gifsicle run: sudo apt-get update && sudo apt-get install -y --no-install-recommends gifsicle - name: Install dependencies run: npm ci + # the browser of the tests, the version that matches puppeteer + - name: Install chrome-headless-shell + run: npx puppeteer browsers install chrome-headless-shell + - name: Lint run: npm run lint diff --git a/.gitignore b/.gitignore index 5ccbe95..842bb3d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ docs/superpowers/ .superpowers/ .playwright-mcp/ +benchmark/results/* +!benchmark/results/.gitkeep diff --git a/AGENTS.md b/AGENTS.md index 9e437c9..c240972 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,10 +2,10 @@ ## Project overview -Capture renders Web pages with puppeteer (google-chrome) and returns screenshots (png, jpg, +Capture renders Web pages with puppeteer (chrome-headless-shell) and returns screenshots (png, jpg, animated gif through gifsicle) or prints (pdf, html). API routes in `api/routers/capture.ts`, -browser contexts pool in `api/utils/page.ts`, config in `config/` (types generated in -`config/type/` by `npm run build-types`). +browser and capture slots in `api/utils/page.ts` (one fresh incognito context per capture), config +in `config/` (types generated in `config/type/` by `npm run build-types`). Node.js v24, TypeScript run directly by Node's type stripping (no build step, erasable syntax only), Express 5. @@ -42,7 +42,8 @@ Branch names: letters, digits, `.`, `_`, `-`, no `/`, and `capture_` at ## Testing `node:test` suites in `test-it/`, they start the API in-process on `TEST_API_PORT` (`.env`) and -need google-chrome-stable and gifsicle on the machine. +need gifsicle on the machine and puppeteer's chrome-headless-shell +(`npx puppeteer browsers install chrome-headless-shell`). ```bash npm test # all tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3610ab7..3fc1b6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,8 +4,9 @@ - Node.js, the version of `.nvmrc` (start zellij from a shell on it: the panes inherit its PATH) - zellij >= 0.44 -- google-chrome-stable (`/usr/bin/google-chrome-stable`, the `executablePath` of `config/default.cjs`) - and gifsicle (animated captures), installed on your system +- gifsicle (animated captures), installed on your system. The browser is chrome-headless-shell: + the image installs Debian's package, development and tests use the build downloaded by puppeteer + (`npx puppeteer browsers install chrome-headless-shell`, the version that matches puppeteer) ## Setup @@ -13,10 +14,11 @@ The dev environment is the shared one of the data-fair services (`@data-fair/lib session per checkout, described by `.zellij.kdl` and `dev/dev.json`. No containers: in development capture runs without authentication (no `privateDirectoryUrl`). -1. Install dependencies (puppeteer's own browser download is not needed), then write `.env` (free - ports drawn for this checkout) and build the types: +1. Install dependencies and chrome-headless-shell, then write `.env` (free ports drawn for this + checkout) and build the types: ```bash PUPPETEER_SKIP_DOWNLOAD=true npm ci + npx puppeteer browsers install chrome-headless-shell npx --no df-dev setup ``` diff --git a/Dockerfile b/Dockerfile index 717ba10..b225ca7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,14 +15,12 @@ FROM base AS nativedeps ARG TARGETARCH RUN echo "Building for architecture $TARGETARCH" -# Install chrome and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +# Install chromium-headless-shell (Debian's build of the "old" headless Chromium, amd64 and arm64, +# security updates with the rest of the distribution) and fonts to support major charsets +# (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y wget gnupg ca-certificates && \ - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg && \ - sh -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] https://dl-ssl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \ - apt-get update && \ - apt-get install -y google-chrome-stable gifsicle fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst-one fonts-freefont-ttf libxss1 dbus dbus-x11 --no-install-recommends && \ + apt-get install -y chromium-headless-shell gifsicle wget ca-certificates fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst-one fonts-freefont-ttf dbus dbus-x11 --no-install-recommends && \ service dbus start ENV DBUS_SESSION_BUS_ADDRESS=autolaunch: @@ -37,9 +35,10 @@ RUN case "$TARGETARCH" in \ wget -q -O /usr/local/bin/dumb-init "https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_$DUMB_INIT_ARCH" && \ chmod +x /usr/local/bin/dumb-init -# cleanup (wget stays: google-chrome-stable depends on it, purging it removes chrome) -RUN apt-get clean -RUN apt-get purge -y --auto-remove gnupg apt-transport-https +# cleanup: wget was only needed to download dumb-init +RUN apt-get purge -y --auto-remove wget && \ + test -x /usr/lib/chromium/chromium-headless-shell && \ + apt-get clean && rm -rf /var/lib/apt/lists/* # skip the browser download when installing puppeteer ENV PUPPETEER_SKIP_DOWNLOAD=true diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 8a57ba2..d7a9638 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -13,11 +13,21 @@ import type { PaperFormat, PDFOptions } from 'puppeteer' const debug = debugModule('capture') -// /inspect renders are expensive, a small number of them run at the same time -const inspectSemaphore = createSemaphore(config.inspectConcurrency) +// /inspect renders are expensive, a small number of them run at the same time (each one then takes +// a capture slot like the other routes) +const inspectSemaphore = createSemaphore(config.inspectConcurrency, config.queue) export const router = express.Router() +// aborted when the client closes the request before its response is sent (a proxy timeout, a +// user who left): the capture is dropped instead of being rendered for nobody +const clientSignal = (res: Response) => { + const controller = new AbortController() + if (res.destroyed) controller.abort() + else res.on('close', () => { if (!res.writableFinished) controller.abort() }) + return controller.signal +} + async function auth (req: Request, res: Response, next: NextFunction) { if (config.privateDirectoryUrl) { const user = reqSession(req).user @@ -113,18 +123,27 @@ router.get('/screenshot', auth, async (req, res, next) => { if (typeof req.query.filename === 'string') res.attachment(req.query.filename.replace('.png', '.gif')) res.send(buffer) } else { - const buffer = await page.screenshot({ fullPage: req.query.height === 'auto' }) - debug(`[${target}] png screenshot is taken`) - timer.type = 'png' + // a gif was asked from a page that does not support animation: a png is returned + const imageType = (type === 'jpg' || type === 'jpeg') ? 'jpeg' : 'png' + const extension = imageType === 'jpeg' ? type : 'png' + const fullPage = req.query.height === 'auto' + // png encoding runs on the browser's main thread, shared by all captures: the large + // captures are encoded for speed (cf benchmark/FINDINGS.md) + const optimizeForSpeed = imageType === 'png' && (fullPage || width * height > config.pngFastEncodingPixels) + const buffer = await page.screenshot({ fullPage, type: imageType, optimizeForSpeed }) + debug(`[${target}] ${imageType} screenshot is taken`) + timer.type = imageType if (req.query.timer === 'true') { res.send(timer.times) } else { - res.type(type) - if (typeof req.query.filename === 'string') res.attachment(req.query.filename.replace('.gif', '.' + type)) + res.type(extension) + if (typeof req.query.filename === 'string') res.attachment(req.query.filename.replace(/\.(gif|png|jpe?g)$/, '.' + extension)) res.send(buffer) } } - } + }, + undefined, + clientSignal(res) ) timer.finish() }) @@ -191,7 +210,9 @@ router.get('/print', auth, async (req, res, next) => { if (typeof req.query.filename === 'string') res.attachment(req.query.filename) res.send(buffer.toString()) } - } + }, + undefined, + clientSignal(res) ) timer.finish() }) @@ -246,9 +267,9 @@ router.get('/inspect', async (req, res) => { const timer = createTimer(req.originalUrl, 'inspect') let collector: ReturnType | undefined + const signal = clientSignal(res) + // the client may leave while waiting for a slot: the signal drops the render await inspectSemaphore.run(async () => { - // the client may have left while waiting for a slot, no need to render for nobody - if (req.socket.destroyed || res.destroyed) return await pageUtils.withPage( target, lang, @@ -260,22 +281,25 @@ router.get('/inspect', async (req, res) => { `Failed to inspect page "${target}" before timeout`, async ({ page, ready }) => { const report = await collector!.finish(ready) - let buffer + // the base64 string of the browser as is: decoding it to bytes to encode it back to base64 + // would only copy megabytes on the main thread + let data if (fullPage) { const height = await page.evaluate(() => document.documentElement.scrollHeight) - buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) + data = await page.screenshot({ encoding: 'base64', captureBeyondViewport: true, optimizeForSpeed: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) } else { - buffer = await page.screenshot() + data = await page.screenshot({ encoding: 'base64' }) } timer.type = 'inspect' - res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) + res.json({ image: { data, mediaType: 'image/png' }, text: reportText(report), report }) }, { onPage: page => { collector = createReportCollector(page, target) }, declaredErrorUrl, onDeclaredError: body => collector?.declareError(body) - } + }, + signal ) - }) + }, signal).catch(err => { throw pageUtils.slotError(err) }) timer.finish() }) diff --git a/api/server.ts b/api/server.ts index a90d464..6208cfa 100644 --- a/api/server.ts +++ b/api/server.ts @@ -7,6 +7,7 @@ import { createHttpTerminator } from 'http-terminator' import app from './app.ts' import * as pageUtils from './utils/page.ts' import * as gifsicle from './utils/gifsicle.ts' +import * as animation from './utils/animation.ts' import config from '#config' const server = createServer(app) @@ -38,4 +39,5 @@ export const stop = async () => { await httpTerminator.terminate() if (config.observer.active) await stopObserver() await pageUtils.stop() + await animation.stop() } diff --git a/api/threads/gif-encode.ts b/api/threads/gif-encode.ts new file mode 100644 index 0000000..49dd483 --- /dev/null +++ b/api/threads/gif-encode.ts @@ -0,0 +1,38 @@ +// Runs in a piscina worker thread: decodes the png frames of an animated capture and encodes them +// as a gif. Both are pure JS and CPU heavy (NeuQuant quantization), on the main thread they +// blocked every other request of the service for seconds (cf benchmark/FINDINGS.md). +import { promisify } from 'node:util' +import GifEncoder from 'gif-encoder' +import getPixelsCb from 'get-pixels' +import { move } from 'piscina' + +const getPixels = promisify(getPixelsCb) + +export type GifEncodeTask = { width: number, height: number, frameRate: number, frames: Uint8Array[] } + +export default async ({ width, height, frameRate, frames }: GifEncodeTask) => { + const gif = new GifEncoder(width, height, { highWaterMark: 64 * 1024 * 1024 }) + const chunks: Buffer[] = [] + gif.on('data', (chunk: Buffer) => chunks.push(chunk)) + // finish() pushes the end of the stream, the last chunks are emitted asynchronously + const ended = new Promise(resolve => gif.once('end', () => resolve())) + gif.setFrameRate(frameRate) + gif.writeHeader() + for (const frame of frames) { + // get-pixels switches on Buffer.isBuffer to tell raw image data from a file path + const pixels = await getPixels(Buffer.from(frame.buffer, frame.byteOffset, frame.byteLength), 'image/png') + gif.addFrame(pixels.data) + } + gif.finish() + await ended + // the chunks are joined in an ArrayBuffer of our own, then transferred to the main thread, not + // copied. Buffer.concat would not do: below half of Buffer.poolSize it returns a slice of the + // shared Buffer pool, which cannot be transferred (DataCloneError) + const result = new Uint8Array(chunks.reduce((size, chunk) => size + chunk.byteLength, 0)) + let offset = 0 + for (const chunk of chunks) { + result.set(chunk, offset) + offset += chunk.byteLength + } + return move(result) +} diff --git a/api/utils/animation.ts b/api/utils/animation.ts index bc2c7aa..f0522f6 100644 --- a/api/utils/animation.ts +++ b/api/utils/animation.ts @@ -1,29 +1,38 @@ -import { file } from 'tmp-promise' +import path from 'node:path' +import { Piscina } from 'piscina' import config from '#config' -import fs from 'node:fs' -import { promisify } from 'node:util' -import { pipeline } from 'node:stream/promises' -import GifEncoder from 'gif-encoder' -import getPixelsCb from 'get-pixels' import { optimizeGif } from './gifsicle.ts' import debugModule from 'debug' +import log from '@data-fair/lib-node/log.js' import { type Page } from 'puppeteer' +import type { GifEncodeTask } from '../threads/gif-encode.ts' const debug = debugModule('capture') -const getPixels = promisify(getPixelsCb) +// the gif encoding is CPU heavy pure JS, it runs in worker threads so that it does not block the +// service. maxThreads also bounds the number of gifs encoded at the same time +export const gifEncodePiscina = new Piscina({ + filename: path.resolve(import.meta.dirname, '../threads/gif-encode.ts'), + minThreads: 0, + idleTimeout: 10 * 60 * 1000, + maxThreads: config.gifThreads +}) +// a pool emits 'error' when a thread dies with no task in flight; without a listener it crashes the process +gifEncodePiscina.on('error', (err) => log.error('gif encoding worker error', { code: 'piscina-gif-encode', err })) + +// a typed array alone in its ArrayBuffer, so that the ArrayBuffer can be transferred to a worker. +// puppeteer returns Buffers decoded from base64, the small ones are slices of the shared Buffer +// pool (not transferable, and other buffers live in it): those are copied, the others kept +const ownBuffer = (frame: Uint8Array) => { + if (frame.byteOffset === 0 && frame.byteLength === frame.buffer.byteLength && !(frame.buffer instanceof SharedArrayBuffer)) return frame + return new Uint8Array(frame) +} export const capture = async (target: string, page: Page, width: number, height: number) => { let stopped = false - const gif = new GifEncoder(width, height) - gif.setFrameRate(15) // 15fps seams like a good compromise for a gif - gif.writeHeader() - const { path, cleanup } = await file({ postfix: '.gif' }) - const pipelinePromise = pipeline(gif, fs.createWriteStream(path)) - let i = 0 - while (!stopped && i < config.maxAnimationFrames) { - i++ - if (i % 15 === 0) debug(`[${target}] ${i} frames taken`) + const frames: Uint8Array[] = [] + while (!stopped && frames.length < config.maxAnimationFrames) { + if (frames.length % 15 === 0) debug(`[${target}] ${frames.length} frames taken`) stopped = await page.evaluate(() => { // @ts-ignore return window.animateCaptureFrame() @@ -35,16 +44,14 @@ export const capture = async (target: string, page: Page, width: number, height: new Promise(resolve => setTimeout(resolve, config.screenshotTimeout)) ]) if (!buffer) throw new Error(`Failed to capture animation frame of page "${target}" before timeout`) - // puppeteer returns a plain Uint8Array, but get-pixels switches on Buffer.isBuffer to tell - // raw image data from a file path, and would treat the frame as a filename otherwise - const frame = Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength) - const pixels = await getPixels(frame, 'image/png') - gif.addFrame(pixels.data) + frames.push(ownBuffer(buffer)) } - gif.finish() - await pipelinePromise + debug(`[${target}] ${frames.length} frames taken, encode the gif`) + // 15fps seams like a good compromise for a gif + // the frames are transferred to the worker, not copied: they are not usable here afterwards + const raw = await gifEncodePiscina.run({ width, height, frameRate: 15, frames }, { transferList: frames.map(f => f.buffer as ArrayBuffer) }) debug(`[${target}] gif screenshot is taken`) - const rawBuffer = await fs.promises.readFile(path) - cleanup() - return await optimizeGif(rawBuffer, 2) + return await optimizeGif(Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength), 2) } + +export const stop = () => gifEncodePiscina.destroy() diff --git a/api/utils/metrics.ts b/api/utils/metrics.ts index 8cac84d..0f8db4b 100644 --- a/api/utils/metrics.ts +++ b/api/utils/metrics.ts @@ -14,27 +14,27 @@ export const tasks = new client.Histogram({ }) export const contextsCleanups = new client.Counter({ name: 'df_capture_contexts_cleanups', - help: 'Number of times a browser context was cleaned up for later reuse' + help: 'Number of browser contexts closed after their capture (one fresh context per capture)' }) export const acquireContextPending = new client.Gauge({ name: 'df_capture_acquire_context_pending', - help: 'Number of tasks waiting to acquire a browser context', + help: 'Number of captures waiting for a slot', async collect () { - this.set(pageUtils.contextPool && pageUtils.contextPool().pending) + this.set(pageUtils.captureSlots.pending()) } }) export const acquireContextMax = new client.Gauge({ name: 'df_capture_acquire_context_max', - help: 'Max number of browser contexts that can be borrowed by tasks', + help: 'Max number of captures running at the same time', async collect () { - this.set(pageUtils.contextPool && pageUtils.contextPool().max) + this.set(pageUtils.captureSlots.max) } }) export const acquireContextBorrowed = new client.Gauge({ name: 'df_capture_acquire_context_borrowed', - help: 'Number of browser contexts currently borrowed by a task', + help: 'Number of captures running', async collect () { - this.set(pageUtils.contextPool && pageUtils.contextPool().borrowed) + this.set(pageUtils.captureSlots.active()) } }) diff --git a/api/utils/page.ts b/api/utils/page.ts index 47dab1f..14f44b4 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -1,11 +1,11 @@ import config from '#config' import puppeteer, { type Viewport, type Browser, type BrowserContext, type Page, type CookieData } from 'puppeteer' -import genericPool, { type Factory, type Pool } from 'generic-pool' import { contextsCleanups } from './metrics.ts' import debugModule from 'debug' import log from '@data-fair/lib-node/log.js' import { Timer } from './timer.ts' import { httpError } from '@data-fair/lib-express' +import { createSemaphore, SemaphoreError } from './semaphore.ts' import type { Ready } from './report.ts' const debug = debugModule('capture') @@ -13,38 +13,20 @@ const debug = debugModule('capture') let _closed = false let _browser: Browser -let _contextPool: Pool -export function contextPool () { - if (!_contextPool) throw new Error('contextPool not initialized') - return _contextPool -} +// the captures running at the same time, each one in its own incognito context. The others wait +// in a bounded queue (config.queue) +export const captureSlots = createSemaphore(config.concurrency, config.queue) export const start = async () => { log.info('init puppeteer browser', { launchOptions: config.puppeteerLaunchOptions }) _browser = await puppeteer.launch(config.puppeteerLaunchOptions) - const contextFactory: Factory = { - async create () { - // create pages in incognito contexts so that cookies are not shared - // each context is used sequentially only because of cookies or other states conflicts - return _browser.createBrowserContext({}) - }, - async destroy (context) { - await context.close() - } - } - _contextPool = genericPool.createPool(contextFactory, { max: config.concurrency }) // auto reconnection, cf https://github.com/GoogleChrome/puppeteer/issues/4428 + // the captures running in the lost browser fail, the next ones use the new browser _browser.on('disconnected', async () => { if (!_closed) { log.error('browser was disconnected, reconnect', { code: 'browser-disconnected' }) - try { - await _contextPool.drain() - _contextPool.clear() - } catch (err) { - log.error('failure while draining the replaced contexts pool', { code: 'contexts-pool-drain', err }) - } start().catch(err => log.error('failure while restarting the browser', { code: 'browser-restart', err })) } }) @@ -53,10 +35,6 @@ export const start = async () => { export const stop = async () => { _closed = true if (_browser) await _browser.close() - if (_contextPool) { - await _contextPool.drain() - _contextPool.clear() - } } const debugPage = (page: Page) => { @@ -166,10 +144,47 @@ async function openInPage ( } export const promiseTimeout = async (promise: Promise, ms: number, msg = 'timeout') => { - return Promise.race([ - promise, - new Promise((resolve, reject) => setTimeout(() => reject(new Error(msg)), ms)) - ]) + let timer: NodeJS.Timeout | undefined + try { + return await Promise.race([ + promise, + new Promise((resolve, reject) => { timer = setTimeout(() => reject(new Error(msg)), ms) }) + ]) + } finally { + clearTimeout(timer) + } +} + +// the client left (proxy timeout, user gone): the render is stopped, nobody would receive it. +// 499 is nginx's "client closed request", below 500 it is not logged as a failure +const clientGone = () => httpError(499, 'client closed the request') + +// rejects as soon as the signal aborts, the promise keeps running but its page gets closed +const unlessAborted = async (promise: Promise, signal: AbortSignal | undefined) => { + if (!signal) return promise + if (signal.aborted) throw clientGone() + let onAbort: (() => void) | undefined + try { + return await Promise.race([ + promise, + new Promise((resolve, reject) => { + onAbort = () => reject(clientGone()) + signal.addEventListener('abort', onAbort, { once: true }) + }) + ]) + } finally { + if (onAbort) signal.removeEventListener('abort', onAbort) + } +} + +// a full queue or a long wait for a slot mean an overloaded service: better a quick 503 that the +// caller can retry than a response after it has given up +export const slotError = (err: unknown) => { + if (err instanceof SemaphoreError) { + if (err.code === 'aborted') return clientGone() + return httpError(503, 'the capture service is overloaded, retry later') + } + return err } export const withPage = async ( @@ -182,83 +197,92 @@ export const withPage = async ( timer: Timer, callbackTimeoutMsg: string, callback: (result: OpenPageResult) => Promise, - inspect?: InspectOptions) => { + inspect?: InspectOptions, + // aborted when the client closes the request + signal?: AbortSignal) => { if (target.includes('capture-test-error=true')) { await new Promise(resolve => setTimeout(resolve, 1000)) throw new Error('forced error trigger') } - let context, page, openSuccess try { - debug(`[${target}] use incognito context from pool`) - let pageAttempts = 0 - while (!page) { - pageAttempts++ - context = await _contextPool.acquire() + await captureSlots.run(async () => { timer.step('acquire-context') + // a fresh incognito context per capture, closed after it: nothing of a capture (cookies, + // localStorage, IndexedDB, cache, service workers) can leak into the next one, whatever + // the user it is made for. It costs a few milliseconds + const context = await _browser.createBrowserContext() try { - page = await context.newPage() - } catch (err) { - if (pageAttempts === config.concurrency) { - throw err - } else { - debug(`[${target}] delete incognito context from pool because it couldn't be used to open a page`) - await _contextPool.destroy(context) - } - } - } - debugPage(page) - inspect?.onPage?.(page) - timer.step('newPage') + const page = await context.newPage() + // the console listener costs 2 CDP round trips per argument of every console message, on + // the main thread of the service: only paid when the debug output is enabled (DEBUG=capture) + if (debug.enabled) debugPage(page) + inspect?.onPage?.(page) + timer.step('newPage') - const result = await promiseTimeout( - openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), - config.screenshotTimeout * 2, - `Failed to open "${target}" in context before timeout` - ) - timer.step('open-in-page') - openSuccess = true - await promiseTimeout( - callback(result), - result.animationActivated ? config.screenshotTimeout * 2 : config.screenshotTimeout, - callbackTimeoutMsg - ) - } finally { - if (page) await safeCleanContext(page, !!openSuccess) + const result = await unlessAborted(promiseTimeout( + openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), + config.screenshotTimeout * 2, + `Failed to open "${target}" in context before timeout` + ), signal) + timer.step('open-in-page') + await unlessAborted(promiseTimeout( + callback(result), + result.animationActivated ? config.screenshotTimeout * 2 : config.screenshotTimeout, + callbackTimeoutMsg + ), signal) + } finally { + await closeContext(context) + } + }, signal) + } catch (err) { + throw slotError(err) } timer.step('close-page') } -const cleanContext = async (page: Page) => { - // always empty cookies to prevent inheriting them in next use of the context - const context = page.browserContext() - const cookies = await context.cookies() - // deleting them in a single call spares one CDP round trip per cookie, and this - // runs inside a 2s timeout that a session's worth of cookies could otherwise eat - if (cookies.length) await context.deleteCookie(...cookies) - await page.close() +const closeContext = async (context: BrowserContext) => { + try { + await promiseTimeout(context.close(), 5000, 'timed out while closing a browser context') + contextsCleanups.inc() + } catch (err) { + // the browser may be gone (disconnected), or stuck: nothing more to do for this capture + log.warn('failed to close a browser context', { code: 'context-close', err }) + } } -const safeCleanContext = async (page: Page, openSuccess: boolean) => { - if (openSuccess) { - try { - await promiseTimeout( - cleanContext(page), - 2000, - 'timed out while cleaning page context' - ) - await _contextPool.release(page.browserContext()) - contextsCleanups.inc() - } catch (err) { - log.warn('failed to clean page properly, do not reuse this context', { code: 'page-clean', err }) - await _contextPool.destroy(page.browserContext()) - } - } else { - await _contextPool.destroy(page.browserContext()) - } +// injected before any script of the page: records the last time the page did visible work, read +// by waitForQuiet. A copy of benchmark/lib/capture.ts, where the strategy was measured +const installActivityTracker = () => { + const w = window as any + w.__captureLastActivity = performance.now() + const mark = () => { w.__captureLastActivity = performance.now() } + const raf = window.requestAnimationFrame.bind(window) + window.requestAnimationFrame = (cb) => raf((t) => { mark(); cb(t) }) + new MutationObserver(mark).observe(document, { subtree: true, childList: true, attributes: true, characterData: true }) } -// quite complex strategy to wait for the page to be ready for capture. -// it can either explitly call a triggerCapture function or we wait for idle network + 1s +// resolves when the page has been visually quiet for quietTime (no DOM mutation, no animation frame, +// no running finite animation, no font loading), or after maxSettleTime. Endless animations +// (spinners) are ignored, they never settle +const waitForQuiet = (page: Page) => page.evaluate((quietTime, maxSettleTime) => new Promise(resolve => { + const start = performance.now() + const check = () => { + const now = performance.now() + const busy = [] + if (now - ((window as any).__captureLastActivity ?? 0) < quietTime) busy.push('activity') + if (document.getAnimations().some(a => a.playState === 'running' && a.effect?.getTiming().iterations !== Infinity)) busy.push('animations') + if (document.fonts.status === 'loading') busy.push('fonts') + if (!busy.length) return resolve('quiet') + if (now - start >= maxSettleTime) return resolve('max settle time, busy with ' + busy.join(' and ')) + setTimeout(check, 50) + } + check() +}), config.wait.quietTime, config.wait.maxSettleTime) + +// the strategy to wait for the page to be ready for capture. +// the page can explicitly call a triggerCapture function, this is the fast and reliable way. +// otherwise we wait for the network to be idle, then for the page to be visually quiet (cf +// benchmark/FINDINGS.md for the measures behind this strategy) const waitForPage = async ( page: Page, target: string, @@ -269,6 +293,7 @@ const waitForPage = async ( let captureTriggered = false let timeoutReached = false let animationActivated = false + let settledBy = '' const triggerCapture = new Promise(resolve => page.exposeFunction('triggerCapture', (animationSupported?: boolean) => { captureTriggered = true animationActivated = !!(animate && animationSupported) @@ -276,18 +301,25 @@ const waitForPage = async ( // return animate to the page inside the capture return animate })) + await page.evaluateOnNewDocument(installActivityTracker) try { // wait for network inactivity, but it can be interrupted if triggerCapture is called - const waitOptions = { waitUntil: 'networkidle0' as const, timeout: config.screenshotTimeout } - await Promise.race([page.goto(target, waitOptions), triggerCapture]) + // puppeteer's own network idle: networkidle0 is Chrome's lifecycle event, its idleness + // detector fires 0.5 to 1.5s after the last request depending on the page's activity + const deadline = Date.now() + config.screenshotTimeout + const networkIdle = page.goto(target, { waitUntil: 'load', timeout: config.screenshotTimeout }) + .then(() => page.waitForNetworkIdle({ idleTime: config.wait.networkIdleTime, timeout: Math.max(1, deadline - Date.now()) })) + // the race may be won by triggerCapture, a later rejection (page closed) must not go unhandled + networkIdle.catch(() => {}) + await Promise.race([networkIdle, triggerCapture]) if (captureTriggered) { timer.step('wait1-capture-triggered') debug(`[${target}] capture was expicitly triggered by window.triggerCapture`) } else { timer.step('wait1-network-idle') - debug(`[${target}] network was idle during 500ms`) + debug(`[${target}] network was idle during ${config.wait.networkIdleTime}ms`) } } catch (err: any) { if (err.name !== 'TimeoutError') throw err @@ -302,9 +334,6 @@ const waitForPage = async ( // we are done here, capture was already explicitly triggered or we already waited for a long time } else { // network was found to be idle, adapt the wait strategy based on some meta - // Adapt the wait strategy based on the x-capture meta - - // Adapt the wait strategy based on the x-capture meta let captureDelayMeta try { captureDelayMeta = await page.$eval('head > meta[name=\'df:capture-delay\']', el => el.content) @@ -323,6 +352,7 @@ const waitForPage = async ( debug(`[${target}] capture was expicitly triggered by window.triggerCapture`) } else { timer.step('wait2-delay') + settledBy = `the df:capture-delay meta (${delay}ms)` debug(`[${target}] delay of ${delay / 1000} seconds was reached`) } } else { @@ -349,12 +379,27 @@ const waitForPage = async ( timeoutReached = true } } else { - debug(`[${target}] wait 1000ms more after idle network for safety`) - await new Promise(resolve => setTimeout(resolve, 1000)) - timer.step('wait2-1s') + // the page may still render after its last response (charts animations, vue rendering..) + const quiet = await waitForQuiet(page) + if (captureTriggered) { + timer.step('wait2-capture-triggered') + } else { + timer.step('wait2-settle') + settledBy = quiet === 'quiet' ? 'network idle then visual quiet' : 'network idle then ' + quiet + debug(`[${target}] settled: ${settledBy}`) + } } } } + if (!captureTriggered) { + // the heuristics are slower and less reliable than an explicit signal, this guides the + // applications that should call window.triggerCapture when they are rendered + log.warn('captured page did not call window.triggerCapture', { + code: 'capture-no-trigger', + target, + settledBy: timeoutReached ? `the timeout (${config.screenshotTimeout}ms)` : settledBy + }) + } const ready: Ready = captureTriggered ? 'triggered' : timeoutReached ? 'timeout' : 'idle' return { animationActivated, ready } } diff --git a/api/utils/semaphore.ts b/api/utils/semaphore.ts index 49bdbcd..2769395 100644 --- a/api/utils/semaphore.ts +++ b/api/utils/semaphore.ts @@ -1,17 +1,51 @@ // Limits how many async functions run at the same time, the others wait in FIFO order. -export const createSemaphore = (max: number) => { +// The queue can be bounded (maxWaiting), the wait limited (maxWaitTime), and a waiting call is +// dropped when its signal aborts. Rejections carry a code: queue-full, queue-timeout, aborted. + +export type SemaphoreOptions = { maxWaiting?: number, maxWaitTime?: number } + +export class SemaphoreError extends Error { + code: 'queue-full' | 'queue-timeout' | 'aborted' + constructor (code: SemaphoreError['code'], message: string) { + super(message) + this.code = code + } +} + +export const createSemaphore = (max: number, options: SemaphoreOptions = {}) => { let active = 0 const queue: Array<() => void> = [] - const acquire = () => new Promise(resolve => { + const acquire = (signal?: AbortSignal) => new Promise((resolve, reject) => { + if (signal?.aborted) return reject(new SemaphoreError('aborted', 'aborted before acquiring a slot')) if (active < max) { + active++ + return resolve() + } + if (options.maxWaiting !== undefined && queue.length >= options.maxWaiting) { + return reject(new SemaphoreError('queue-full', `more than ${options.maxWaiting} calls are already waiting`)) + } + let timer: NodeJS.Timeout | undefined + const leave = (err: SemaphoreError) => { + const i = queue.indexOf(grant) + if (i !== -1) queue.splice(i, 1) + cleanup() + reject(err) + } + const onAbort = () => leave(new SemaphoreError('aborted', 'aborted while waiting for a slot')) + const cleanup = () => { + clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + } + const grant = () => { + cleanup() active++ resolve() - } else { - queue.push(() => { - active++ - resolve() - }) + } + queue.push(grant) + signal?.addEventListener('abort', onAbort, { once: true }) + if (options.maxWaitTime !== undefined) { + timer = setTimeout(() => leave(new SemaphoreError('queue-timeout', `waited more than ${options.maxWaitTime}ms for a slot`)), options.maxWaitTime) } }) @@ -21,8 +55,9 @@ export const createSemaphore = (max: number) => { } return { - async run (fn: () => Promise): Promise { - await acquire() + max, + async run (fn: () => Promise, signal?: AbortSignal): Promise { + await acquire(signal) try { return await fn() } finally { @@ -30,6 +65,8 @@ export const createSemaphore = (max: number) => { } }, // number of calls waiting for a free slot - pending: () => queue.length + pending: () => queue.length, + // number of calls running + active: () => active } } diff --git a/benchmark/FINDINGS.md b/benchmark/FINDINGS.md new file mode 100644 index 0000000..db4e671 --- /dev/null +++ b/benchmark/FINDINGS.md @@ -0,0 +1,384 @@ +# capture performance investigation, 2026-11 + +Scope: how the service orchestrates captures, how it drives the browser through puppeteer, which +browser it embeds, and the alternatives at each of these levels. Every claim marked +**measured** comes from an experiment of this harness (`npm run benchmark -- `, see +[README.md](./README.md)). Claims marked **code** come from reading the code. Claims marked +**not measured** are reasoned but untested. + +Machine: 16 cores, 31 GB of RAM, a shared desktop under load (other apps running). Browsers: +google-chrome-stable 150 (the one the Docker image installs), Chrome for Testing 156 and +chrome-headless-shell 156. Treat absolute numbers as indicative and trust the ratios. The +tables below are from the 8-run pass. Raw JSON is in `results/` (git-ignored). + +## Status (2026-11-09) + +- **Done on this branch**: the wait strategy (§1, network idle then visual quiet, capped), a + `capture-no-trigger` warning for pages that do not call `window.triggerCapture`, the PNG + fast encoding above `pngFastEncodingPixels` and for full-page captures (§2), and the fixes of + `type=jpg` (PNG bytes labelled `image/jpeg`) and of `type=gif` on a page without + animation (PNG bytes labelled `image/gif`). +- **Done too**: the debug listeners only when `DEBUG` is set (§6); abandoned captures skipped + and the contexts queue bounded (`config.queue`, §7); the GIF encoding in piscina worker + threads (`config.gifThreads`, §5). Event loop probe during a GIF: p90 83 → 1.2 ms, max + 131 → 4.6 ms. chrome-headless-shell (§3): Debian's `chromium-headless-shell` in the image, + puppeteer's download in development, tests and CI. Image 1.12 GB → 904 MB, container memory + 324 → 142 MiB idle and ~780 → ~550 MiB under 10 concurrent captures. It runs sandboxed with + `seccomp.json` (renderers in their own user namespace). Its version comes from Debian's + security updates, not from puppeteer: puppeteer 25.16 expects 156, trixie ships 156.0.8078.79. + Keep the two close when upgrading puppeteer. +- **Concurrency review (2026-11-09)**: the contexts pool reused incognito contexts across + captures and only deleted cookies, so localStorage, IndexedDB and Cache Storage of one capture + were visible to the next one, whatever its user (shown by a test on master). Each capture now + runs in a fresh incognito context, closed after it (+3 ms). generic-pool is replaced by the + service's own semaphore, with a bounded queue, a wait timeout (30 s, below data-fair's 60 s) + and abort while waiting. `/inspect` uses the same mechanism. Infrastructure side: the + production pod runs at its 1.5 GiB limit at concurrency 3 on the real mix and uses >5 cores with + no CPU request. Concurrency 2 is planned. +- **Declined**: untagged PDFs (§4). Accessibility comes first. + +## Steady load on real applications (2026-11-09) + +The two images (master vs this branch), `CONCURRENCY=5`, 8 clients looping for 60 s over the 66 +public applications of opendata.koumoul.com (52 call `triggerCapture`), 1050×450, 2 runs per +image in alternating order, a fresh container each run: + +| | master | this branch | +|---|---|---| +| captures per minute | 145, 160 | **163, 170** (+~10%) | +| p50 latency, apps with triggerCapture | 2221, 1965 ms | 1930, 1904 ms | +| p50 latency, apps without | 4154, 3615 ms | **2646, 2627 ms** (−30%) | +| CPU per capture (container) | 2966, 2841 ms | 2900, 3004 ms | +| cores busy | 7.2, 7.6 | 7.9, 8.5 | +| memory peak (cgroup) | 2815, 2927 MB | 2873, 2858 MB | + +**The instance is CPU-bound on real applications, and the CPU goes to WebGL maps.** Measured app +by app (browser process tree, sequential): 3.4 s of CPU per capture on average. 21 of the 66 +applications use WebGL (maplibre maps: admin-divs-levels, data-fair-geo-shapes, carto-stats, +admin-divs-catchment, infos-parcelles, infos-loc, carto-explore, atelier-carto...), and they take +**90% of the CPU**, 5-18 s each. Without a GPU, Chrome renders WebGL in software (SwiftShader). +The other 45 applications average ~0.5 s. Consequences: +- the shorter waits mostly cut latency. Throughput only gains ~10%, because the CPU is the limit; +- the browser savings of chrome-headless-shell (~100 ms of CPU per light capture) are a few + percent of a real mix. Its memory gain shows at idle, not at the peak, which is set by the + renderers of heavy pages; +- **the next lever is the WebGL maps**: a capture mode in the map applications (no + transitions or fly-to, `fadeDuration: 0`, `triggerCapture` on the map's `idle` event), or + measuring Chrome's other software GL backends. + +## TL;DR + +Where a capture's time goes today, for a typical data-fair page that does not call +`triggerCapture`: + +``` +newPage+setup ~35-90 ms (negligible) +networkidle0 1000-2100 ms <- Chrome's idleness detector, not 500 ms +fixed sleep 1000 ms <- "for safety" +screenshot 30-330 ms (PNG encoded on the browser's main thread) +``` + +Ranked by gain for effort: + +| # | change | gain (measured) | cost / risk | +|---|---|---|---| +| 1 | replace `networkidle0` + 1 s with puppeteer's `waitForNetworkIdle` + 2 animation frames (+ a configurable short settle) | static page 2.0 s → 0.52 s, app page 3.1 s → 1.1 s; a context is held ~2 s less, so ~2× throughput at the same concurrency on I/O-bound pages | pages that render >0.5 s after their last response without any network activity are captured too early. Today they are only caught by accident (see §1). A product decision | +| 2 | `optimizeForSpeed: true` on PNG screenshots | encoding ×2.5 faster, browser CPU −60%; under CPU-bound concurrency **+40% throughput, −32% p50** (PNG encoding serializes on the browser's main thread) | files 1.1-1.9× bigger | +| 3 | switch to **chrome-headless-shell** (Debian `chromium-headless-shell`) | idle memory −51% (145 vs 296 MB), launch 39 vs 155 ms, browser CPU −20 to −43% per capture (PDF ×1.6 faster), +30% CPU-bound throughput; multi-arch (arm64), 230 MB installed vs 416 MB, security updates through apt | a different build (Chromium, not Google Chrome). Needs the test suite and a visual check of real pages | +| 4 | ~~PDF `tagged: false` (puppeteer defaults to `true`)~~ declined | PDF ×2 faster, files **÷9** (1.2 MB → 137 KB for 30 sections) | tagged PDF is what makes a PDF accessible (screen readers, RGAA). A product decision: maybe opt-in per request | +| 5 | move the GIF encoding off the main thread (worker thread, or sharp/libvips) | today a 30-frame GIF burns ~4.3 s of main-thread CPU and stalls every other request (probe p90 83 ms, max 131 ms) | small code change. sharp would be a new native dependency | +| 6 | attach the `console` listener only when `DEBUG` is set | node CPU ÷4 on a page logging 500 messages | none | +| 7 | drop queued work whose client left; bound the queue | after 30 abandoned requests, a 0.7 s capture answered in **18 s** (§7) | small | + +Not worth it (measured): launch flags (none helps, the commented-out swiftshader option costs +×3.7 CPU), websocket vs pipe CDP transport (no difference), a fresh context vs the pool (no +difference), the sandbox (no measurable cost). Not worth it (reasoned): Playwright (same engine, +same CDP), Lightpanda (no rendering engine, so no screenshots or PDFs), Firefox through BiDi. + +## 1. The wait strategy: most of the latency + +**code**: `waitForPage` (`api/utils/page.ts`) runs `page.goto(target, { waitUntil: 'networkidle0' })` +raced against `window.triggerCapture`. Then it looks up 2 metas, and when neither is set it +sleeps 1000 ms "for safety". + +**measured** (`wait`): `networkidle0` is not "500 ms without requests". It maps to Chrome's own +`networkIdle` lifecycle event, fired by Blink's `IdlenessDetector`. That detector only checks for +quiet when a main-thread task runs or when its watchdog timer fires. It also pushes the quiet +window forward by the time spent in tasks. So the event lands well after the nominal 500 ms, at a +moment that depends on page activity: + +Median wait in ms (8 runs), and the share of captures where the page had finished rendering: + +| page | current | idle0 (no +1s) | idle0-300 | idle200-raf | **idle500-raf** | idle2 | +|---|---|---|---|---|---|---| +| static (no sub-resource) | 2014, 8/8 | 1012, 8/8 | 1313, 8/8 | 223, 8/8 | **524, 8/8** | 2013, 8/8 | +| app (3 API calls of 150 ms) | 3114, 8/8 | 2113, 8/8 | 2413, 8/8 | 781, 8/8 | **1080, 8/8** | 1669, 8/8 | +| app, render 300 ms after the last response | 3115, 8/8 | 2112, 8/8 | 2413, 8/8 | 780, **0/8** | **1078, 8/8** | 1669, 8/8 | +| app, render 800 ms after the last response | 2381, 8/8 | 1377, 8/8 | 1678, 8/8 | 782, **0/8** | **1075, 0/8** | 1670, 8/8 | +| app, API calls of 400 ms | 3113, 8/8 | 2109, 8/8 | 2414, 8/8 | 1533, 8/8 | **1830, 8/8** | 1914, 8/8 | +| app calling triggerCapture | 572 | 573 | 573 | 571 | 574 | 574 | + +Oddity that shows the detector is not a network timer: the page that renders 800 ms late reaches +`networkidle0` *sooner* (1377 ms) than the one that renders 300 ms late (2112 ms). + +Reading: +- the floor of the service is **2 s on a static page with no sub-resource**. `idle500-raf` + (puppeteer-side `waitForNetworkIdle({ idleTime: 500 })` after `load`, then 2 animation frames) + does the same page in ~0.55 s; +- on the app page (data rendered ~0.55 s after navigation), the service answers in 2.7-3.1 s. + `idle500-raf` answers in ~1.1 s and the render is complete; +- `late300`/`late800` render 300/800 ms after their last response, with no network in between. + The current strategy catches them through Chrome's detector delay plus the 1 s. The fast + strategies miss renders later than their idle window. **This is the real trade-off.** Today's + margin is accidental and unbounded (up to 2+ s). A deliberate margin would be a config value, + e.g. `idleTime` 500 ms plus a configurable settle, with the existing `df:capture-delay` meta + and `triggerCapture` for pages that need more; +- `networkidle2` is not a fix: it fires earlier, but the 1 s sleep remains. + +**measured** (`e2e`, the real service): app page at concurrency 5, the service's own timer steps: +`wait1-network-idle` 2110 ms + `wait2-1s` 1000 ms out of a 3.2 s total; with `triggerCapture` +the total is 0.69 s. + +### Synthetic risks after the last response (`wait`, fixtures/settle.html, 4 runs) + +| page | current | idle500-raf | idle500-500 | **quiet** (idle 500 ms, then 200 ms of visual quiet, cap 2 s) | +|---|---|---|---|---| +| canvas chart animated 1 s by requestAnimationFrame | 1699 ms, 4/4 | 692, 0/4 | 1169, 0/4 | **1421, 4/4** | +| canvas chart animated 2.5 s | 1694, **0/4** | 690, 0/4 | 1170, 0/4 | 2676 (cap), 3/4 | +| CSS transition 1 s | 1695, 4/4 | 691, 0/4 | 1171, 1/4 | **1424, 4/4** | +| endless spinner next to the content | 1676, 4/4 | 688, 4/4 | 1170, 4/4 | **672, 4/4** (endless animations ignored) | +| web font loaded late | 3009, 4/4 | 1289, 4/4 | 1782, 4/4 | **1281, 4/4** | +| content inserted by a 800 ms setTimeout | 1971, 4/4 | 684, 0/4 | 1170, 4/4 | 671, **0/4** | + +`quiet` is faster than today and safer on animations: an animation longer than today's +accidental margin is captured half-drawn today. Its blind spot is a render delayed by a plain +timer after a quiet period. The `capture-no-trigger` warning exists for that: such pages should +call `triggerCapture`. + +### Real pages (`real`: the 66 public applications of opendata.koumoul.com, 1050×450) + +Each capture is compared to a reference, today's strategy + 3 s, and to a second reference that +measures the page's own noise. +- 54 of 66 applications call `triggerCapture` (app-charts, app-dashboards, maps, catalog...): + the strategy barely changes them. +- the 12 without any signal: app-timelines, list-details, periodic-series, data-fair-quizz, + data-fair-locate-game, data-fair-sort-game, app-carousel. `quiet` takes 1.0-1.3 s vs 2.0-2.3 s, + with no difference to the reference, except: + - app-carousel rotates its slides, so every strategy differs from the reference (noise 90%); + - one list-details app has a map whose tiles load and decode in web workers, invisible to + every page-level signal. Today's capture is half-loaded too (5.5% off the reference), + `quiet` captures the map empty (25%). The fix belongs in the application: call + `triggerCapture` on the map's `idle` event; +- a semantic signal was tried and rejected: waiting while a visible `role=progressbar` is present. + The same list-details page shows 19 Vuetify progress bars that are value bars, not loaders. + +**A/B of the real service** (this branch vs master, 2 interleaved captures per application and +side, public applications): + +| | master | this branch | pixel diff | +|---|---|---|---| +| 9 applications without triggerCapture, median | 2093 ms | **1075 ms** | 0% on all | +| 5 applications with triggerCapture, median | 1345 ms | 1334 ms | 0% on all | + +### Throughput + +Since a capture holds its browser context while it waits, the wait also caps throughput: +`concurrency` captures every ~3.2 s = 1.5/s at the default concurrency of 5 (**measured**). + +## 2. Screenshot encoding, and the single-browser ceiling + +**measured** (`output`): + +| page | png (today) | png `optimizeForSpeed` | jpeg q80 | webp q80 | +|---|---|---|---|---| +| app 800×450 | 25 ms, 44 KB | 23 ms, 83 KB | 23 ms, 40 KB | 43 ms, 14 KB | +| heavy 1920×1080 | 302 ms, 2.5 MB | **113 ms**, 2.7 MB | **52 ms**, 675 KB | 232 ms, 570 KB | +| long page, full height at 1280 | 552 ms, 1.8 MB | **267 ms**, 2.7 MB | 306 ms, 2.8 MB | 1110 ms, 1.6 MB | + +On small viewports encoding is negligible. It matters for large viewports and full-page +captures (`height=auto`). + +**measured** (`concurrency`, `hotspot`): on a paint-heavy page, one browser stops scaling at about +2 cores, whatever the concurrency (c10 and c20 give the same throughput). Splitting c10 over 2 +browsers gives +40% throughput and −30% p50. The CPU split by process and by thread shows the +serial bottleneck: the **browser process main thread** (`CrBrowserMain`) at ~0.7-0.85 of a +core. Renderers use ~0.7 cores in total, the GPU process 0.1. `Page.captureScreenshot` encodes +the image in the browser process. Proof: on the same browser at c10, `optimizeForSpeed` gives +5.6/s vs 4.0/s (+40%), and jpeg-80 gives 6.2/s (+55%). + +| load | c1 | c2 | c5 (default) | c10 | c20 | c10 over 2 browsers | +|---|---|---|---|---|---|---| +| app, current wait (I/O-bound): captures/s | 0.3 | 0.6 | 1.5 | 2.7 | 4.6 | 2.8 | +| p50 | 3.2 s | 3.3 s | 3.4 s | 3.5 s | 3.7 s | 3.5 s | +| cores used / peak PSS | 0.1 / 320 MB | 0.2 / 351 | 0.4 / 443 | 0.9 / 623 | 1.3 / 942 | 0.8 / 855 | +| heavy 1280×800 (CPU-bound): captures/s | 0.8 | 1.4 | 2.9 | 3.9 | **3.9** | **5.3** | +| p50 | 1.25 s | 1.42 s | 1.52 s | 2.35 s | 4.98 s | 1.58 s | +| cores used / peak PSS | 0.3 / 423 MB | 0.7 / 500 | 1.4 / 720 | 2.1 / 1097 | 2.1 / 1860 | 2.9 / 1339 | + +`hotspot` (heavy, c10): browser process 0.8-0.9 core (its main thread alone ~0.7), renderers +0.7 cores in total, GPU process 0.1. With chrome-headless-shell: 4.9/s, p50 1.74 s. + +Consequences: +- for light pages, the service is I/O-bound (waiting): throughput scales linearly with + concurrency, at ~25-30 MB of PSS per extra concurrent page. The default concurrency of 5 uses + 0.4 core. With a shorter wait (§1), the same concurrency gives ~2× the throughput; +- for heavy pages, scale with more browsers (more pods, or N browsers per pod with the contexts + pool sharded), not with more contexts in one browser; +- `optimizeForSpeed` raises the ceiling of a browser by ~40%. The bigger files matter for + thumbnails that are stored; jpeg/webp could be an option for callers that accept it. Beware, **code**: + `type=jpeg` does not work today. The router sets the response type but calls + `page.screenshot()` without `type`, so it returns PNG bytes labelled `image/jpeg`. + +## 3. The browser binary + +**measured** (`browser`): + +| | chrome-stable 150 (today) | same, --no-sandbox | Chrome for Testing 156 | **chrome-headless-shell 156** | +|---|---|---|---|---| +| launch | 155 ms | 147 ms | 141 ms | **39 ms** | +| idle memory (PSS) | 296 MB | 299 MB | 269 MB | **145 MB** | +| processes | 11 | 10 | 8 | 6 | +| app png: total / browser CPU | 639 ms / 240 ms | 637 / 230 | 651 / 230 | 637 / **150** | +| heavy 1920×1080 png: total / CPU | 1372 / 660 | 1372 / 670 | 1373 / 700 | 1337 / **535** | +| long PDF: pdf() time / CPU | 267 / 595 | 267 / 600 | 291 / 625 | **169** / **340** | +| memory after the captures | 266 MB | 266 MB | 662 MB | **149 MB** | + +Under CPU-bound concurrency (c10, heavy page, `hotspot`): 5.0-5.1/s with the shell vs 3.8/s, +p50 1.6 s vs 2.2-2.3 s. + +- **chrome-headless-shell** is the "old headless" Chromium, shipped as a separate binary since + Chrome 132 removed old headless from the main binary. It has no browser UI layer. It uses half + the memory at idle, 15-45% less browser CPU per capture, and handles ~30% more throughput when + CPU-bound (**measured**, `hotspot`). Page latency is the same, because the waits dominate; +- the 150 → 156 version change (chrome-stable vs CfT) explains little. The headless mode + (CfT 156 vs shell 156) explains most of the difference; +- the sandbox has no measurable cost (chrome-stable vs chrome-stable-nosandbox); +- packaging: Debian trixie (the base of the image) ships `chromium-headless-shell` 156 for amd64 + and arm64, 230 MB installed vs 416 MB for `/opt/google/chrome`, with fewer dependencies (no + GTK, cups…), and updates through the same `apt-get upgrade` as the rest of the image. The + current Dockerfile pins the Google repo to `arch=amd64` while it handles arm64 for dumb-init, + so an arm64 image has no browser today. Puppeteer drives it with `headless: 'shell'`; +- risks (**not measured**): Chromium is not Google Chrome (no proprietary codecs, which do not + matter for captures), and old headless has small behavioural differences. The test suite and + a visual diff of a sample of real data-fair pages should gate the switch; +- alternatives: + - **Playwright**: same Chromium, same CDP, and its default headless is already the headless + shell. No performance gain to expect over puppeteer + shell. The setup calls measured below + cost < 1 ms each, so the driver is not a cost; + - **Lightpanda**: a fast headless browser without a rendering engine. It cannot produce + screenshots or PDFs (placeholder images per its docs), so it is out of scope; + - **Firefox through WebDriver BiDi** (puppeteer supports it): a different rendering of the + same pages, no known performance advantage. Not measured; + - dedicated PDF engines (WeasyPrint, Gotenberg): Gotenberg is Chromium too. WeasyPrint does not + run JavaScript, and data-fair pages are JS applications. Out of scope. + +## 4. PDF + +**measured** (`output`): + +| document | default (tagged) | `tagged: false` | `outline: true` | footer template | +|---|---|---|---|---| +| 30 sections (~40 pages) | 245 ms, 1222 KB | **119 ms, 137 KB** | 251 ms, 1247 KB | 282 ms, 1233 KB | +| 120 sections | 1026 ms, 4.9 MB | **422 ms, 489 KB** | 1064 ms, 5.0 MB | 1169 ms, 4.9 MB | + +Puppeteer defaults to `tagged: true` and also launches Chrome with `--export-tagged-pdf`. +Tagging builds the accessibility structure of the document. It doubles the PDF time and makes +the file ~9× bigger. `outline` costs nothing. The footer template costs +10-15%. + +## 5. Animated GIF + +**code**: `api/utils/animation.ts` takes a PNG screenshot per frame, decodes it in pure JS +(get-pixels/pngjs), quantizes and encodes it in pure JS (gif-encoder, NeuQuant), then runs +gifsicle. All of it runs on the service's main thread. + +**measured** (`gif`, 800×450, 30 frames): + +| variant | total | screenshots | JS decode | JS encode | gifsicle | node CPU | max event loop stall | size | +|---|---|---|---|---|---|---|---|---| +| today (png `optimizeForSpeed`, get-pixels, gif-encoder) | 4.5 s | 1.43 s | 0.37 s | **2.44 s** | 0.26 s | **4.2 s** | **138 ms** | 3.4 MB | +| jpeg-90 frames | 4.4 s | 1.03 s | 0.38 s | 2.72 s | 0.27 s | 4.0 s | 169 ms | 3.3 MB | +| sharp, default options | 7.1 s | 1.76 s | - | 5.03 s (libvips threads) | 0.28 s | 5.2 s (threads) | **12 ms** | 3.7 MB | + +**measured** (`e2e`): during one GIF capture, a probe requesting `api-docs.json` every 20 ms sees +p50 1 ms but p90 83 ms and max 131 ms. Every request of the service (other captures, metrics, +health checks) stalls for the length of a frame encoding. + +- the browser side is ~1/3 of the time. Node-side decode + quantize + encode is ~2/3, all on + the main thread; +- jpeg frames do not help: a cheaper screenshot, but a slower JS decode; +- sharp (libvips + libimagequant + cgif) with `effort: 1, dither: 0` uses about the same CPU + as today (2.3 s vs 3.0 s for 30 frames, **measured** on saved frames), but in libuv threads. + The event loop stays free (max stall 13 ms vs 139 ms). Its defaults (effort 7, dithering) are + 2× slower than today; +- the cheapest fix with no new dependency is to run the existing decode + encode in a + `worker_thread`. A semaphore like the one of `/inspect` should bound the concurrent GIFs: + each one burns a full core for seconds; +- an animated WebP (sharp, q80) takes ~0.5 s and 359 KB vs ~4.7 s and 3.4 MB for the GIF. That + could be an option for clients that accept it; +- **code**: `maxAnimationFrames` is 1800 ("2 minutes at 15 fps"), but the callback of an + animated capture has a `screenshotTimeout * 2` = 40 s budget. At ~160 ms per 800×450 frame, + that budget allows ~250 frames (~17 s of animation). Larger viewports allow fewer. + +## 6. Plumbing: contexts, pages, setup calls, console, transport + +**measured** (`context`): `newPage` ~30 ms, each setup call (`emulateTimezone`, +`setExtraHTTPHeaders`, `evaluateOnNewDocument`, `exposeFunction`, `setViewport`, request +interception) < 1 ms, the meta lookups ~4 ms, cleanup ~2.5 ms. A fresh incognito context per +capture costs the same as the pool. The pool is fine, and so is the per-capture setup. + +**measured** (`console`): `debugPage` attaches a `console` listener on every page, even when +`DEBUG` is off. The listener does 2 CDP round trips per argument of every message. On a page +logging 500 messages: node CPU 167 ms → 40 ms without it. Latency barely moves, because the work +lands after the response, but the CPU is paid on the service's main thread. Fix: attach it only +when `debug.enabled`. + +**measured** (`transport`): websocket vs `pipe: true`, on a 1920×3000 PNG and an 800 KB PDF: no +difference. + +**measured** (`args`): `--disable-gpu`, no site isolation, `--renderer-process-limit=2`, +disabling background features: within noise for throughput. No site isolation or `--renderer-process-limit=2` save +10-18% of browser CPU (fewer renderer processes). That is not worth giving up process isolation +in a service that renders arbitrary pages. The +commented-out option of `config/default.cjs` (`--use-angle=swiftshader --in-process-gpu`) costs +**×3.7 CPU**: remove it, so nobody enables it. + +## 7. Orchestration under load + +**code**, `api/utils/page.ts`: +- the contexts pool (`generic-pool`) has no `maxWaitingClients` and no `acquireTimeoutMillis`. + Under overload the queue is unbounded. The capture timeouts only start after the acquisition, + so a queued request can wait indefinitely; +- `/screenshot` and `/print` never check whether their client is still there. `/inspect` does + (`req.socket.destroyed` before rendering). A request abandoned by a proxy timeout (60 s) or by a + user still gets rendered for nobody when its turn comes, and delays the requests behind it. + **measured** (`e2e`, overload scenario): 30 clients give up after 2 s, then one client asks for a page +that takes 0.7 s alone. It gets its answer after **18.0 s**, because the service first renders the +abandoned requests for nobody (concurrency 5, ~3.2 s each); +- `promiseTimeout` never clears its timer: each capture leaves 2 pending timers of 20-40 s. Minor; +- **not measured**: one browser process lives as long as the pod. Chrome's memory tends to grow + over long uptimes, and nothing recycles it except a crash. A restart after N captures or above + a memory threshold (drain the pool, launch a new browser, close the old one) would bound it. + +Suggested shape: check `req.socket.destroyed` before and after the acquisition in `withPage`, +and abort the render when the client closes (`res.on('close')`). Bound the queue +(`maxWaitingClients`, then a 503 that the proxy or client can retry), and give the acquisition a +timeout. + +## 8. Applications to fix + +The `capture-no-trigger` warning (`code: capture-no-trigger`, with the target and what settled the +capture) lists them in production. From the public sample: app-timelines, list-details (on the +map's `idle` event), periodic-series, data-fair-quizz, data-fair-locate-game, +data-fair-sort-game, app-carousel, and infos-territoires (it relies on its `df:capture-delay` +meta, 2 s, without calling `triggerCapture`). + +## 9. Suggested order of work (initial) + +1. Low risk, no product decision: console listener behind `debug.enabled`; `optimizeForSpeed` + on PNG screenshots (if bigger files are acceptable); skip abandoned requests and bound the + queue; GIF encoding in a worker thread plus a GIF semaphore; remove the swiftshader comment. +2. Wait strategy: decide the safety margin (§1), then implement it behind config, A/B it with + `e2e` (`BENCH_E2E_DIR` = a master worktree), and check real data-fair applications. +3. Browser: build an image on `chromium-headless-shell`, run the tests, diff screenshots of real + pages, then roll it out. Revisit `concurrency` per pod with the memory figures above. +4. PDF tagging: product decision (accessibility vs size/time), maybe a `tagged` query parameter. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..9a0f824 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,76 @@ +# capture benchmark + +A local harness to test performance hypotheses about capture: the wait strategy, the browser +binary and its flags, the output encoders, the GIF pipeline, the concurrency. Findings and +recommendations are in [FINDINGS.md](./FINDINGS.md). + +No dev environment needed. The harness starts its own fixtures server (pages served with +artificial latency, no internet) and its own browsers. Only `e2e` starts the service itself, as a +child process. + +```bash +npm run benchmark # list the experiments +npm run benchmark -- wait --runs=8 # one experiment +npm run benchmark -- all --runs=8 # everything, ~1 hour +npm run benchmark -- concurrency --browser=shell +``` + +`--browser` picks the binary for single-browser experiments: +- `chrome-stable`, the default, is what the Docker image installs (`/usr/bin/google-chrome-stable`); +- `cft` and `shell` are the Chrome for Testing and chrome-headless-shell builds in + `~/.cache/puppeteer`. Install them with `npx puppeteer browsers install chrome-headless-shell`. + +On Ubuntu ≥ 23.10, AppArmor blocks the Chrome sandbox for binaries that have no AppArmor profile. +Only the google-chrome-stable deb installs one. So `cft` and `shell` run with `--no-sandbox`, and +`chrome-stable-nosandbox` gives the like-for-like comparison. + +## Experiments + +| name | question | +|---|---| +| `wait` | latency, and share of complete renders, for each wait strategy after navigation | +| `browser` | chrome-stable vs Chrome for Testing vs chrome-headless-shell: launch, latency, CPU, memory | +| `context` | pooled context vs fresh context vs reused page, and the cost of each page setup call | +| `console` | cost of the console listener that `debugPage` always attaches | +| `output` | screenshot formats (png, png `optimizeForSpeed`, jpeg, webp) and PDF options (tagged, outline, footer) | +| `gif` | per-stage cost of the animated GIF pipeline, and how long it blocks the event loop | +| `concurrency` | throughput, latency, CPU and memory for 1 to 20 concurrent captures, with 1 or 2 browsers | +| `args` | chrome launch flags: CPU, memory and process count at concurrency 5 | +| `transport` | websocket vs pipe CDP transport, on large outputs | +| `e2e` | the real service over HTTP: timer steps under load, event loop blocking during a GIF, overload with abandoned requests | +| `hotspot` | CPU by chrome process type under paint-heavy load, to find what caps a single browser | +| `real` | the wait strategies on the public applications of a data-fair instance, pixel-compared to a settled reference | + +`e2e` options, as environment variables: +- `BENCH_E2E_DIR`: run the service from another checkout, for example a worktree of master, to + A/B two versions of the code; +- `BENCH_E2E_CONFIG`: JSON merged into the service config, for example + `{"puppeteerLaunchOptions":{"headless":"shell","executablePath":"..."}}`. + +`real` options: `BENCH_REAL_URL` (default `https://opendata.koumoul.com/data-fair`) and +`BENCH_REAL_APPS` (comma separated ids; default: up to 2 applications per base application). +It captures one page at a time, to stay gentle with the instance. Suspect captures and their +reference are written to `results/real/`. + +`wait` option: `BENCH_WAIT_PAGES` (comma separated page names) for a subset. + +`gif` option: `BENCH_SHARP` is the path of a sharp install +(`npm i sharp` in a scratch directory, then point to `node_modules/sharp/dist/index.cjs`). +It adds a variant that encodes with libvips, cgif and libimagequant. sharp is not a dependency +of the service. + +## Method + +- Variants are interleaved (A B C, B C A, …), so that drift in the machine's load hits all + variants alike. One warmup iteration per variant is discarded. +- Each cell shows the median, then `[min-max]`, then the ratio to the first row. `LOWER` and + `HIGHER` flag a difference only when the min–max ranges do not overlap and the medians differ by + more than 10%. This is the rule of the data-fair harness. +- Browser CPU is the CPU time of the browser process tree, read from `/proc` (Linux only). Child + processes that already exited are counted through their parent's cumulated fields. Memory is the + PSS of the tree, which does not count shared pages twice the way a sum of RSS would. +- `lib/capture.ts` copies the page lifecycle of `api/utils/page.ts`, with each lever turned + into a parameter. It is kept in sync by hand. `e2e` is the check against the real code. +- Results are printed as markdown tables and saved as JSON in `results/` (git-ignored), tagged + with the commit. +- The dev machine is shared and loaded. Trust the ratios more than the absolute values. diff --git a/benchmark/experiments/args.ts b/benchmark/experiments/args.ts new file mode 100644 index 0000000..30e4431 --- /dev/null +++ b/benchmark/experiments/args.ts @@ -0,0 +1,45 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch } from '../lib/browsers.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import { runLoad } from './concurrency.ts' +import type { Experiment } from './common.ts' + +// H: some launch flags reduce the browser's CPU or memory per capture: no GPU process (software +// raster in the renderer), fewer background features, no site isolation (fewer processes), +// a renderer process limit. Measured at concurrency 5 on the CPU-bound page and on the app page. + +const argVariants: Record = { + default: [], + 'no-sandbox': ['--no-sandbox'], + 'disable-gpu': ['--disable-gpu'], + 'no-site-isolation': ['--disable-site-isolation-trials', '--disable-features=IsolateOrigins,site-per-process'], + 'renderer-limit-2': ['--renderer-process-limit=2'], + 'disable-features': ['--disable-features=Translate,OptimizationHints,MediaRouter,DialMediaRouteProvider,InterestFeedContentSuggestions,CertificateTransparencyComponentUpdater,AutofillServerCommunication,PrivacySandboxSettings4,LensOverlay,PaintHolding', '--disable-component-update', '--no-default-browser-check'], + 'swiftshader-in-process': ['--use-gl=angle', '--use-angle=swiftshader', '--in-process-gpu'] +} + +export const args: Experiment = { + description: 'chrome launch flags: CPU, memory and process count at concurrency 5', + async run (opts) { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + const all: Record = {} + for (const [loadName, load] of Object.entries({ + 'heavy 1280x800': { path: '/fixtures/heavy.html?n=15000', viewport: { width: 1280, height: 800 }, strategy: 'idle0-raf' as const }, + 'app-trigger 800x450': { path: '/fixtures/app.html?trigger=1', viewport: { width: 800, height: 450 }, strategy: 'idle0-raf' as const } + })) { + const results = await runInterleaved(Object.keys(argVariants), Math.max(2, Math.round(opts.runs / 3)), async (name) => { + const l = await launch(variant, { args: argVariants[name] }) + try { + return await runLoad([l], server.url, load, 5, 20) + } finally { + await l.browser.close() + } + }, 0) + all[loadName] = results + report(loadName, results, ['perSec', 'p50Ms', 'cpuMsPerCapture', 'peakPssMB', 'peakProcs']) + } + await server.close() + save('args', all) + } +} diff --git a/benchmark/experiments/browser.ts b/benchmark/experiments/browser.ts new file mode 100644 index 0000000..86a5eb2 --- /dev/null +++ b/benchmark/experiments/browser.ts @@ -0,0 +1,70 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch, browserFootprint, type Launched } from '../lib/browsers.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool } from '../lib/capture.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save, step } from '../lib/runner.ts' +import type { Experiment } from './common.ts' + +// H: chrome-headless-shell (the old headless mode, a separate lighter binary) launches faster, +// renders and encodes faster and uses less memory than the full chrome binary in new headless mode +// that the image installs today. Chrome for Testing in new headless mode separates the effect of +// the version (stable 150 vs CfT 156) from the effect of the headless mode (CfT 156 vs shell 156). + +const tasks = { + 'app-png': { path: '/fixtures/app.html?trigger=1', viewport: { width: 800, height: 450 }, out: 'png' }, + 'heavy-png': { path: '/fixtures/heavy.html', viewport: { width: 1920, height: 1080 }, out: 'png' }, + 'long-pdf': { path: '/fixtures/long.html', viewport: undefined, out: 'pdf' } +} as const + +export const browser: Experiment = { + description: 'chrome-stable vs Chrome for Testing vs chrome-headless-shell: launch, capture time, CPU, memory', + async run (opts) { + const server = await startFixturesServer() + const names = Object.keys(browserVariants) + const launched: Record = {} + const launchResults = await runInterleaved(names, Math.min(opts.runs, 5), async (name) => { + const l = await launch(browserVariants[name]) + const sample = { launchMs: l.launchMs, idlePssMB: browserFootprint(l).pssMB, processes: browserFootprint(l).processes } + await l.browser.close() + return sample + }, 0) + report('launch (fresh process, no page)', launchResults, ['launchMs', 'idlePssMB', 'processes']) + for (const name of names) { + launched[name] = await launch(browserVariants[name]) + console.log(`${name}: ${launched[name].version}`) + } + const pools = Object.fromEntries(names.map(n => [n, createContextPool(() => launched[n].browser.createBrowserContext())])) + const all: Record = { launch: launchResults } + for (const [taskName, task] of Object.entries(tasks)) { + const results = await runInterleaved(names, opts.runs, async (name) => { + const l = launched[name] + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + step('newPage') + const context = await pools[name].acquire() + const page = await context.newPage() + if (task.viewport) await page.setViewport(task.viewport) + step('setup') + await setupPage(page) + step('goto') + await gotoAndWait(page, server.url(task.path), 'idle0-raf') + const t1 = performance.now() + step('output') + const buf = task.out === 'png' ? await page.screenshot() : await page.pdf({ format: 'A4', printBackground: true }) + const t2 = performance.now() + step('clean') + await cleanPage(page) + pools[name].release(context) + await new Promise(resolve => setTimeout(resolve, 100)) // let the renderer exit before reading CPU + return { openMs: t1 - t0, outputMs: t2 - t1, totalMs: t2 - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0, KB: buf.length / 1024 } + }) + all[taskName] = results + report(taskName, results, ['openMs', 'outputMs', 'totalMs', 'browserCpuMs', 'KB']) + } + const footprint = Object.fromEntries(names.map(n => [n, [browserFootprint(launched[n])]])) + report('footprint after the captures above (contexts kept in pool)', footprint, ['pssMB', 'processes']) + for (const name of names) await launched[name].browser.close() + await server.close() + save('browser', all) + } +} diff --git a/benchmark/experiments/common.ts b/benchmark/experiments/common.ts new file mode 100644 index 0000000..614fd77 --- /dev/null +++ b/benchmark/experiments/common.ts @@ -0,0 +1,23 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch, type Launched } from '../lib/browsers.ts' +import type { LaunchOptions } from 'puppeteer' + +export type Opts = { runs: number, browser: string } +export type Experiment = { description: string, run: (opts: Opts) => Promise } + +// a fixtures server and a browser for the duration of fn +export const withEnv = async (opts: Opts, fn: (env: { url: (p: string) => string, l: Launched }) => Promise, launchOptions: LaunchOptions = {}) => { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + if (!variant) throw new Error('unknown browser ' + opts.browser) + const l = await launch(variant, launchOptions) + console.log(`browser: ${variant.name} ${l.version}`) + try { + return await fn({ url: server.url, l }) + } finally { + await l.browser.close() + await server.close() + } +} + +export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) diff --git a/benchmark/experiments/concurrency.ts b/benchmark/experiments/concurrency.ts new file mode 100644 index 0000000..77022d8 --- /dev/null +++ b/benchmark/experiments/concurrency.ts @@ -0,0 +1,90 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch, type Launched } from '../lib/browsers.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool, type WaitStrategy } from '../lib/capture.ts' +import { treeCpuMs, treePssMB, treeSize } from '../lib/proc.ts' +import { aggregate } from '../lib/stats.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Experiment } from './common.ts' +import type { ScreenshotOptions } from 'puppeteer' + +// H: with the current wait strategy a capture is mostly idle time, so throughput scales almost +// linearly with the number of contexts, and the default concurrency (5) leaves the machine idle. +// On a paint-heavy page the browser CPU saturates instead and more concurrency only adds latency. +// Also: one browser vs 2 browsers for the same concurrency. + +type Load = { path: string, viewport: { width: number, height: number }, strategy: WaitStrategy, shot?: ScreenshotOptions } +const loads: Record = { + 'app (I/O bound, current wait)': { path: '/fixtures/app.html', viewport: { width: 800, height: 450 }, strategy: 'current' }, + 'heavy 1280x800 (CPU bound)': { path: '/fixtures/heavy.html?n=15000', viewport: { width: 1280, height: 800 }, strategy: 'idle0-raf' } +} + +const runLoad = async (browsers: Launched[], url: (p: string) => string, load: Load, concurrency: number, total: number) => { + const pools = browsers.map(l => createContextPool(() => l.browser.createBrowserContext())) + const latencies: number[] = [] + let next = 0 + let peakPss = 0 + let peakProcs = 0 + const sampler = setInterval(() => { + peakPss = Math.max(peakPss, browsers.reduce((a, l) => a + treePssMB(l.pid), 0)) + peakProcs = Math.max(peakProcs, browsers.reduce((a, l) => a + treeSize(l.pid), 0)) + }, 250) + const cpu0 = browsers.reduce((a, l) => a + treeCpuMs(l.pid), 0) + const t0 = performance.now() + await Promise.all(Array.from({ length: concurrency }, async (_, w) => { + const pool = pools[w % pools.length] + while (next < total) { + next++ + const s = performance.now() + const c = await pool.acquire() + const page = await c.newPage() + await page.setViewport(load.viewport) + await setupPage(page) + await gotoAndWait(page, url(load.path), load.strategy) + await page.screenshot(load.shot) + await cleanPage(page) + pool.release(c) + latencies.push(performance.now() - s) + } + })) + const wallMs = performance.now() - t0 + clearInterval(sampler) + await new Promise(resolve => setTimeout(resolve, 200)) + const cpuMs = browsers.reduce((a, l) => a + treeCpuMs(l.pid), 0) - cpu0 + for (const p of pools) await p.drain() + const lat = aggregate(latencies) + return { perSec: total / wallMs * 1000, p50Ms: lat.median, p90Ms: lat.p90, cpuMsPerCapture: cpuMs / total, cores: cpuMs / wallMs, peakPssMB: peakPss, peakProcs } +} + +export const concurrency: Experiment = { + description: 'throughput, latency, CPU and memory for 1..20 concurrent captures, 1 or 2 browsers', + async run (opts) { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + const b1 = await launch(variant) + const b2 = await launch(variant) + console.log(`browser: ${variant.name} ${b1.version}`) + const all: Record = {} + const configs: Record = { + c1: { c: 1, browsers: [b1] }, + c2: { c: 2, browsers: [b1] }, + c5: { c: 5, browsers: [b1] }, + c10: { c: 10, browsers: [b1] }, + c20: { c: 20, browsers: [b1] }, + 'c10-2browsers': { c: 10, browsers: [b1, b2] } + } + for (const [loadName, load] of Object.entries(loads)) { + const results = await runInterleaved(Object.keys(configs), Math.max(2, Math.round(opts.runs / 4)), async (name) => { + const { c, browsers } = configs[name] + return runLoad(browsers, server.url, load, c, Math.max(10, c * 3)) + }, 0) + all[loadName] = results + report(loadName, results, ['perSec', 'p50Ms', 'p90Ms', 'cpuMsPerCapture', 'cores', 'peakPssMB', 'peakProcs']) + } + await b1.browser.close() + await b2.browser.close() + await server.close() + save('concurrency', all) + } +} + +export { runLoad } diff --git a/benchmark/experiments/console.ts b/benchmark/experiments/console.ts new file mode 100644 index 0000000..1713b66 --- /dev/null +++ b/benchmark/experiments/console.ts @@ -0,0 +1,36 @@ +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool } from '../lib/capture.ts' +import { selfCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' + +// H: the debugPage console listener is attached on every page even when DEBUG is off, and does +// 2 CDP round trips per argument of every console message (getProperty + jsonValue). On a verbose +// page this costs latency and node CPU for nothing. + +export const consoleListener: Experiment = { + description: 'cost of the always-on console listener on a page logging 500 messages', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const pool = createContextPool(() => l.browser.createBrowserContext()) + const results = await runInterleaved(['listener', 'no-listener'], opts.runs, async (variant) => { + const c = await pool.acquire() + const page = await c.newPage() + await setupPage(page, { consoleListener: variant === 'listener' }) + const cpu0 = selfCpuMs() + const t0 = performance.now() + await gotoAndWait(page, url('/fixtures/console.html?logs=500'), 'idle0') + await page.screenshot() + const totalMs = performance.now() - t0 + // let the pending listener round trips land, they are paid even if after the response + await new Promise(resolve => setTimeout(resolve, 300)) + const nodeCpuMs = selfCpuMs() - cpu0 + await cleanPage(page) + pool.release(c) + return { totalMs, nodeCpuMs } + }) + report('console.html?logs=500', results, ['totalMs', 'nodeCpuMs']) + await pool.drain() + save('console', results) + }) + } +} diff --git a/benchmark/experiments/context.ts b/benchmark/experiments/context.ts new file mode 100644 index 0000000..dc925c0 --- /dev/null +++ b/benchmark/experiments/context.ts @@ -0,0 +1,81 @@ +import { withEnv, type Experiment } from './common.ts' +import { cleanPage, createContextPool } from '../lib/capture.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Page } from 'puppeteer' + +// H: the per-capture cost of the browser plumbing (context, page, setup CDP calls) is small next +// to the waits. Variants: the service's pooled context + new page; a fresh context per capture +// (no pool, no cookie cleanup); a reused page (the floor, not safe across users). +// Also: the cost of each setup step taken one by one. + +export const context: Experiment = { + description: 'pooled context vs fresh context vs reused page, and the cost of each page setup call', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const pool = createContextPool(() => l.browser.createBrowserContext()) + const reused = await (await l.browser.createBrowserContext()).newPage() + await reused.setViewport({ width: 800, height: 450 }) + const target = url('/fixtures/static.html') + const results = await runInterleaved(['pool+newPage', 'fresh-context', 'reused-page'], opts.runs * 2, async (variant) => { + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + let page: Page + if (variant === 'reused-page') { + page = reused + } else if (variant === 'fresh-context') { + page = await (await l.browser.createBrowserContext()).newPage() + await page.setViewport({ width: 800, height: 450 }) + } else { + page = await (await pool.acquire()).newPage() + await page.setViewport({ width: 800, height: 450 }) + } + const tPage = performance.now() + await page.goto(target, { waitUntil: 'load' }) + await page.screenshot() + const tShot = performance.now() + if (variant === 'fresh-context') await page.browserContext().close() + else if (variant === 'pool+newPage') { + const c = page.browserContext() + await cleanPage(page) + pool.release(c) + } + const tEnd = performance.now() + await new Promise(resolve => setTimeout(resolve, 50)) + return { pageMs: tPage - t0, gotoShotMs: tShot - tPage, cleanupMs: tEnd - tShot, totalMs: tEnd - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0 } + }) + report('static page, goto load + screenshot', results, ['pageMs', 'gotoShotMs', 'cleanupMs', 'totalMs', 'browserCpuMs']) + + // each setup call of the service, timed alone on a fresh page + const steps: Record Promise> = { + newPage: async () => {}, + setViewport: p => p.setViewport({ width: 800, height: 450 }), + emulateTimezone: p => p.emulateTimezone('Europe/Paris'), + setExtraHTTPHeaders: p => p.setExtraHTTPHeaders({ 'Accept-Language': 'fr-FR' }), + evaluateOnNewDocument: p => p.evaluateOnNewDocument(() => {}), + exposeFunction: p => p.exposeFunction('triggerCapture' + Math.random().toString(36).slice(2), () => {}), + setRequestInterception: p => p.setRequestInterception(true), + metaLookup: p => p.$eval('head > meta[name=\'x-capture\']', el => el.content).catch(() => undefined), + cleanPage: p => cleanPage(p) + } + const stepResults = await runInterleaved(['steps'], opts.runs * 2, async () => { + const sample: Record = {} + const t0 = performance.now() + const c = await pool.acquire() + const p = await c.newPage() + sample.newPage = performance.now() - t0 + for (const [k, fn] of Object.entries(steps)) { + if (k === 'newPage') continue + const t = performance.now() + await fn(p) + sample[k] = performance.now() - t + } + pool.release(c) + return sample + }) + report('setup steps, ms each', stepResults, Object.keys(steps)) + await pool.drain() + save('context', { results, stepResults }) + }) + } +} diff --git a/benchmark/experiments/e2e.ts b/benchmark/experiments/e2e.ts new file mode 100644 index 0000000..77da23f --- /dev/null +++ b/benchmark/experiments/e2e.ts @@ -0,0 +1,137 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import net from 'node:net' +import path from 'node:path' +import { startFixturesServer } from '../lib/server.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { aggregate } from '../lib/stats.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Experiment } from './common.ts' + +// The real service, started as a child process from the working tree (or from BENCH_E2E_DIR, e.g. +// a worktree of master, to compare 2 versions), and loaded over HTTP. Scenarios: +// - png: 20 screenshots of the app page at concurrency 5, the timer breakdown of the service +// - gif+ping: an animated capture while a probe requests api-docs every 20ms, the probe latency +// shows how long the service's event loop is blocked for every other request +// BENCH_E2E_CONFIG (JSON) is merged in the service's config, e.g. a browser or a concurrency. + +const freePort = () => new Promise(resolve => { + const s = net.createServer().listen(0, () => { + const port = (s.address() as net.AddressInfo).port + s.close(() => resolve(port)) + }) +}) + +const startService = async (dir: string, extraConfig: Record) => { + const port = await freePort() + const child: ChildProcess = spawn('node', ['--disable-warning=ExperimentalWarning', 'api/index.ts'], { + cwd: dir, + env: { + ...process.env, + NODE_ENV: 'development', + NODE_CONFIG: JSON.stringify({ port, onlySameHost: false, concurrency: 5, observer: { active: false }, ...extraConfig }) + }, + stdio: ['ignore', 'ignore', 'pipe'] + }) + child.stderr?.on('data', () => {}) + const base = `http://127.0.0.1:${port}/api/v1` + for (let i = 0; i < 100; i++) { + try { + if ((await fetch(base + '/api-docs.json')).ok) return { base, child } + } catch {} + await new Promise(resolve => setTimeout(resolve, 200)) + } + child.kill() + throw new Error('service did not start') +} + +export const e2e: Experiment = { + description: 'the real service over HTTP: png load at concurrency 5, event loop blocking during a gif', + async run (opts) { + const fixtures = await startFixturesServer() + const dir = process.env.BENCH_E2E_DIR ?? path.join(import.meta.dirname, '..', '..') + const extra = JSON.parse(process.env.BENCH_E2E_CONFIG ?? '{}') + const { base, child } = await startService(dir, extra) + console.log(`service from ${dir}, config ${JSON.stringify(extra)}`) + const shot = (target: string, q = '') => fetch(`${base}/screenshot?key=capture&target=${encodeURIComponent(target)}${q}`, { headers: { 'x-secret-key': 'capture' } }) + try { + const all: Record = {} + for (const [name, page] of Object.entries({ app: '/fixtures/app.html', 'app-trigger': '/fixtures/app.html?trigger=1' })) { + const results = await runInterleaved(['c5'], Math.max(2, Math.round(opts.runs / 2)), async () => { + const total = 20 + let next = 0 + const latencies: number[] = [] + const steps: Record = {} + const cpu0 = treeCpuMs(child.pid!) + const t0 = performance.now() + await Promise.all(Array.from({ length: 5 }, async () => { + while (next < total) { + next++ + const s = performance.now() + const res = await shot(fixtures.url(page), '&timer=true') + const times = await res.json() as Record + latencies.push(performance.now() - s) + for (const [k, v] of Object.entries(times)) (steps[k] ??= []).push(v) + } + })) + const wallMs = performance.now() - t0 + const lat = aggregate(latencies) + return { + perSec: total / wallMs * 1000, + p50Ms: lat.median, + p90Ms: lat.p90, + cpuMsPerCapture: (treeCpuMs(child.pid!) - cpu0) / total, + ...Object.fromEntries(Object.entries(steps).map(([k, v]) => [k, aggregate(v).median])) + } + }, 1) + all[name] = results + report(`${name}, 20 screenshots at concurrency 5 (service + browser CPU)`, results, ['perSec', 'p50Ms', 'p90Ms', 'cpuMsPerCapture']) + const stepNames = Object.keys(results.c5[0]).filter(k => !['perSec', 'p50Ms', 'p90Ms', 'cpuMsPerCapture'].includes(k)) + report(`${name}, median of the service timer steps (ms)`, results, stepNames) + } + const gifResults = await runInterleaved(['gif'], Math.max(2, Math.round(opts.runs / 4)), async () => { + const state = { stop: false } + const pings: number[] = [] + const probe = (async () => { + while (!state.stop) { + const s = performance.now() + await fetch(base + '/api-docs.json').then(r => r.arrayBuffer()) + pings.push(performance.now() - s) + await new Promise(resolve => setTimeout(resolve, 20)) + } + })() + const s = performance.now() + const res = await shot(fixtures.url('/fixtures/anim.html?frames=30'), '&type=gif&width=800&height=450') + const buf = await res.arrayBuffer() + const gifMs = performance.now() - s + state.stop = true + await probe + const p = aggregate(pings) + return { gifMs, KB: buf.byteLength / 1024, pingP50Ms: p.median, pingP90Ms: p.p90, pingMaxMs: p.max } + }, 0) + all.gif = gifResults + + // overload: 30 clients that give up after 2s (a proxy timeout, a user leaving), then one + // patient client. If the abandoned requests still wait for a context and get rendered, + // the patient one queues behind work done for nobody. + const abandon = await (async () => { + const controllers = Array.from({ length: 30 }, () => new AbortController()) + const pending = controllers.map(c => fetch(`${base}/screenshot?target=${encodeURIComponent(fixtures.url('/fixtures/app.html'))}`, { headers: { 'x-secret-key': 'capture' }, signal: c.signal }).then(r => r.arrayBuffer()).catch(() => {})) + await new Promise(resolve => setTimeout(resolve, 2000)) + for (const c of controllers) c.abort() + await Promise.all(pending) + const s = performance.now() + const res = await shot(fixtures.url('/fixtures/app.html?trigger=1')) + await res.arrayBuffer() + return { patientMs: performance.now() - s, status: String(res.status) } + })() + all.abandon = abandon + console.log(`\n### overload: 30 requests abandoned after 2s, then 1 request (app?trigger=1, ~0.7s alone)\n\npatient request: ${Math.round(abandon.patientMs)}ms, status ${abandon.status}`) + report('gif 800x450 30 frames, with a probe on api-docs every 20ms', gifResults, ['gifMs', 'KB', 'pingP50Ms', 'pingP90Ms', 'pingMaxMs']) + save('e2e', all) + } finally { + child.kill('SIGINT') + await new Promise(resolve => child.once('exit', resolve)) + await fixtures.close() + } + } +} diff --git a/benchmark/experiments/gif.ts b/benchmark/experiments/gif.ts new file mode 100644 index 0000000..2182c93 --- /dev/null +++ b/benchmark/experiments/gif.ts @@ -0,0 +1,108 @@ +import { promisify } from 'node:util' +import { monitorEventLoopDelay } from 'node:perf_hooks' +import GifEncoder from 'gif-encoder' +import getPixelsCb from 'get-pixels' +import { optimizeGif } from '../../api/utils/gifsicle.ts' +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage } from '../lib/capture.ts' +import { selfCpuMs, treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' + +// H: an animated capture is dominated by node-side work, not by the browser: each frame is a PNG +// that is decoded in pure JS (get-pixels/pngjs) then color-quantized in pure JS (gif-encoder, +// NeuQuant), all on the main thread, which blocks the event loop of the whole service (other +// requests, metrics, health checks) for the duration. The stages are timed separately, with the +// event loop delay measured meanwhile. Variant: jpeg frames instead of png. + +const getPixels = promisify(getPixelsCb) + +// sharp (libvips + cgif + libimagequant, native, off the main thread) is not a dependency of the +// service: point BENCH_SHARP to an install of it to add the variant (npm i sharp in a scratch dir) +const loadSharp = async () => { + if (!process.env.BENCH_SHARP) return undefined + return (await import(process.env.BENCH_SHARP)).default +} + +export const gif: Experiment = { + description: 'animated GIF pipeline: per-stage cost and event loop blocking', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const context = await l.browser.createBrowserContext() + const width = 800 + const height = 450 + const frames = 30 + const sharp = await loadSharp() + const variants = ['png-fast', 'jpeg-90'] + if (sharp) variants.push('sharp') + const results = await runInterleaved(variants, Math.max(2, Math.round(opts.runs / 2)), async (variant) => { + const page = await context.newPage() + await page.setViewport({ width, height }) + await setupPage(page) + await gotoAndWait(page, url(`/fixtures/anim.html?frames=${frames}`), 'idle0') + const h = monitorEventLoopDelay({ resolution: 5 }) + h.enable() + const cpu0 = selfCpuMs() + const bcpu0 = treeCpuMs(l.pid) + const t = { evaluate: 0, screenshot: 0, decode: 0, encode: 0, gifsicle: 0 } + const pngs: Buffer[] = [] + const gifEnc = new GifEncoder(width, height) + gifEnc.setFrameRate(15) + const chunks: Buffer[] = [] + gifEnc.on('data', (c: Buffer) => chunks.push(c)) + gifEnc.writeHeader() + let stopped = false + let n = 0 + const t0 = performance.now() + while (!stopped) { + n++ + let s = performance.now() + // @ts-ignore + stopped = await page.evaluate(() => window.animateCaptureFrame()) + t.evaluate += performance.now() - s + s = performance.now() + const buffer = await page.screenshot(variant === 'jpeg-90' ? { type: 'jpeg', quality: 90 } : { optimizeForSpeed: true }) + t.screenshot += performance.now() - s + if (variant === 'sharp') { + pngs.push(Buffer.from(buffer)) + continue + } + s = performance.now() + const pixels = await getPixels(Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength), variant === 'jpeg-90' ? 'image/jpeg' : 'image/png') + t.decode += performance.now() - s + s = performance.now() + gifEnc.addFrame(pixels.data) + t.encode += performance.now() - s + } + gifEnc.finish() + await new Promise(resolve => setImmediate(resolve)) + let raw = Buffer.concat(chunks) + if (variant === 'sharp') { + // decoding, quantization (libimagequant) and encoding (cgif) in libvips threads + const s = performance.now() + raw = await sharp(pngs, { join: { animated: true } }).gif({ delay: pngs.map(() => 67), loop: 0 }).toBuffer() + t.encode = performance.now() - s + } + const s = performance.now() + const optimized = await optimizeGif(raw, 2) + t.gifsicle = performance.now() - s + const totalMs = performance.now() - t0 + h.disable() + await cleanPage(page) + return { + frames: n, + totalMs, + ...Object.fromEntries(Object.entries(t).map(([k, v]) => [k + 'Ms', v])), + nodeCpuMs: selfCpuMs() - cpu0, + browserCpuMs: treeCpuMs(l.pid) - bcpu0, + loopMaxMs: h.max / 1e6, + loopP99Ms: h.percentile(99) / 1e6, + rawKB: raw.length / 1024, + KB: optimized.length / 1024 + } + }) + report(`anim.html ${width}x${height}, 30 frames`, results, ['totalMs', 'evaluateMs', 'screenshotMs', 'decodeMs', 'encodeMs', 'gifsicleMs', 'nodeCpuMs', 'browserCpuMs', 'loopMaxMs', 'KB']) + await context.close() + save('gif', results) + }) + } +} diff --git a/benchmark/experiments/hotspot.ts b/benchmark/experiments/hotspot.ts new file mode 100644 index 0000000..9fd32c4 --- /dev/null +++ b/benchmark/experiments/hotspot.ts @@ -0,0 +1,90 @@ +import fs from 'node:fs' +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch } from '../lib/browsers.ts' +import { processTree } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import { runLoad } from './concurrency.ts' +import type { Experiment } from './common.ts' + +// H: a single browser saturates at about 2 cores on paint-heavy pages, whatever the +// concurrency, because one of its processes is a serial bottleneck (the GPU process +// composites and rasterizes for all renderers). The CPU time is split by process type, from +// the --type= argument of each chrome process, sampled during the load. A process that sits +// near 100% of one core is the bottleneck. + +const typeOf = (pid: number) => { + try { + // chrome rewrites its title as a single space separated string + const cmd = fs.readFileSync(`/proc/${pid}/cmdline`, 'utf8').split(/[\0 ]/) + const type = cmd.find(a => a.startsWith('--type='))?.slice(7) ?? 'browser' + const sub = cmd.find(a => a.startsWith('--utility-sub-type='))?.slice(19).split('.').pop() + return sub ? `${type}:${sub}` : type + } catch { + return 'gone' + } +} +const ticks = (pid: number) => { + try { + const raw = fs.readFileSync(`/proc/${pid}/stat`, 'utf8') + const f = raw.slice(raw.lastIndexOf(')') + 2).split(' ') + return Number(f[11]) + Number(f[12]) + } catch { + return 0 + } +} + +const launchVariants: Record = { + 'chrome-stable': { browser: 'chrome-stable', args: [] }, + 'chrome-stable --disable-gpu': { browser: 'chrome-stable', args: ['--disable-gpu'] }, + shell: { browser: 'shell', args: [] } +} + +export const hotspot: Experiment = { + description: 'CPU by chrome process type at concurrency 10 on a paint-heavy page: which process caps a browser', + async run (opts) { + const server = await startFixturesServer() + const results = await runInterleaved(Object.keys(launchVariants), Math.max(1, Math.round(opts.runs / 4)), async (name) => { + const v = launchVariants[name] + const l = await launch(browserVariants[v.browser], { args: v.args }) + // per pid: first and last ticks seen, the processes of closed pages included + const seen = new Map() + const sample = () => { + for (const pid of processTree(l.pid)) { + const t = ticks(pid) + const s = seen.get(pid) + if (s) s.last = t + else seen.set(pid, { type: typeOf(pid), first: t, last: t }) + } + } + sample() + const t0 = performance.now() + const sampler = setInterval(sample, 100) + const load = await runLoad([l], server.url, { path: '/fixtures/heavy.html?n=15000', viewport: { width: 1280, height: 800 }, strategy: 'idle0-raf' }, 10, 30) + clearInterval(sampler) + sample() + const wallMs = performance.now() - t0 + await l.browser.close() + // share of one core used by each process type, and by its busiest single process + const byType: Record = {} + const maxProc: Record = {} + for (const s of seen.values()) { + const cores = (s.last - s.first) * 10 / wallMs + const key = s.type.startsWith('renderer') ? 'renderer' : s.type + byType[key] = (byType[key] ?? 0) + cores + maxProc[key] = Math.max(maxProc[key] ?? 0, cores) + } + if (process.env.BENCH_DEBUG) console.log([...seen.entries()].map(([pid, s]) => `${pid} ${s.type} ${s.last - s.first}`).join('\n')) + const out: Record = { perSec: load.perSec, p50Ms: load.p50Ms } + for (const k of ['browser', 'gpu-process', 'renderer', 'utility:mojom']) { + out[k + '.cores'] = byType[k] ?? 0 + out[k + '.maxProc'] = maxProc[k] ?? 0 + } + for (const [k, c] of Object.entries(byType)) if (!(k + '.cores' in out)) out[k + '.cores'] = c + return out + }, 0) + const metrics = [...new Set(Object.values(results).flat().flatMap(s => Object.keys(s)))] + report('heavy 1280x800 at concurrency 10, CPU in cores by process type (maxProc = busiest single process)', results, metrics) + await server.close() + save('hotspot', results) + } +} diff --git a/benchmark/experiments/index.ts b/benchmark/experiments/index.ts new file mode 100644 index 0000000..0092da7 --- /dev/null +++ b/benchmark/experiments/index.ts @@ -0,0 +1,15 @@ +import type { Experiment } from './common.ts' +import { wait } from './wait.ts' +import { browser } from './browser.ts' +import { context } from './context.ts' +import { consoleListener } from './console.ts' +import { output } from './output.ts' +import { gif } from './gif.ts' +import { concurrency } from './concurrency.ts' +import { args } from './args.ts' +import { transport } from './transport.ts' +import { e2e } from './e2e.ts' +import { hotspot } from './hotspot.ts' +import { real } from './real.ts' + +export const experiments: Record = { wait, browser, context, console: consoleListener, output, gif, concurrency, args, transport, e2e, hotspot, real } diff --git a/benchmark/experiments/output.ts b/benchmark/experiments/output.ts new file mode 100644 index 0000000..e3cdf15 --- /dev/null +++ b/benchmark/experiments/output.ts @@ -0,0 +1,72 @@ +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage } from '../lib/capture.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Page, ScreenshotOptions, PDFOptions } from 'puppeteer' + +// H: encoding the output is a sizable share of a capture on large viewports. PNG compression in +// the browser is slow, optimizeForSpeed / jpeg / webp are cheaper. For PDF, the tagged structure +// (accessibility tree, on by default) and the document outline cost time on long documents. +// The page is opened once per task and the output is taken repeatedly. + +const shots: Record = { + png: {}, + 'png-fast': { optimizeForSpeed: true }, + 'jpeg-80': { type: 'jpeg', quality: 80 }, + 'webp-80': { type: 'webp', quality: 80 } +} +const shotPages = { + 'app 800x450': { path: '/fixtures/app.html?trigger=1', viewport: { width: 800, height: 450 }, fullPage: false }, + 'heavy 1920x1080': { path: '/fixtures/heavy.html', viewport: { width: 1920, height: 1080 }, fullPage: false }, + 'long fullPage 1280': { path: '/fixtures/long.html?sections=10', viewport: { width: 1280, height: 800 }, fullPage: true } +} + +const pdfs: Record = { + default: {}, + 'tagged-false': { tagged: false }, + outline: { outline: true }, + footer: { displayHeaderFooter: true, headerTemplate: ' ', footerTemplate: '
/
' } +} + +export const output: Experiment = { + description: 'screenshot formats (png, png optimizeForSpeed, jpeg, webp) and pdf options (tagged, outline, footer)', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const all: Record = {} + const context = await l.browser.createBrowserContext() + const open = async (path: string, viewport?: { width: number, height: number }) => { + const page: Page = await context.newPage() + if (viewport) await page.setViewport(viewport) + await setupPage(page) + await gotoAndWait(page, url(path), 'idle0-raf') + return page + } + for (const [name, p] of Object.entries(shotPages)) { + const page = await open(p.path, p.viewport) + const results = await runInterleaved(Object.keys(shots), opts.runs, async (v) => { + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + const buf = await page.screenshot({ ...shots[v], fullPage: p.fullPage }) + return { ms: performance.now() - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0, KB: buf.length / 1024 } + }) + all[name] = results + report(`screenshot: ${name}`, results, ['ms', 'browserCpuMs', 'KB']) + await cleanPage(page) + } + for (const sections of [30, 120]) { + const page = await open(`/fixtures/long.html?sections=${sections}`) + const results = await runInterleaved(Object.keys(pdfs), opts.runs, async (v) => { + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + const buf = await page.pdf({ format: 'A4', printBackground: true, margin: { left: '1.5cm', right: '1.5cm', top: '1.5cm', bottom: '1.5cm' }, ...pdfs[v] }) + return { ms: performance.now() - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0, KB: buf.length / 1024 } + }) + all[`pdf-${sections}`] = results + report(`pdf: long.html, ${sections} sections`, results, ['ms', 'browserCpuMs', 'KB']) + await cleanPage(page) + } + await context.close() + save('output', all) + }) + } +} diff --git a/benchmark/experiments/real.ts b/benchmark/experiments/real.ts new file mode 100644 index 0000000..fb58ebf --- /dev/null +++ b/benchmark/experiments/real.ts @@ -0,0 +1,107 @@ +import fs from 'node:fs' +import path from 'node:path' +import { PNG } from 'pngjs' +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage, type WaitStrategy } from '../lib/capture.ts' +import { report, save } from '../lib/runner.ts' + +// Wait strategies on real pages: the public applications of a data-fair instance (by default +// opendata.koumoul.com), captured at the size of data-fair thumbnails. Each capture is compared +// pixel by pixel to a reference taken on a surely settled page (current strategy + 3s). A second +// reference measures the noise of the page itself (maps, dates, random colors...). +// BENCH_REAL_URL data-fair base url (default https://opendata.koumoul.com/data-fair) +// BENCH_REAL_APPS comma separated application ids (default: up to 2 per base application) +// Captures run one at a time to stay gentle with the instance. Suspect captures are written +// to results/real/ for a visual check. + +const baseUrl = process.env.BENCH_REAL_URL ?? 'https://opendata.koumoul.com/data-fair' +const candidates: WaitStrategy[] = ['current', 'idle500-raf', 'idle500-500', 'quiet'] +const viewport = { width: 1050, height: 450 } + +type App = { id: string, title: string, base: string } + +// https://cdn.jsdelivr.net/npm/@data-fair/app-charts@1.4/dist/ and https://koumoul.com/apps/atelier-carto/1.8/ +// -> app-charts and atelier-carto +const baseName = (url: string) => url.split('/').filter(s => s && s !== 'dist' && !/^v?\d[\d.]*$/.test(s)).pop()?.replace(/@[^@]*$/, '') || '?' + +const listApps = async (): Promise => { + const res = await fetch(`${baseUrl}/api/v1/applications?size=1000&select=id,title,url&visibility=public`) + const { results } = await res.json() as { results: { id: string, title: string, url: string }[] } + const apps = results.map(a => ({ id: a.id, title: a.title, base: baseName(a.url ?? '') })) + const only = process.env.BENCH_REAL_APPS?.split(',') + if (only) return apps.filter(a => only.includes(a.id)) + const perBase: Record = {} + for (const a of apps) { + perBase[a.base] ??= [] + if (perBase[a.base].length < 2) perBase[a.base].push(a) + } + return Object.values(perBase).flat() +} + +// share of pixels (%) whose channels differ by more than 40 out of 255 +const diffPct = (a: Uint8Array, b: Uint8Array) => { + const pa = PNG.sync.read(Buffer.from(a)) + const pb = PNG.sync.read(Buffer.from(b)) + if (pa.width !== pb.width || pa.height !== pb.height) return 100 + let diff = 0 + for (let i = 0; i < pa.data.length; i += 4) { + if (Math.abs(pa.data[i] - pb.data[i]) > 40 || Math.abs(pa.data[i + 1] - pb.data[i + 1]) > 40 || Math.abs(pa.data[i + 2] - pb.data[i + 2]) > 40) diff++ + } + return Math.round(diff / (pa.width * pa.height) * 10000) / 100 +} + +export const real: Experiment = { + description: 'wait strategies on the public applications of a data-fair instance: latency and pixel diff to a settled reference', + async run (opts) { + const apps = await listApps() + console.log(`${apps.length} applications from ${baseUrl}`) + const outDir = path.join(import.meta.dirname, '..', 'results', 'real') + fs.mkdirSync(outDir, { recursive: true }) + await withEnv(opts, async ({ l }) => { + const capture = async (target: string, strategy: WaitStrategy) => { + const context = await l.browser.createBrowserContext() + try { + const page = await context.newPage() + await page.setViewport(viewport) + await setupPage(page) + const t0 = performance.now() + const r = await gotoAndWait(page, target, strategy) + const waitMs = performance.now() - t0 + const png = await page.screenshot() + await cleanPage(page) + return { ...r, waitMs, png } + } finally { + await context.close() + } + } + const rows: Record[] = [] + const perStrategy: Record = Object.fromEntries(candidates.map(c => [c, []])) + for (const app of apps) { + const target = `${baseUrl}/app/${app.id}` + try { + const ref = await capture(target, 'reference') + const ref2 = await capture(target, 'reference') + const noise = diffPct(ref.png, ref2.png) + const row: Record = { app: app.id, base: app.base, signal: ref.ready === 'triggered' ? 'trigger' : (ref.meta ?? 'none'), noise } + for (const strategy of candidates) { + const c = await capture(target, strategy) + const diff = diffPct(ref.png, c.png) + // a capture differs from the reference beyond what the page itself varies + const excess = Math.max(0, diff - noise) + const suspect = excess > 1 + if (suspect) fs.writeFileSync(path.join(outDir, `${app.base}-${app.id}-${strategy}.png`), c.png) + row[strategy] = `${Math.round(c.waitMs)}ms ${diff}%${c.quiet ? ' ' + c.quiet : ''}` + perStrategy[strategy].push({ waitMs: c.waitMs, diff, excess, suspect, ready: c.ready }) + } + if (Object.values(perStrategy).some(s => s.at(-1)?.suspect)) fs.writeFileSync(path.join(outDir, `${app.base}-${app.id}-reference.png`), ref.png) + rows.push(row) + console.log(JSON.stringify(row)) + } catch (err: any) { + console.log(`${app.id} (${app.base}) failed: ${err.message}`) + } + } + report('per strategy, over all applications', Object.fromEntries(Object.entries(perStrategy).map(([k, v]) => [k, v.map(s => ({ waitMs: s.waitMs, excessDiffPct: s.excess, suspect: s.suspect }))])), ['waitMs', 'excessDiffPct', 'suspect']) + save('real', rows) + }) + } +} diff --git a/benchmark/experiments/transport.ts b/benchmark/experiments/transport.ts new file mode 100644 index 0000000..437694a --- /dev/null +++ b/benchmark/experiments/transport.ts @@ -0,0 +1,41 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch } from '../lib/browsers.ts' +import { setupPage, gotoAndWait } from '../lib/capture.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Experiment } from './common.ts' +import type { Page } from 'puppeteer' + +// H: the outputs travel back base64 encoded in CDP messages over a websocket. A pipe transport +// (--remote-debugging-pipe) spares the websocket framing and the TCP loopback on large outputs. + +export const transport: Experiment = { + description: 'websocket vs pipe CDP transport on large screenshots and PDFs', + async run (opts) { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + const ws = await launch(variant, { pipe: false }) + const pipe = await launch(variant, { pipe: true }) + const pages: Record = {} + for (const [name, l] of Object.entries({ websocket: ws, pipe })) { + const page = await (await l.browser.createBrowserContext()).newPage() + await page.setViewport({ width: 1920, height: 1080 }) + await setupPage(page) + await gotoAndWait(page, server.url('/fixtures/long.html?sections=20'), 'idle0-raf') + pages[name] = page + } + const results = await runInterleaved(Object.keys(pages), opts.runs, async (name) => { + let t = performance.now() + const shot = await pages[name].screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: 1920, height: 3000 } }) + const shotMs = performance.now() - t + t = performance.now() + const pdf = await pages[name].pdf({ format: 'A4' }) + const pdfMs = performance.now() - t + return { shotMs, shotKB: shot.length / 1024, pdfMs, pdfKB: pdf.length / 1024 } + }) + report('1920x3000 png and long pdf', results, ['shotMs', 'shotKB', 'pdfMs', 'pdfKB']) + await ws.browser.close() + await pipe.browser.close() + await server.close() + save('transport', results) + } +} diff --git a/benchmark/experiments/wait.ts b/benchmark/experiments/wait.ts new file mode 100644 index 0000000..ea2d480 --- /dev/null +++ b/benchmark/experiments/wait.ts @@ -0,0 +1,60 @@ +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool, type WaitStrategy } from '../lib/capture.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' + +// H: on pages that do not call triggerCapture, the latency floor is the networkidle0 window (500ms +// of no request) plus a fixed 1s. The 1s is a safety margin for rendering after the last response; +// the question is how much of it is needed, measured by the share of captures taken before the page +// declared itself done (data-done), with renders that land 0, 300 and 800ms after the last response. + +const strategies: WaitStrategy[] = ['current', 'idle0', 'idle0-raf', 'idle0-300', 'idle200-raf', 'idle500-raf', 'idle500-500', 'quiet', 'idle2'] +// BENCH_WAIT_PAGES: comma separated subset of the pages below +const allPages: Record = { + static: '/fixtures/static.html', + app: '/fixtures/app.html', + 'app-late300': '/fixtures/app.html?late=300', + 'app-late800': '/fixtures/app.html?late=800', + // a slower backend: 400ms per API call + 'app-slow-api': '/fixtures/app.html?apiDelay=400', + 'app-trigger': '/fixtures/app.html?trigger=1', + // what may happen after the last response (fixtures/settle.html) + 'raf chart 1s': '/fixtures/settle.html?mode=raf&ms=1000', + 'raf chart 2.5s': '/fixtures/settle.html?mode=raf&ms=2500', + 'css transition 1s': '/fixtures/settle.html?mode=css&ms=1000', + spinner: '/fixtures/settle.html?mode=spinner', + 'font 600ms': '/fixtures/settle.html?mode=font&ms=600', + 'timer 800ms': '/fixtures/settle.html?mode=timer&ms=800' +} + +const pages = process.env.BENCH_WAIT_PAGES + ? Object.fromEntries(Object.entries(allPages).filter(([k]) => process.env.BENCH_WAIT_PAGES!.split(',').includes(k))) + : allPages + +export const wait: Experiment = { + description: 'latency and completeness of the wait strategies after navigation', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const pool = createContextPool(() => l.browser.createBrowserContext()) + const all: Record = {} + for (const [pageName, path] of Object.entries(pages)) { + const results = await runInterleaved(strategies, opts.runs, async (strategy) => { + const context = await pool.acquire() + const page = await context.newPage() + await page.setViewport({ width: 800, height: 450 }) + await setupPage(page) + const t0 = performance.now() + const { ready, done, quiet } = await gotoAndWait(page, url(path), strategy as WaitStrategy) + const waitMs = performance.now() - t0 + await page.screenshot() + await cleanPage(page) + pool.release(context) + return { waitMs, done, ready, quiet: quiet ?? '' } + }) + all[pageName] = results + report(`${pageName}`, results, ['waitMs', 'done', 'ready', 'quiet']) + } + await pool.drain() + save('wait', all) + }) + } +} diff --git a/benchmark/fixtures/anim.html b/benchmark/fixtures/anim.html new file mode 100644 index 0000000..5f91574 --- /dev/null +++ b/benchmark/fixtures/anim.html @@ -0,0 +1,34 @@ + + +Anim + + + + + + + diff --git a/benchmark/fixtures/app.html b/benchmark/fixtures/app.html new file mode 100644 index 0000000..fdcd2b3 --- /dev/null +++ b/benchmark/fixtures/app.html @@ -0,0 +1,54 @@ + + + + + Bench app + + + + + +

Bench app

+
+
+ + + diff --git a/benchmark/fixtures/console.html b/benchmark/fixtures/console.html new file mode 100644 index 0000000..9eb485a --- /dev/null +++ b/benchmark/fixtures/console.html @@ -0,0 +1,13 @@ + + +Console + + +

Console

+ + + diff --git a/benchmark/fixtures/heavy.html b/benchmark/fixtures/heavy.html new file mode 100644 index 0000000..dd047fe --- /dev/null +++ b/benchmark/fixtures/heavy.html @@ -0,0 +1,23 @@ + + +Heavy + + + +
+ + + diff --git a/benchmark/fixtures/long.html b/benchmark/fixtures/long.html new file mode 100644 index 0000000..fae5680 --- /dev/null +++ b/benchmark/fixtures/long.html @@ -0,0 +1,31 @@ + + +Long report + + + +

Rapport

+
+ + + diff --git a/benchmark/fixtures/settle.html b/benchmark/fixtures/settle.html new file mode 100644 index 0000000..edecba4 --- /dev/null +++ b/benchmark/fixtures/settle.html @@ -0,0 +1,68 @@ + + + + + Settle + + + + +

Settle

+
+ + + diff --git a/benchmark/fixtures/static.html b/benchmark/fixtures/static.html new file mode 100644 index 0000000..d727b13 --- /dev/null +++ b/benchmark/fixtures/static.html @@ -0,0 +1,10 @@ + + +Static + + +

Static page

+

No script, no sub-resource: the floor cost of a capture.

+ + + diff --git a/benchmark/index.ts b/benchmark/index.ts new file mode 100644 index 0000000..c6ee78d --- /dev/null +++ b/benchmark/index.ts @@ -0,0 +1,23 @@ +import { parseArgs } from 'node:util' +import { experiments } from './experiments/index.ts' + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + runs: { type: 'string', default: '8' }, + browser: { type: 'string', default: 'chrome-stable' } + } +}) + +const name = positionals[0] +if (!name || !(name in experiments || name === 'all')) { + console.log(`usage: npm run benchmark -- [--runs=8] [--browser=chrome-stable|cft|shell]\n\nexperiments:\n${Object.entries(experiments).map(([k, e]) => ` ${k.padEnd(12)} ${e.description}`).join('\n')}`) + process.exit(name ? 1 : 0) +} + +const opts = { runs: Number(values.runs), browser: values.browser } +for (const [k, e] of Object.entries(experiments)) { + if (name !== 'all' && k !== name) continue + console.log(`\n## ${k}: ${e.description}`) + await e.run(opts) +} diff --git a/benchmark/lib/browsers.ts b/benchmark/lib/browsers.ts new file mode 100644 index 0000000..dd56bdd --- /dev/null +++ b/benchmark/lib/browsers.ts @@ -0,0 +1,45 @@ +import os from 'node:os' +import fs from 'node:fs' +import path from 'node:path' +import puppeteer, { type Browser, type LaunchOptions } from 'puppeteer' +import { treePssMB, treeSize } from './proc.ts' + +// Browser binaries compared. chrome-stable is what the Docker image installs (google-chrome-stable +// deb), the others are Chrome for Testing builds downloaded by @puppeteer/browsers +// (npx puppeteer browsers install chrome-headless-shell@) +const cache = path.join(os.homedir(), '.cache', 'puppeteer') +const latest = (dir: string) => { + if (!fs.existsSync(dir)) return undefined + return fs.readdirSync(dir).sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).pop() +} +const cftVersion = latest(path.join(cache, 'chrome')) +const shellVersion = latest(path.join(cache, 'chrome-headless-shell')) + +export type BrowserVariant = { name: string, launch: LaunchOptions } + +// On Ubuntu >= 23.10 AppArmor forbids the unprivileged user namespaces of the sandbox to any +// binary without a profile: the google-chrome-stable deb installs one, the Chrome for Testing +// builds do not and only run with --no-sandbox. chrome-stable-nosandbox is the fair comparison. +export const browserVariants: Record = { + 'chrome-stable': { name: 'chrome-stable', launch: { executablePath: '/usr/bin/google-chrome-stable', headless: true } }, + 'chrome-stable-nosandbox': { name: 'chrome-stable-nosandbox', launch: { executablePath: '/usr/bin/google-chrome-stable', headless: true, args: ['--no-sandbox'] } }, + cft: { name: `cft-${cftVersion}`, launch: { executablePath: path.join(cache, 'chrome', cftVersion ?? '', 'chrome-linux64', 'chrome'), headless: true, args: ['--no-sandbox'] } }, + shell: { name: `shell-${shellVersion}`, launch: { executablePath: path.join(cache, 'chrome-headless-shell', shellVersion ?? '', 'chrome-headless-shell-linux64', 'chrome-headless-shell'), headless: 'shell', args: ['--no-sandbox'] } } +} + +export type Launched = { browser: Browser, pid: number, launchMs: number, version: string } + +export const launch = async (variant: BrowserVariant, extra: LaunchOptions = {}): Promise => { + const t0 = performance.now() + const browser = await puppeteer.launch({ + ...variant.launch, + ...extra, + args: [...(variant.launch.args ?? []), ...(extra.args ?? [])] + }) + const launchMs = performance.now() - t0 + const pid = browser.process()?.pid + if (!pid) throw new Error('no browser pid') + return { browser, pid, launchMs, version: await browser.version() } +} + +export const browserFootprint = (l: Launched) => ({ pssMB: treePssMB(l.pid), processes: treeSize(l.pid) }) diff --git a/benchmark/lib/capture.ts b/benchmark/lib/capture.ts new file mode 100644 index 0000000..9de831d --- /dev/null +++ b/benchmark/lib/capture.ts @@ -0,0 +1,155 @@ +import type { BrowserContext, Page } from 'puppeteer' + +// A copy of the capture service's page lifecycle (api/utils/page.ts) with each lever made a +// parameter. Kept in sync by hand: the point is to measure the variants, not to import the +// service with its config. + +export type WaitStrategy = + | 'current' // goto networkidle0 raced with triggerCapture, 2 meta lookups, then +1s + | 'idle0' // same without the final 1s + | 'idle0-raf' // same, the 1s replaced by 2 animation frames + | 'idle0-300' // same, the 1s replaced by 300ms + | 'idle200-raf' // goto 'load', then a 200ms network idle, then 2 animation frames + | 'idle500-raf' // goto 'load', then a 500ms network idle (puppeteer side), then 2 animation frames + | 'idle500-500' // goto 'load', then a 500ms network idle, then a fixed 500ms settle + | 'quiet' // goto 'load', a 500ms network idle, then visual quiet (see waitForQuiet), capped + | 'idle2' // goto networkidle2 (<= 2 connections for 500ms), then +1s + | 'reference' // 'current' + 3s: a page surely settled, the reference of the real pages comparison + +// the visual quiet of the 'quiet' strategy: no DOM mutation, no animation frame callback, no +// running finite CSS/Web animation and no font loading for quietMs, or maxMs elapsed +export const quietOptions = { quietMs: 200, maxMs: 2000 } + +// injected before any script of the page: records the last time the page did visible work +const installActivityTracker = () => { + const w = window as any + w.__captureLastActivity = performance.now() + const mark = () => { w.__captureLastActivity = performance.now() } + const raf = window.requestAnimationFrame.bind(window) + window.requestAnimationFrame = (cb) => raf((t) => { mark(); cb(t) }) + new MutationObserver(mark).observe(document, { subtree: true, childList: true, attributes: true, characterData: true }) +} + +export const waitForQuiet = (page: Page, quietMs = quietOptions.quietMs, maxMs = quietOptions.maxMs) => page.evaluate((quietMs, maxMs) => new Promise(resolve => { + const start = performance.now() + const check = () => { + const now = performance.now() + const busy = [] + if (now - ((window as any).__captureLastActivity ?? 0) < quietMs) busy.push('activity') + // endless animations (spinners, pulses) never settle, they are not waited for + if (document.getAnimations().some(a => a.playState === 'running' && a.effect?.getTiming().iterations !== Infinity)) busy.push('animations') + if (document.fonts.status === 'loading') busy.push('fonts') + if (!busy.length) return resolve('quiet') + if (now - start >= maxMs) return resolve('cap:' + busy.join('+')) + setTimeout(check, 50) + } + check() +}), quietMs, maxMs) + +export type SetupOptions = { + lang?: string, + timezone?: string, + consoleListener?: boolean, // the debugPage listener, attached whether DEBUG is set or not + interception?: boolean // setRequestInterception, as /inspect does +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +export const setupPage = async (page: Page, opts: SetupOptions = {}) => { + const lang = opts.lang ?? 'fr-FR' + await page.emulateTimezone(opts.timezone ?? 'Europe/Paris') + await page.setExtraHTTPHeaders({ 'Accept-Language': lang }) + await page.evaluateOnNewDocument((lang) => { + const langs = [lang] + if (lang.includes('-')) langs.push(lang.split('-')[0]) + Object.defineProperty(navigator, 'language', { get: () => lang }) + Object.defineProperty(navigator, 'languages', { get: () => langs }) + }, lang) + if (opts.consoleListener) { + page.on('console', async msg => { + const args = [] + for (const arg of msg.args()) { + try { + args.push((await (await arg.getProperty('message')).jsonValue()) || await arg.jsonValue()) + } catch {} + } + }) + } + if (opts.interception) { + await page.setRequestInterception(true) + page.on('request', request => { request.continue().catch(() => {}) }) + } +} + +export type WaitResult = { ready: 'triggered' | 'idle' | 'timeout', done: boolean, meta?: string, quiet?: string } + +export const gotoAndWait = async (page: Page, url: string, strategy: WaitStrategy, timeout = 20000): Promise => { + let triggered = false + const triggerCapture = new Promise(resolve => { + page.exposeFunction('triggerCapture', (animationSupported?: boolean) => { + triggered = true + resolve() + return !!animationSupported + }).catch(() => {}) + }) + if (strategy === 'quiet') await page.evaluateOnNewDocument(installActivityTracker) + const puppeteerIdle = ['idle200-raf', 'idle500-raf', 'idle500-500', 'quiet'].includes(strategy) + let ready: WaitResult['ready'] = 'idle' + try { + if (puppeteerIdle) { + const idleTime = strategy === 'idle200-raf' ? 200 : 500 + await Promise.race([ + page.goto(url, { waitUntil: 'load', timeout }).then(() => page.waitForNetworkIdle({ idleTime, timeout })), + triggerCapture + ]) + } else { + const waitUntil = strategy === 'idle2' ? 'networkidle2' : 'networkidle0' + await Promise.race([page.goto(url, { waitUntil, timeout }), triggerCapture]) + } + } catch (err: any) { + if (err.name !== 'TimeoutError') throw err + ready = 'timeout' + } + if (triggered) ready = 'triggered' + let meta: string | undefined + let quiet: string | undefined + if (ready === 'idle') { + // the 2 metas of the service: an explicit delay, or a promise to call triggerCapture + const delayMeta = await page.$eval('head > meta[name=\'df:capture-delay\']', el => el.content).catch(() => undefined) + const captureMeta = delayMeta ? undefined : await page.$eval('head > meta[name=\'x-capture\']', el => el.content).catch(() => undefined) + if (delayMeta) { + meta = 'delay=' + delayMeta + await Promise.race([triggerCapture, sleep(Math.min(Number(delayMeta) * 1000, timeout))]) + } else if (captureMeta === 'trigger') { + meta = 'trigger' + await Promise.race([triggerCapture, sleep(timeout)]) + } else if (strategy === 'current' || strategy === 'idle2') await sleep(1000) + else if (strategy === 'reference') await sleep(4000) + else if (strategy === 'idle0-300') await sleep(300) + else if (strategy === 'idle500-500') await sleep(500) + else if (strategy === 'quiet') quiet = await waitForQuiet(page) + else await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))) + if (triggered) ready = 'triggered' + } + const done = await page.evaluate(() => document.body?.dataset.done === '1') + return { ready, done, meta, quiet } +} + +// what the service does after a capture: delete cookies, close the page, give the context back +export const cleanPage = async (page: Page) => { + const context = page.browserContext() + const cookies = await context.cookies() + if (cookies.length) await context.deleteCookie(...cookies) + await page.close() +} + +// a minimal pool, enough to reproduce generic-pool's behaviour of the service (lazy creation, +// reuse of released contexts) +export const createContextPool = (create: () => Promise) => { + const free: BrowserContext[] = [] + return { + acquire: async () => free.pop() ?? await create(), + release: (c: BrowserContext) => { free.push(c) }, + drain: async () => { await Promise.all(free.splice(0).map(c => c.close().catch(() => {}))) } + } +} diff --git a/benchmark/lib/pngjs.d.ts b/benchmark/lib/pngjs.d.ts new file mode 100644 index 0000000..efe140c --- /dev/null +++ b/benchmark/lib/pngjs.d.ts @@ -0,0 +1,4 @@ +// the subset of pngjs (a transitive dependency, via get-pixels) used by the real pages experiment +declare module 'pngjs' { + export const PNG: { sync: { read: (buffer: Buffer) => { width: number, height: number, data: Buffer } } } +} diff --git a/benchmark/lib/proc.ts b/benchmark/lib/proc.ts new file mode 100644 index 0000000..51e62a3 --- /dev/null +++ b/benchmark/lib/proc.ts @@ -0,0 +1,72 @@ +import fs from 'node:fs' + +// CPU and memory of a browser: the launched process and all its descendants (zygotes, renderers, +// gpu, network service...), read from /proc. Linux only. + +const CLK_TCK = 100 + +const readStat = (pid: number) => { + try { + const raw = fs.readFileSync(`/proc/${pid}/stat`, 'utf8') + // the command name is between parentheses and may contain spaces + const fields = raw.slice(raw.lastIndexOf(')') + 2).split(' ') + return { + ppid: Number(fields[1]), + // utime + stime + cutime + cstime: the children reaped by a process (renderers closed with + // their page) are accounted in the cumulated fields of their parent + ticks: Number(fields[11]) + Number(fields[12]) + Number(fields[13]) + Number(fields[14]) + } + } catch { + return null + } +} + +export const processTree = (rootPid: number) => { + const parents = new Map() + for (const entry of fs.readdirSync('/proc')) { + const pid = Number(entry) + if (!pid) continue + const stat = readStat(pid) + if (stat) parents.set(pid, stat.ppid) + } + const tree = new Set([rootPid]) + let grown = true + while (grown) { + grown = false + for (const [pid, ppid] of parents) { + if (!tree.has(pid) && tree.has(ppid)) { + tree.add(pid) + grown = true + } + } + } + return [...tree] +} + +// cumulated CPU milliseconds of the tree +export const treeCpuMs = (rootPid: number) => { + let ticks = 0 + for (const pid of processTree(rootPid)) ticks += readStat(pid)?.ticks ?? 0 + return ticks * 1000 / CLK_TCK +} + +// proportional set size of the tree in MB: shared pages are split between the processes that map +// them, so the sum is not inflated like a sum of RSS +export const treePssMB = (rootPid: number) => { + let kb = 0 + for (const pid of processTree(rootPid)) { + try { + const m = fs.readFileSync(`/proc/${pid}/smaps_rollup`, 'utf8').match(/^Pss:\s+(\d+) kB/m) + if (m) kb += Number(m[1]) + } catch {} + } + return Math.round(kb / 1024) +} + +export const treeSize = (rootPid: number) => processTree(rootPid).length + +// CPU of the current node process +export const selfCpuMs = () => { + const u = process.cpuUsage() + return (u.user + u.system) / 1000 +} diff --git a/benchmark/lib/runner.ts b/benchmark/lib/runner.ts new file mode 100644 index 0000000..6c97fc8 --- /dev/null +++ b/benchmark/lib/runner.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs' +import path from 'node:path' +import { execSync } from 'node:child_process' +import { aggregate, compare } from './stats.ts' + +export type Sample = Record +export type Results = Record + +// Runs every variant `runs` times, interleaved (A B C, B C A, C A B...) so that a drift of the +// machine load hits all the variants alike. Warmup iterations are run and discarded. +// the current step of a sample, named by the experiment with step(), reported if it hangs +let currentStep = '' +export const step = (name: string) => { currentStep = name } + +const watchdog = async (variant: string, i: number, fn: () => Promise, ms: number) => { + let timer: NodeJS.Timeout | undefined + currentStep = '' + try { + return await Promise.race([ + fn(), + new Promise((resolve, reject) => { + timer = setTimeout(() => reject(new Error(`watchdog: variant ${variant}, run ${i}, step "${currentStep}" did not end in ${ms}ms`)), ms) + }) + ]) + } finally { + clearTimeout(timer) + } +} + +export const runInterleaved = async ( + variants: string[], + runs: number, + fn: (variant: string, i: number) => Promise, + warmup = 1, + timeoutMs = 120000 +): Promise => { + const results: Results = Object.fromEntries(variants.map(v => [v, []])) + for (let w = 0; w < warmup; w++) for (const v of variants) await watchdog(v, -1, () => fn(v, -1), timeoutMs) + for (let i = 0; i < runs; i++) { + const order = variants.map((_, k) => variants[(k + i) % variants.length]) + for (const v of order) results[v].push(await watchdog(v, i, () => fn(v, i), timeoutMs)) + process.stdout.write('.') + } + process.stdout.write('\n') + return results +} + +const fmt = (x: number) => Number.isNaN(x) ? '-' : Math.abs(x) >= 100 ? Math.round(x).toString() : x.toFixed(1) + +// one table: a row per variant, a column per metric, median [min-max] and the ratio to the first +// variant. Boolean metrics are shown as a rate. +export const report = (title: string, results: Results, metrics: string[]) => { + const variants = Object.keys(results) + const base = variants[0] + const rows = [['variant', ...metrics]] + for (const v of variants) { + const row = [v] + for (const m of metrics) { + const values = results[v].map(s => s[m]) + if (typeof values[0] === 'boolean') { + row.push(`${values.filter(Boolean).length}/${values.length}`) + continue + } + if (typeof values[0] === 'string') { + row.push([...new Set(values)].join(',')) + continue + } + const agg = aggregate(values.filter(x => typeof x === 'number') as number[]) + let cell = `${fmt(agg.median)} [${fmt(agg.min)}-${fmt(agg.max)}]` + if (v !== base) { + const b = aggregate(results[base].map(s => s[m]).filter(x => typeof x === 'number') as number[]) + cell += ' ' + compare(b, agg) + } + row.push(cell) + } + rows.push(row) + } + const widths = rows[0].map((_, c) => Math.max(...rows.map(r => r[c].length))) + const line = (r: string[]) => '| ' + r.map((cell, c) => cell.padEnd(widths[c])).join(' | ') + ' |' + const out = [`\n### ${title}\n`, line(rows[0]), '|' + widths.map(w => '-'.repeat(w + 2)).join('|') + '|', ...rows.slice(1).map(line)].join('\n') + console.log(out) + return out +} + +const sha = () => { + try { + return execSync('git rev-parse --short HEAD', { encoding: 'utf8' }).trim() + (execSync('git status --porcelain', { encoding: 'utf8' }).trim() ? '-dirty' : '') + } catch { return 'unknown' } +} + +export const save = (experiment: string, data: unknown) => { + const dir = path.join(import.meta.dirname, '..', 'results') + const file = path.join(dir, `${experiment}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`) + fs.writeFileSync(file, JSON.stringify({ experiment, commit: sha(), date: new Date().toISOString(), data }, null, 2)) + console.log(`\nsaved ${path.relative(process.cwd(), file)}`) +} diff --git a/benchmark/lib/server.ts b/benchmark/lib/server.ts new file mode 100644 index 0000000..7887c29 --- /dev/null +++ b/benchmark/lib/server.ts @@ -0,0 +1,68 @@ +import http from 'node:http' +import fs from 'node:fs' +import path from 'node:path' +import type { AddressInfo } from 'node:net' + +// Serves the fixtures pages, with an artificial latency (?delay=ms) on any response so that pages +// behave like data-fair applications that fetch from a remote API. +// /fixtures/.html a page +// /asset.js?kb=300&delay=80 a script of the given size (parsed and executed by the page) +// /data?n=500&delay=150 a JSON array of n random points +// /font.woff2?delay=500 a font (DejaVu Sans from the system, whatever the extension says) + +const fixturesDir = path.join(import.meta.dirname, '..', 'fixtures') + +const assetCache = new Map() +const asset = (kb: number) => { + let src = assetCache.get(kb) + if (!src) { + const lines = ['window.__lib = {}'] + let i = 0 + while (lines.join('\n').length < kb * 1024) { + lines.push(`window.__lib.f${i} = function (a, b) { const r = []; for (let k = 0; k < a; k++) r.push({ k, v: Math.sin(k * ${i}) * b, s: 'item-' + k }); return r.filter(x => x.v > 0).map(x => x.s).join(',') }`) + i++ + } + src = lines.join('\n') + assetCache.set(kb, src) + } + return src +} + +const data = (n: number) => { + const points = [] + for (let i = 0; i < n; i++) points.push({ x: i, y: Math.round(Math.abs(Math.sin(i / 10) * 100 + Math.random() * 20)), label: 'Point ' + i }) + return JSON.stringify(points) +} + +export const startFixturesServer = async () => { + const server = http.createServer((req, res) => { + const url = new URL(req.url ?? '/', 'http://localhost') + const delay = Number(url.searchParams.get('delay') ?? 0) + setTimeout(() => { + if (url.pathname.startsWith('/fixtures/')) { + const file = path.join(fixturesDir, path.basename(url.pathname)) + if (!fs.existsSync(file)) return res.writeHead(404).end() + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }) + res.end(fs.readFileSync(file)) + } else if (url.pathname === '/asset.js') { + res.writeHead(200, { 'content-type': 'application/javascript', 'cache-control': 'no-store' }) + res.end(asset(Number(url.searchParams.get('kb') ?? 100))) + } else if (url.pathname === '/font.woff2') { + res.writeHead(200, { 'content-type': 'font/ttf', 'cache-control': 'no-store' }) + res.end(fs.readFileSync('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf')) + } else if (url.pathname === '/data') { + res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'no-store' }) + res.end(data(Number(url.searchParams.get('n') ?? 100))) + } else { + res.writeHead(404).end() + } + }, delay) + }) + server.keepAliveTimeout = 60000 + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const port = (server.address() as AddressInfo).port + return { + url: (p: string) => `http://127.0.0.1:${port}${p}`, + close: () => new Promise(resolve => { server.closeAllConnections(); server.close(() => resolve()) }) + } +} diff --git a/benchmark/lib/stats.ts b/benchmark/lib/stats.ts new file mode 100644 index 0000000..b0b6f46 --- /dev/null +++ b/benchmark/lib/stats.ts @@ -0,0 +1,24 @@ +export type Agg = { n: number, median: number, p90: number, min: number, max: number, mean: number } + +export const aggregate = (samples: number[]): Agg => { + if (!samples.length) return { n: 0, median: NaN, p90: NaN, min: NaN, max: NaN, mean: NaN } + const s = [...samples].sort((a, b) => a - b) + const q = (p: number) => { + const i = (s.length - 1) * p + const lo = Math.floor(i) + const hi = Math.ceil(i) + return s[lo] + (s[hi] - s[lo]) * (i - lo) + } + return { n: s.length, median: q(0.5), p90: q(0.9), min: s[0], max: s[s.length - 1], mean: s.reduce((a, b) => a + b, 0) / s.length } +} + +// a variant is flagged against the baseline only when the min-max ranges do not overlap and the +// medians differ by more than 10%, the rule of the data-fair harness: noise on a loaded dev +// machine is large and a single median is not evidence +export const compare = (base: Agg, other: Agg) => { + if (!base.n || !other.n) return '' + const ratio = other.median / base.median + const disjoint = other.max < base.min || other.min > base.max + if (!disjoint || Math.abs(ratio - 1) < 0.1) return `x${ratio.toFixed(2)}` + return `x${ratio.toFixed(2)} ${ratio < 1 ? 'LOWER' : 'HIGHER'}` +} diff --git a/benchmark/results/.gitkeep b/benchmark/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/custom-environment-variables.cjs b/config/custom-environment-variables.cjs index 597219c..4d8dc0a 100644 --- a/config/custom-environment-variables.cjs +++ b/config/custom-environment-variables.cjs @@ -10,6 +10,11 @@ module.exports = { screenshotTimeout: 'SCREENSHOT_TIMEOUT', concurrency: 'CONCURRENCY', inspectConcurrency: 'INSPECT_CONCURRENCY', + gifThreads: 'GIF_THREADS', + queue: { + maxWaiting: 'QUEUE_MAX_WAITING', + maxWaitTime: 'QUEUE_MAX_WAIT_TIME' + }, defaultLang: 'DEFAULT_LANG', defaultTimezone: 'DEFAULT_TIMEZONE', secretKeys: { diff --git a/config/default.cjs b/config/default.cjs index c43bb0a..59f4a59 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -7,7 +7,23 @@ module.exports = { active: true }, screenshotTimeout: 20000, + // for pages that do not call window.triggerCapture: network idle, then visual quiet, capped + wait: { + networkIdleTime: 500, + quietTime: 200, + maxSettleTime: 2000 + }, + // PNG encoding runs on the main thread of the browser, shared by all the captures: the large + // ones are encoded fast (2-3x quicker, 1.1-1.5x bigger files) + pngFastEncodingPixels: 1500000, concurrency: 5, + // captures waiting for a slot: beyond maxWaiting, or after maxWaitTime of waiting, they are + // answered by a 503. data-fair gives up after 60s without a response and retries once, the wait + // plus the render should stay below + queue: { + maxWaiting: 100, + maxWaitTime: 30000 + }, inspectConcurrency: 1, // concurrent /inspect renders, each one holds a browser page defaultLang: 'fr-FR', defaultTimezone: 'Europe/Paris', @@ -16,11 +32,14 @@ module.exports = { inspect: null // required on /inspect, which is disabled without it }, maxAnimationFrames: 1800, // 2 minutes at 15fps + gifThreads: 2, // worker threads encoding the animated captures, also the max number of gifs encoded at the same time + // chrome-headless-shell (Debian's chromium-headless-shell package in the image): the "old" + // headless Chromium, without the browser UI layer. Compared to google-chrome-stable it uses half + // the memory and 20-45% less CPU per capture (cf benchmark/FINDINGS.md) puppeteerLaunchOptions: { - executablePath: '/usr/bin/google-chrome-stable', - // args: ['--use-gl=egl', '--use-angle=swiftshader', '--in-process-gpu'], + executablePath: '/usr/lib/chromium/chromium-headless-shell', args: [], - headless: true + headless: 'shell' }, observer: { active: true, diff --git a/config/development.cjs b/config/development.cjs index f85bfcf..0616888 100644 --- a/config/development.cjs +++ b/config/development.cjs @@ -8,6 +8,12 @@ module.exports = { capture: 'capture', inspect: 'inspect' }, + // the chrome-headless-shell downloaded by puppeteer (npx puppeteer browsers install + // chrome-headless-shell). Outside of a container, AppArmor (Ubuntu >= 23.10) forbids its sandbox + puppeteerLaunchOptions: { + executablePath: null, + args: ['--no-sandbox'] + }, observer: { active: false }, diff --git a/config/test.cjs b/config/test.cjs index 6231010..df5911b 100644 --- a/config/test.cjs +++ b/config/test.cjs @@ -9,6 +9,12 @@ module.exports = { capture: 'capture', inspect: 'inspect' }, + // the chrome-headless-shell downloaded by puppeteer (npx puppeteer browsers install + // chrome-headless-shell). Outside of a container, AppArmor (Ubuntu >= 23.10) forbids its sandbox + puppeteerLaunchOptions: { + executablePath: null, + args: ['--no-sandbox'] + }, observer: { active: false }, diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index 5023df9..ff3114a 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -22,13 +22,51 @@ export type ApiConfig = { [k: string]: unknown; }; concurrency: number; + /** + * Captures waiting for a browser context + */ + queue: { + /** + * beyond this number of waiting captures, new ones are answered by a 503 + */ + maxWaiting: number; + /** + * milliseconds of waiting for a context before a 503 + */ + maxWaitTime: number; + }; inspectConcurrency: number; defaultLang: string; defaultTimezone: string; onlySameHost?: boolean; useHostHeader?: boolean; screenshotTimeout: number; + /** + * How long a page that does not call window.triggerCapture is waited for + */ + wait: { + /** + * milliseconds without any network request before the page is considered loaded + */ + networkIdleTime: number; + /** + * then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture + */ + quietTime: number; + /** + * the longest wait for that quiet, after the network is idle + */ + maxSettleTime: number; + }; + /** + * above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size + */ + pngFastEncodingPixels: number; maxAnimationFrames: number; + /** + * worker threads encoding the animated captures, also the max number of gifs encoded at the same time + */ + gifThreads: number; util?: unknown; get?: unknown; has?: unknown; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index 156d33f..8201c3d 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","queue","defaultLang","defaultTimezone","screenshotTimeout","wait","pngFastEncodingPixels","maxAnimationFrames","gifThreads"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"queue":{"type":"object","description":"Captures waiting for a browser context","additionalProperties":false,"required":["maxWaiting","maxWaitTime"],"properties":{"maxWaiting":{"type":"integer","minimum":0,"description":"beyond this number of waiting captures, new ones are answered by a 503"},"maxWaitTime":{"type":"number","description":"milliseconds of waiting for a context before a 503"}}},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"wait":{"type":"object","description":"How long a page that does not call window.triggerCapture is waited for","additionalProperties":false,"required":["networkIdleTime","quietTime","maxSettleTime"],"properties":{"networkIdleTime":{"type":"number","description":"milliseconds without any network request before the page is considered loaded"},"quietTime":{"type":"number","description":"then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture"},"maxSettleTime":{"type":"number","description":"the longest wait for that quiet, after the network is idle"}}},"pngFastEncodingPixels":{"type":"number","description":"above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size"},"maxAnimationFrames":{"type":"number"},"gifThreads":{"type":"integer","minimum":1,"description":"worker threads encoding the animated captures, also the max number of gifs encoded at the same time"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -83,8 +83,8 @@ vErrors.push(err6); } errors++; } -if(data.defaultLang === undefined){ -const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; +if(data.queue === undefined){ +const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "queue"},message:"must have required property '"+"queue"+"'"}; if(vErrors === null){ vErrors = [err7]; } @@ -93,8 +93,8 @@ vErrors.push(err7); } errors++; } -if(data.defaultTimezone === undefined){ -const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; +if(data.defaultLang === undefined){ +const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; if(vErrors === null){ vErrors = [err8]; } @@ -103,8 +103,8 @@ vErrors.push(err8); } errors++; } -if(data.screenshotTimeout === undefined){ -const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; +if(data.defaultTimezone === undefined){ +const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; if(vErrors === null){ vErrors = [err9]; } @@ -113,8 +113,8 @@ vErrors.push(err9); } errors++; } -if(data.maxAnimationFrames === undefined){ -const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(data.screenshotTimeout === undefined){ +const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; if(vErrors === null){ vErrors = [err10]; } @@ -123,9 +123,8 @@ vErrors.push(err10); } errors++; } -for(const key0 in data){ -if(!(func2.call(schema16.properties, key0))){ -const err11 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(data.wait === undefined){ +const err11 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "wait"},message:"must have required property '"+"wait"+"'"}; if(vErrors === null){ vErrors = [err11]; } @@ -134,6 +133,47 @@ vErrors.push(err11); } errors++; } +if(data.pngFastEncodingPixels === undefined){ +const err12 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "pngFastEncodingPixels"},message:"must have required property '"+"pngFastEncodingPixels"+"'"}; +if(vErrors === null){ +vErrors = [err12]; +} +else { +vErrors.push(err12); +} +errors++; +} +if(data.maxAnimationFrames === undefined){ +const err13 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(vErrors === null){ +vErrors = [err13]; +} +else { +vErrors.push(err13); +} +errors++; +} +if(data.gifThreads === undefined){ +const err14 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "gifThreads"},message:"must have required property '"+"gifThreads"+"'"}; +if(vErrors === null){ +vErrors = [err14]; +} +else { +vErrors.push(err14); +} +errors++; +} +for(const key0 in data){ +if(!(func2.call(schema16.properties, key0))){ +const err15 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err15]; +} +else { +vErrors.push(err15); +} +errors++; +} } if(data.port !== undefined){ let data0 = data.port; @@ -153,12 +193,12 @@ if(dataType0 == "boolean" || data0 === null coerced0 = +data0; } else { -const err12 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err16 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err12]; +vErrors = [err16]; } else { -vErrors.push(err12); +vErrors.push(err16); } errors++; } @@ -191,12 +231,12 @@ else if(data1 === null){ coerced1 = ""; } else { -const err13 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err17 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err13]; +vErrors = [err17]; } else { -vErrors.push(err13); +vErrors.push(err17); } errors++; } @@ -229,12 +269,12 @@ else if(data2 === null){ coerced2 = ""; } else { -const err14 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err18 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err14]; +vErrors = [err18]; } else { -vErrors.push(err14); +vErrors.push(err18); } errors++; } @@ -248,12 +288,12 @@ data["privateDirectoryUrl"] = coerced2; } if(typeof data2 === "string"){ if(!pattern0.test(data2)){ -const err15 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; +const err19 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; if(vErrors === null){ -vErrors = [err15]; +vErrors = [err19]; } else { -vErrors.push(err15); +vErrors.push(err19); } errors++; } @@ -263,23 +303,23 @@ if(data.helmet !== undefined){ let data3 = data.helmet; if(data3 && typeof data3 == "object" && !Array.isArray(data3)){ if(data3.active === undefined){ -const err16 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; +const err20 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; if(vErrors === null){ -vErrors = [err16]; +vErrors = [err20]; } else { -vErrors.push(err16); +vErrors.push(err20); } errors++; } for(const key1 in data3){ if(!(key1 === "active")){ -const err17 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; +const err21 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err17]; +vErrors = [err21]; } else { -vErrors.push(err17); +vErrors.push(err21); } errors++; } @@ -304,12 +344,12 @@ else if(data4 === "true" || data4 === 1){ coerced3 = true; } else { -const err18 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err22 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err18]; +vErrors = [err22]; } else { -vErrors.push(err18); +vErrors.push(err22); } errors++; } @@ -324,12 +364,12 @@ data3["active"] = coerced3; } } else { -const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err23 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err19]; +vErrors = [err23]; } else { -vErrors.push(err19); +vErrors.push(err23); } errors++; } @@ -338,23 +378,23 @@ if(data.secretKeys !== undefined){ let data5 = data.secretKeys; if(data5 && typeof data5 == "object" && !Array.isArray(data5)){ if(data5.capture === undefined){ -const err20 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; +const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; if(vErrors === null){ -vErrors = [err20]; +vErrors = [err24]; } else { -vErrors.push(err20); +vErrors.push(err24); } errors++; } for(const key2 in data5){ if(!((key2 === "capture") || (key2 === "inspect"))){ -const err21 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; +const err25 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err21]; +vErrors = [err25]; } else { -vErrors.push(err21); +vErrors.push(err25); } errors++; } @@ -379,12 +419,12 @@ else if(data6 === null){ coerced4 = ""; } else { -const err22 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err26 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err22]; +vErrors = [err26]; } else { -vErrors.push(err22); +vErrors.push(err26); } errors++; } @@ -417,12 +457,12 @@ else if(data7 === null){ coerced5 = ""; } else { -const err23 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err27 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err23]; +vErrors = [err27]; } else { -vErrors.push(err23); +vErrors.push(err27); } errors++; } @@ -437,12 +477,12 @@ data5["inspect"] = coerced5; } } else { -const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err28 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err24]; +vErrors = [err28]; } else { -vErrors.push(err24); +vErrors.push(err28); } errors++; } @@ -470,12 +510,12 @@ else if(data9 === "true" || data9 === 1){ coerced6 = true; } else { -const err25 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err29 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err25]; +vErrors = [err29]; } else { -vErrors.push(err25); +vErrors.push(err29); } errors++; } @@ -506,12 +546,12 @@ if(dataType7 == "boolean" || data10 === null coerced7 = +data10; } else { -const err26 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err30 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err26]; +vErrors = [err30]; } else { -vErrors.push(err26); +vErrors.push(err30); } errors++; } @@ -526,12 +566,12 @@ data8["port"] = coerced7; } } else { -const err27 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err31 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err27]; +vErrors = [err31]; } else { -vErrors.push(err27); +vErrors.push(err31); } errors++; } @@ -539,12 +579,12 @@ errors++; if(data.puppeteerLaunchOptions !== undefined){ let data11 = data.puppeteerLaunchOptions; if(!(data11 && typeof data11 == "object" && !Array.isArray(data11))){ -const err28 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err32 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err28]; +vErrors = [err32]; } else { -vErrors.push(err28); +vErrors.push(err32); } errors++; } @@ -567,12 +607,12 @@ if(dataType8 === "boolean" || data12 === null coerced8 = +data12; } else { -const err29 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err33 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err29]; +vErrors = [err33]; } else { -vErrors.push(err29); +vErrors.push(err33); } errors++; } @@ -585,286 +625,666 @@ data["concurrency"] = coerced8; } } } -if(data.inspectConcurrency !== undefined){ -let data13 = data.inspectConcurrency; -if(!((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13)))){ -let dataType9 = typeof data13; +if(data.queue !== undefined){ +let data13 = data.queue; +if(data13 && typeof data13 == "object" && !Array.isArray(data13)){ +if(data13.maxWaiting === undefined){ +const err34 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaiting"},message:"must have required property '"+"maxWaiting"+"'"}; +if(vErrors === null){ +vErrors = [err34]; +} +else { +vErrors.push(err34); +} +errors++; +} +if(data13.maxWaitTime === undefined){ +const err35 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaitTime"},message:"must have required property '"+"maxWaitTime"+"'"}; +if(vErrors === null){ +vErrors = [err35]; +} +else { +vErrors.push(err35); +} +errors++; +} +for(const key3 in data13){ +if(!((key3 === "maxWaiting") || (key3 === "maxWaitTime"))){ +const err36 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key3},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err36]; +} +else { +vErrors.push(err36); +} +errors++; +} +} +if(data13.maxWaiting !== undefined){ +let data14 = data13.maxWaiting; +if(!((typeof data14 == "number") && (!(data14 % 1) && !isNaN(data14)))){ +let dataType9 = typeof data14; let coerced9 = undefined; -if(dataType9 == 'object' && Array.isArray(data13) && data13.length == 1){ -data13 = data13[0]; -dataType9 = typeof data13; -if((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13))){ -coerced9 = data13; +if(dataType9 == 'object' && Array.isArray(data14) && data14.length == 1){ +data14 = data14[0]; +dataType9 = typeof data14; +if((typeof data14 == "number") && (!(data14 % 1) && !isNaN(data14))){ +coerced9 = data14; } } if(!(coerced9 !== undefined)){ -if(dataType9 === "boolean" || data13 === null - || (dataType9 === "string" && data13 && data13 == +data13 && !(data13 % 1))){ -coerced9 = +data13; +if(dataType9 === "boolean" || data14 === null + || (dataType9 === "string" && data14 && data14 == +data14 && !(data14 % 1))){ +coerced9 = +data14; } else { -const err30 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err37 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err30]; +vErrors = [err37]; } else { -vErrors.push(err30); +vErrors.push(err37); } errors++; } } if(coerced9 !== undefined){ -data13 = coerced9; -if(data !== undefined){ -data["inspectConcurrency"] = coerced9; +data14 = coerced9; +if(data13 !== undefined){ +data13["maxWaiting"] = coerced9; } } } -if(typeof data13 == "number"){ -if(data13 < 1 || isNaN(data13)){ -const err31 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(typeof data14 == "number"){ +if(data14 < 0 || isNaN(data14)){ +const err38 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/minimum",keyword:"minimum",params:{comparison: ">=", limit: 0},message:"must be >= 0"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err38]; } else { -vErrors.push(err31); +vErrors.push(err38); } errors++; } } } -if(data.defaultLang !== undefined){ -let data14 = data.defaultLang; -if(typeof data14 !== "string"){ -let dataType10 = typeof data14; +if(data13.maxWaitTime !== undefined){ +let data15 = data13.maxWaitTime; +if(!(typeof data15 == "number")){ +let dataType10 = typeof data15; let coerced10 = undefined; -if(dataType10 == 'object' && Array.isArray(data14) && data14.length == 1){ -data14 = data14[0]; -dataType10 = typeof data14; -if(typeof data14 === "string"){ -coerced10 = data14; +if(dataType10 == 'object' && Array.isArray(data15) && data15.length == 1){ +data15 = data15[0]; +dataType10 = typeof data15; +if(typeof data15 == "number"){ +coerced10 = data15; } } if(!(coerced10 !== undefined)){ -if(dataType10 == "number" || dataType10 == "boolean"){ -coerced10 = "" + data14; -} -else if(data14 === null){ -coerced10 = ""; +if(dataType10 == "boolean" || data15 === null + || (dataType10 == "string" && data15 && data15 == +data15)){ +coerced10 = +data15; } else { -const err32 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err39 = {instancePath:instancePath+"/queue/maxWaitTime",schemaPath:"#/properties/queue/properties/maxWaitTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err32]; +vErrors = [err39]; } else { -vErrors.push(err32); +vErrors.push(err39); } errors++; } } if(coerced10 !== undefined){ -data14 = coerced10; -if(data !== undefined){ -data["defaultLang"] = coerced10; +data15 = coerced10; +if(data13 !== undefined){ +data13["maxWaitTime"] = coerced10; } } } } -if(data.defaultTimezone !== undefined){ -let data15 = data.defaultTimezone; -if(typeof data15 !== "string"){ -let dataType11 = typeof data15; +} +else { +const err40 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err40]; +} +else { +vErrors.push(err40); +} +errors++; +} +} +if(data.inspectConcurrency !== undefined){ +let data16 = data.inspectConcurrency; +if(!((typeof data16 == "number") && (!(data16 % 1) && !isNaN(data16)))){ +let dataType11 = typeof data16; let coerced11 = undefined; -if(dataType11 == 'object' && Array.isArray(data15) && data15.length == 1){ -data15 = data15[0]; -dataType11 = typeof data15; -if(typeof data15 === "string"){ -coerced11 = data15; +if(dataType11 == 'object' && Array.isArray(data16) && data16.length == 1){ +data16 = data16[0]; +dataType11 = typeof data16; +if((typeof data16 == "number") && (!(data16 % 1) && !isNaN(data16))){ +coerced11 = data16; } } if(!(coerced11 !== undefined)){ -if(dataType11 == "number" || dataType11 == "boolean"){ -coerced11 = "" + data15; -} -else if(data15 === null){ -coerced11 = ""; +if(dataType11 === "boolean" || data16 === null + || (dataType11 === "string" && data16 && data16 == +data16 && !(data16 % 1))){ +coerced11 = +data16; } else { -const err33 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err41 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err33]; +vErrors = [err41]; } else { -vErrors.push(err33); +vErrors.push(err41); } errors++; } } if(coerced11 !== undefined){ -data15 = coerced11; +data16 = coerced11; if(data !== undefined){ -data["defaultTimezone"] = coerced11; +data["inspectConcurrency"] = coerced11; } } } +if(typeof data16 == "number"){ +if(data16 < 1 || isNaN(data16)){ +const err42 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err42]; } -if(data.onlySameHost !== undefined){ -let data16 = data.onlySameHost; -if(typeof data16 !== "boolean"){ -let dataType12 = typeof data16; +else { +vErrors.push(err42); +} +errors++; +} +} +} +if(data.defaultLang !== undefined){ +let data17 = data.defaultLang; +if(typeof data17 !== "string"){ +let dataType12 = typeof data17; let coerced12 = undefined; -if(dataType12 == 'object' && Array.isArray(data16) && data16.length == 1){ -data16 = data16[0]; -dataType12 = typeof data16; -if(typeof data16 === "boolean"){ -coerced12 = data16; +if(dataType12 == 'object' && Array.isArray(data17) && data17.length == 1){ +data17 = data17[0]; +dataType12 = typeof data17; +if(typeof data17 === "string"){ +coerced12 = data17; } } if(!(coerced12 !== undefined)){ -if(data16 === "false" || data16 === 0 || data16 === null){ -coerced12 = false; +if(dataType12 == "number" || dataType12 == "boolean"){ +coerced12 = "" + data17; } -else if(data16 === "true" || data16 === 1){ -coerced12 = true; +else if(data17 === null){ +coerced12 = ""; } else { -const err34 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err43 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err34]; +vErrors = [err43]; } else { -vErrors.push(err34); +vErrors.push(err43); } errors++; } } if(coerced12 !== undefined){ -data16 = coerced12; +data17 = coerced12; if(data !== undefined){ -data["onlySameHost"] = coerced12; +data["defaultLang"] = coerced12; } } } } -if(data.useHostHeader !== undefined){ -let data17 = data.useHostHeader; -if(typeof data17 !== "boolean"){ -let dataType13 = typeof data17; +if(data.defaultTimezone !== undefined){ +let data18 = data.defaultTimezone; +if(typeof data18 !== "string"){ +let dataType13 = typeof data18; let coerced13 = undefined; -if(dataType13 == 'object' && Array.isArray(data17) && data17.length == 1){ -data17 = data17[0]; -dataType13 = typeof data17; -if(typeof data17 === "boolean"){ -coerced13 = data17; +if(dataType13 == 'object' && Array.isArray(data18) && data18.length == 1){ +data18 = data18[0]; +dataType13 = typeof data18; +if(typeof data18 === "string"){ +coerced13 = data18; } } if(!(coerced13 !== undefined)){ -if(data17 === "false" || data17 === 0 || data17 === null){ -coerced13 = false; +if(dataType13 == "number" || dataType13 == "boolean"){ +coerced13 = "" + data18; } -else if(data17 === "true" || data17 === 1){ -coerced13 = true; +else if(data18 === null){ +coerced13 = ""; } else { -const err35 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err44 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err35]; +vErrors = [err44]; } else { -vErrors.push(err35); +vErrors.push(err44); } errors++; } } if(coerced13 !== undefined){ -data17 = coerced13; +data18 = coerced13; if(data !== undefined){ -data["useHostHeader"] = coerced13; +data["defaultTimezone"] = coerced13; } } } } -if(data.screenshotTimeout !== undefined){ -let data18 = data.screenshotTimeout; -if(!(typeof data18 == "number")){ -let dataType14 = typeof data18; +if(data.onlySameHost !== undefined){ +let data19 = data.onlySameHost; +if(typeof data19 !== "boolean"){ +let dataType14 = typeof data19; let coerced14 = undefined; -if(dataType14 == 'object' && Array.isArray(data18) && data18.length == 1){ -data18 = data18[0]; -dataType14 = typeof data18; -if(typeof data18 == "number"){ -coerced14 = data18; +if(dataType14 == 'object' && Array.isArray(data19) && data19.length == 1){ +data19 = data19[0]; +dataType14 = typeof data19; +if(typeof data19 === "boolean"){ +coerced14 = data19; } } if(!(coerced14 !== undefined)){ -if(dataType14 == "boolean" || data18 === null - || (dataType14 == "string" && data18 && data18 == +data18)){ -coerced14 = +data18; +if(data19 === "false" || data19 === 0 || data19 === null){ +coerced14 = false; +} +else if(data19 === "true" || data19 === 1){ +coerced14 = true; } else { -const err36 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err45 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err36]; +vErrors = [err45]; } else { -vErrors.push(err36); +vErrors.push(err45); } errors++; } } if(coerced14 !== undefined){ -data18 = coerced14; +data19 = coerced14; if(data !== undefined){ -data["screenshotTimeout"] = coerced14; +data["onlySameHost"] = coerced14; } } } } -if(data.maxAnimationFrames !== undefined){ -let data19 = data.maxAnimationFrames; -if(!(typeof data19 == "number")){ -let dataType15 = typeof data19; +if(data.useHostHeader !== undefined){ +let data20 = data.useHostHeader; +if(typeof data20 !== "boolean"){ +let dataType15 = typeof data20; let coerced15 = undefined; -if(dataType15 == 'object' && Array.isArray(data19) && data19.length == 1){ -data19 = data19[0]; -dataType15 = typeof data19; -if(typeof data19 == "number"){ -coerced15 = data19; +if(dataType15 == 'object' && Array.isArray(data20) && data20.length == 1){ +data20 = data20[0]; +dataType15 = typeof data20; +if(typeof data20 === "boolean"){ +coerced15 = data20; } } if(!(coerced15 !== undefined)){ -if(dataType15 == "boolean" || data19 === null - || (dataType15 == "string" && data19 && data19 == +data19)){ -coerced15 = +data19; +if(data20 === "false" || data20 === 0 || data20 === null){ +coerced15 = false; +} +else if(data20 === "true" || data20 === 1){ +coerced15 = true; } else { -const err37 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err46 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err37]; +vErrors = [err46]; } else { -vErrors.push(err37); +vErrors.push(err46); } errors++; } } if(coerced15 !== undefined){ -data19 = coerced15; +data20 = coerced15; if(data !== undefined){ -data["maxAnimationFrames"] = coerced15; +data["useHostHeader"] = coerced15; } } } } +if(data.screenshotTimeout !== undefined){ +let data21 = data.screenshotTimeout; +if(!(typeof data21 == "number")){ +let dataType16 = typeof data21; +let coerced16 = undefined; +if(dataType16 == 'object' && Array.isArray(data21) && data21.length == 1){ +data21 = data21[0]; +dataType16 = typeof data21; +if(typeof data21 == "number"){ +coerced16 = data21; +} +} +if(!(coerced16 !== undefined)){ +if(dataType16 == "boolean" || data21 === null + || (dataType16 == "string" && data21 && data21 == +data21)){ +coerced16 = +data21; } else { -const err38 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err47 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err38]; +vErrors = [err47]; } else { -vErrors.push(err38); +vErrors.push(err47); +} +errors++; +} +} +if(coerced16 !== undefined){ +data21 = coerced16; +if(data !== undefined){ +data["screenshotTimeout"] = coerced16; +} +} +} +} +if(data.wait !== undefined){ +let data22 = data.wait; +if(data22 && typeof data22 == "object" && !Array.isArray(data22)){ +if(data22.networkIdleTime === undefined){ +const err48 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "networkIdleTime"},message:"must have required property '"+"networkIdleTime"+"'"}; +if(vErrors === null){ +vErrors = [err48]; +} +else { +vErrors.push(err48); +} +errors++; +} +if(data22.quietTime === undefined){ +const err49 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "quietTime"},message:"must have required property '"+"quietTime"+"'"}; +if(vErrors === null){ +vErrors = [err49]; +} +else { +vErrors.push(err49); +} +errors++; +} +if(data22.maxSettleTime === undefined){ +const err50 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "maxSettleTime"},message:"must have required property '"+"maxSettleTime"+"'"}; +if(vErrors === null){ +vErrors = [err50]; +} +else { +vErrors.push(err50); +} +errors++; +} +for(const key4 in data22){ +if(!(((key4 === "networkIdleTime") || (key4 === "quietTime")) || (key4 === "maxSettleTime"))){ +const err51 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key4},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err51]; +} +else { +vErrors.push(err51); +} +errors++; +} +} +if(data22.networkIdleTime !== undefined){ +let data23 = data22.networkIdleTime; +if(!(typeof data23 == "number")){ +let dataType17 = typeof data23; +let coerced17 = undefined; +if(dataType17 == 'object' && Array.isArray(data23) && data23.length == 1){ +data23 = data23[0]; +dataType17 = typeof data23; +if(typeof data23 == "number"){ +coerced17 = data23; +} +} +if(!(coerced17 !== undefined)){ +if(dataType17 == "boolean" || data23 === null + || (dataType17 == "string" && data23 && data23 == +data23)){ +coerced17 = +data23; +} +else { +const err52 = {instancePath:instancePath+"/wait/networkIdleTime",schemaPath:"#/properties/wait/properties/networkIdleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err52]; +} +else { +vErrors.push(err52); +} +errors++; +} +} +if(coerced17 !== undefined){ +data23 = coerced17; +if(data22 !== undefined){ +data22["networkIdleTime"] = coerced17; +} +} +} +} +if(data22.quietTime !== undefined){ +let data24 = data22.quietTime; +if(!(typeof data24 == "number")){ +let dataType18 = typeof data24; +let coerced18 = undefined; +if(dataType18 == 'object' && Array.isArray(data24) && data24.length == 1){ +data24 = data24[0]; +dataType18 = typeof data24; +if(typeof data24 == "number"){ +coerced18 = data24; +} +} +if(!(coerced18 !== undefined)){ +if(dataType18 == "boolean" || data24 === null + || (dataType18 == "string" && data24 && data24 == +data24)){ +coerced18 = +data24; +} +else { +const err53 = {instancePath:instancePath+"/wait/quietTime",schemaPath:"#/properties/wait/properties/quietTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err53]; +} +else { +vErrors.push(err53); +} +errors++; +} +} +if(coerced18 !== undefined){ +data24 = coerced18; +if(data22 !== undefined){ +data22["quietTime"] = coerced18; +} +} +} +} +if(data22.maxSettleTime !== undefined){ +let data25 = data22.maxSettleTime; +if(!(typeof data25 == "number")){ +let dataType19 = typeof data25; +let coerced19 = undefined; +if(dataType19 == 'object' && Array.isArray(data25) && data25.length == 1){ +data25 = data25[0]; +dataType19 = typeof data25; +if(typeof data25 == "number"){ +coerced19 = data25; +} +} +if(!(coerced19 !== undefined)){ +if(dataType19 == "boolean" || data25 === null + || (dataType19 == "string" && data25 && data25 == +data25)){ +coerced19 = +data25; +} +else { +const err54 = {instancePath:instancePath+"/wait/maxSettleTime",schemaPath:"#/properties/wait/properties/maxSettleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err54]; +} +else { +vErrors.push(err54); +} +errors++; +} +} +if(coerced19 !== undefined){ +data25 = coerced19; +if(data22 !== undefined){ +data22["maxSettleTime"] = coerced19; +} +} +} +} +} +else { +const err55 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err55]; +} +else { +vErrors.push(err55); +} +errors++; +} +} +if(data.pngFastEncodingPixels !== undefined){ +let data26 = data.pngFastEncodingPixels; +if(!(typeof data26 == "number")){ +let dataType20 = typeof data26; +let coerced20 = undefined; +if(dataType20 == 'object' && Array.isArray(data26) && data26.length == 1){ +data26 = data26[0]; +dataType20 = typeof data26; +if(typeof data26 == "number"){ +coerced20 = data26; +} +} +if(!(coerced20 !== undefined)){ +if(dataType20 == "boolean" || data26 === null + || (dataType20 == "string" && data26 && data26 == +data26)){ +coerced20 = +data26; +} +else { +const err56 = {instancePath:instancePath+"/pngFastEncodingPixels",schemaPath:"#/properties/pngFastEncodingPixels/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err56]; +} +else { +vErrors.push(err56); +} +errors++; +} +} +if(coerced20 !== undefined){ +data26 = coerced20; +if(data !== undefined){ +data["pngFastEncodingPixels"] = coerced20; +} +} +} +} +if(data.maxAnimationFrames !== undefined){ +let data27 = data.maxAnimationFrames; +if(!(typeof data27 == "number")){ +let dataType21 = typeof data27; +let coerced21 = undefined; +if(dataType21 == 'object' && Array.isArray(data27) && data27.length == 1){ +data27 = data27[0]; +dataType21 = typeof data27; +if(typeof data27 == "number"){ +coerced21 = data27; +} +} +if(!(coerced21 !== undefined)){ +if(dataType21 == "boolean" || data27 === null + || (dataType21 == "string" && data27 && data27 == +data27)){ +coerced21 = +data27; +} +else { +const err57 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err57]; +} +else { +vErrors.push(err57); +} +errors++; +} +} +if(coerced21 !== undefined){ +data27 = coerced21; +if(data !== undefined){ +data["maxAnimationFrames"] = coerced21; +} +} +} +} +if(data.gifThreads !== undefined){ +let data28 = data.gifThreads; +if(!((typeof data28 == "number") && (!(data28 % 1) && !isNaN(data28)))){ +let dataType22 = typeof data28; +let coerced22 = undefined; +if(dataType22 == 'object' && Array.isArray(data28) && data28.length == 1){ +data28 = data28[0]; +dataType22 = typeof data28; +if((typeof data28 == "number") && (!(data28 % 1) && !isNaN(data28))){ +coerced22 = data28; +} +} +if(!(coerced22 !== undefined)){ +if(dataType22 === "boolean" || data28 === null + || (dataType22 === "string" && data28 && data28 == +data28 && !(data28 % 1))){ +coerced22 = +data28; +} +else { +const err58 = {instancePath:instancePath+"/gifThreads",schemaPath:"#/properties/gifThreads/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +if(vErrors === null){ +vErrors = [err58]; +} +else { +vErrors.push(err58); +} +errors++; +} +} +if(coerced22 !== undefined){ +data28 = coerced22; +if(data !== undefined){ +data["gifThreads"] = coerced22; +} +} +} +if(typeof data28 == "number"){ +if(data28 < 1 || isNaN(data28)){ +const err59 = {instancePath:instancePath+"/gifThreads",schemaPath:"#/properties/gifThreads/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err59]; +} +else { +vErrors.push(err59); +} +errors++; +} +} +} +} +else { +const err60 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err60]; +} +else { +vErrors.push(err60); } errors++; } diff --git a/config/type/schema.json b/config/type/schema.json index a6e24bc..391195e 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -18,10 +18,14 @@ "puppeteerLaunchOptions", "concurrency", "inspectConcurrency", + "queue", "defaultLang", "defaultTimezone", "screenshotTimeout", - "maxAnimationFrames" + "wait", + "pngFastEncodingPixels", + "maxAnimationFrames", + "gifThreads" ], "properties": { "port": { @@ -74,6 +78,23 @@ "concurrency": { "type": "integer" }, + "queue": { + "type": "object", + "description": "Captures waiting for a browser context", + "additionalProperties": false, + "required": ["maxWaiting", "maxWaitTime"], + "properties": { + "maxWaiting": { + "type": "integer", + "minimum": 0, + "description": "beyond this number of waiting captures, new ones are answered by a 503" + }, + "maxWaitTime": { + "type": "number", + "description": "milliseconds of waiting for a context before a 503" + } + } + }, "inspectConcurrency": { "type": "integer", "minimum": 1 @@ -93,9 +114,38 @@ "screenshotTimeout": { "type": "number" }, + "wait": { + "type": "object", + "description": "How long a page that does not call window.triggerCapture is waited for", + "additionalProperties": false, + "required": ["networkIdleTime", "quietTime", "maxSettleTime"], + "properties": { + "networkIdleTime": { + "type": "number", + "description": "milliseconds without any network request before the page is considered loaded" + }, + "quietTime": { + "type": "number", + "description": "then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture" + }, + "maxSettleTime": { + "type": "number", + "description": "the longest wait for that quiet, after the network is idle" + } + } + }, + "pngFastEncodingPixels": { + "type": "number", + "description": "above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size" + }, "maxAnimationFrames": { "type": "number" }, + "gifThreads": { + "type": "integer", + "minimum": 1, + "description": "worker threads encoding the animated captures, also the max number of gifs encoded at the same time" + }, "util": {}, "get": {}, "has": {} diff --git a/package-lock.json b/package-lock.json index 245717a..cd484c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,14 +17,13 @@ "cookie-parser": "^1.4.7", "debug": "^4.4.3", "express": "^5.3.0", - "generic-pool": "^3.9.0", "get-pixels": "github:scijs/get-pixels#90e79d5786b13aadefe5427cdc23cf1f164ec637", "gif-encoder": "^0.7.2", "helmet": "^8.1.0", "http-terminator": "^3.2.0", + "piscina": "^5.3.2", "prom-client": "^15.1.3", - "puppeteer": "^25.16.0", - "tmp-promise": "^3.0.3" + "puppeteer": "^25.16.0" }, "devDependencies": { "@commitlint/cli": "^21.2.4", @@ -1009,6 +1008,332 @@ "node": ">= 10" } }, + "node_modules/@napi-rs/nice": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice/-/nice-1.1.1.tgz", + "integrity": "sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/nice-android-arm-eabi": "1.1.1", + "@napi-rs/nice-android-arm64": "1.1.1", + "@napi-rs/nice-darwin-arm64": "1.1.1", + "@napi-rs/nice-darwin-x64": "1.1.1", + "@napi-rs/nice-freebsd-x64": "1.1.1", + "@napi-rs/nice-linux-arm-gnueabihf": "1.1.1", + "@napi-rs/nice-linux-arm64-gnu": "1.1.1", + "@napi-rs/nice-linux-arm64-musl": "1.1.1", + "@napi-rs/nice-linux-ppc64-gnu": "1.1.1", + "@napi-rs/nice-linux-riscv64-gnu": "1.1.1", + "@napi-rs/nice-linux-s390x-gnu": "1.1.1", + "@napi-rs/nice-linux-x64-gnu": "1.1.1", + "@napi-rs/nice-linux-x64-musl": "1.1.1", + "@napi-rs/nice-openharmony-arm64": "1.1.1", + "@napi-rs/nice-win32-arm64-msvc": "1.1.1", + "@napi-rs/nice-win32-ia32-msvc": "1.1.1", + "@napi-rs/nice-win32-x64-msvc": "1.1.1" + } + }, + "node_modules/@napi-rs/nice-android-arm-eabi": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-android-arm-eabi/-/nice-android-arm-eabi-1.1.1.tgz", + "integrity": "sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-android-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-android-arm64/-/nice-android-arm64-1.1.1.tgz", + "integrity": "sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-darwin-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-darwin-arm64/-/nice-darwin-arm64-1.1.1.tgz", + "integrity": "sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-darwin-x64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-darwin-x64/-/nice-darwin-x64-1.1.1.tgz", + "integrity": "sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-freebsd-x64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-freebsd-x64/-/nice-freebsd-x64-1.1.1.tgz", + "integrity": "sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-arm-gnueabihf": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm-gnueabihf/-/nice-linux-arm-gnueabihf-1.1.1.tgz", + "integrity": "sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-arm64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm64-gnu/-/nice-linux-arm64-gnu-1.1.1.tgz", + "integrity": "sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-arm64-musl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm64-musl/-/nice-linux-arm64-musl-1.1.1.tgz", + "integrity": "sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-ppc64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-ppc64-gnu/-/nice-linux-ppc64-gnu-1.1.1.tgz", + "integrity": "sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-riscv64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-riscv64-gnu/-/nice-linux-riscv64-gnu-1.1.1.tgz", + "integrity": "sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-s390x-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-s390x-gnu/-/nice-linux-s390x-gnu-1.1.1.tgz", + "integrity": "sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-x64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-x64-gnu/-/nice-linux-x64-gnu-1.1.1.tgz", + "integrity": "sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-x64-musl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-x64-musl/-/nice-linux-x64-musl-1.1.1.tgz", + "integrity": "sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-openharmony-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-openharmony-arm64/-/nice-openharmony-arm64-1.1.1.tgz", + "integrity": "sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-win32-arm64-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-arm64-msvc/-/nice-win32-arm64-msvc-1.1.1.tgz", + "integrity": "sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-win32-ia32-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-ia32-msvc/-/nice-win32-ia32-msvc-1.1.1.tgz", + "integrity": "sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-win32-x64-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-x64-msvc/-/nice-win32-x64-msvc-1.1.1.tgz", + "integrity": "sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@opentelemetry/api": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.10.0.tgz", @@ -3118,15 +3443,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/generic-pool": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", - "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4663,6 +4979,18 @@ "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", "license": "MIT" }, + "node_modules/piscina": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/piscina/-/piscina-5.3.2.tgz", + "integrity": "sha512-vv7l/mM7B9WgrhDaudIqv1x6v4LOBRYYmpWGdWushhpbCmVHp11sUB2v4hj1CAeTGXVxuZFz6xOU1RalAiPKfQ==", + "license": "MIT", + "engines": { + "node": ">=20.x" + }, + "optionalDependencies": { + "@napi-rs/nice": "^1.0.4" + } + }, "node_modules/pngjs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", @@ -5398,24 +5726,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tmp": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", - "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", - "license": "MIT", - "engines": { - "node": ">=14.14" - } - }, - "node_modules/tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "license": "MIT", - "dependencies": { - "tmp": "^0.2.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/package.json b/package.json index b2e0c0a..61c0455 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint-fix": "eslint --fix .", "check-types": "tsc", "quality": "npm run lint && npm run build-types && npm run check-types && npm run test && npm audit --omit=dev --audit-level=critical", - "prepare": "husky || true" + "prepare": "husky || true", + "benchmark": "node --disable-warning=ExperimentalWarning benchmark/index.ts" }, "imports": { "#config": "./api/config.ts" @@ -45,14 +46,13 @@ "cookie-parser": "^1.4.7", "debug": "^4.4.3", "express": "^5.3.0", - "generic-pool": "^3.9.0", "get-pixels": "github:scijs/get-pixels#90e79d5786b13aadefe5427cdc23cf1f164ec637", "gif-encoder": "^0.7.2", "helmet": "^8.1.0", "http-terminator": "^3.2.0", + "piscina": "^5.3.2", "prom-client": "^15.1.3", - "puppeteer": "^25.16.0", - "tmp-promise": "^3.0.3" + "puppeteer": "^25.16.0" }, "devDependencies": { "@commitlint/cli": "^21.2.4", diff --git a/test-it/capture.ts b/test-it/capture.ts index fcd2863..6bdffb0 100644 --- a/test-it/capture.ts +++ b/test-it/capture.ts @@ -58,6 +58,58 @@ describe('capture', () => { assert.equal(res.headers['content-disposition'], 'attachment; filename=screenshot.png') }) + it('return jpeg images, and png ones when a gif is asked from a page without animation', async () => { + let res = await ax.get('screenshot', { params: { key, target, type: 'jpg', filename: 'screenshot.jpg' }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.equal(res.headers['content-type'], 'image/jpeg') + assert.equal(res.headers['content-disposition'], 'attachment; filename=screenshot.jpg') + assert.deepEqual([...new Uint8Array(res.data).subarray(0, 3)], [0xff, 0xd8, 0xff]) + + // test1.html calls triggerCapture() without declaring animation support + res = await ax.get('screenshot', { params: { key, target, type: 'gif', filename: 'screenshot.gif' }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.equal(res.headers['content-type'], 'image/png') + assert.equal(res.headers['content-disposition'], 'attachment; filename=screenshot.png') + assert.equal(Buffer.from(res.data).subarray(1, 4).toString(), 'PNG') + }) + + it('wait for a page without triggerCapture to settle', async () => { + const settleTarget = origin + '/test/resources/test-settle.html' + const res = await ax.get('print', { params: { key, target: settleTarget, type: 'html' } }) + assert.equal(res.status, 200) + assert.ok(res.data.includes('animation done'), 'the capture was taken before the end of the animation') + + const times = (await ax.get('screenshot', { params: { key, target: origin + '/test/resources/test-clean.html', timer: 'true' } })).data + assert.ok('wait1-network-idle' in times) + assert.ok('wait2-settle' in times) + }) + + it('isolate captures from each other: nothing is left in the storages of the page', async () => { + const storageTarget = origin + '/test/resources/test-storage.html' + // sequential captures, so that a reused browser context would be reused here + for (const mark of ['user-a', 'user-b', 'user-c']) { + const res = await ax.get('print', { params: { key, target: storageTarget + '?mark=' + mark, type: 'html' } }) + assert.equal(res.status, 200) + assert.ok(res.data.includes('previous=null'), `a previous capture left a mark: ${res.data.match(/previous=[^<]*/)?.[0]}`) + } + }) + + it('drop the captures of clients that left', async () => { + // more abandoned captures than browser contexts (concurrency 5) of a page that is never idle, + // each would hold a context for the whole screenshot timeout (2s in tests) + const busyTarget = origin + '/test/resources/test-busy.html' + const abandoned = [] + for (let i = 0; i < 15; i++) { + abandoned.push(ax.get('screenshot', { params: { key, target: busyTarget }, timeout: 300 }).catch(err => err.code)) + } + assert.ok((await Promise.all(abandoned)).every(code => code === 'ECONNABORTED')) + // 3 rounds of 5 abandoned captures would delay this one by ~6s if they were rendered + const t0 = Date.now() + const res = await ax.get('screenshot', { params: { key, target }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.ok(Date.now() - t0 < 3500, `the capture waited ${Date.now() - t0}ms behind abandoned ones`) + }) + it('make a few prints of a page', async () => { let res = await ax.get('print', { params: { key, target }, responseType: 'arraybuffer' }) assert.equal(res.status, 200) @@ -158,6 +210,12 @@ describe('capture', () => { const size1 = Number(res.headers['content-length']) assert.ok(size1 < 100000) + // a small gif is encoded in a slice of the Buffer pool of the worker, it must be returned anyway + res = await ax.get('screenshot', { params: { key, target: origin + '/test/resources/test-anim.html', type: 'gif', width: 100, height: 50 }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.equal(res.headers['content-type'], 'image/gif') + assert.equal(Buffer.from(res.data).subarray(0, 3).toString(), 'GIF') + res = await ax.get('screenshot', { params: { key, target: origin + '/test/resources/test-anim.html', filename: 'test.gif' } }) assert.equal(res.status, 200) assert.equal(res.headers['content-type'], 'image/gif') diff --git a/test-it/resources/test-settle.html b/test-it/resources/test-settle.html new file mode 100644 index 0000000..507c99c --- /dev/null +++ b/test-it/resources/test-settle.html @@ -0,0 +1,28 @@ + + + + + Settle page + + +

Settle page

+ +

loading

+ + + diff --git a/test-it/resources/test-storage.html b/test-it/resources/test-storage.html new file mode 100644 index 0000000..3e1bc13 --- /dev/null +++ b/test-it/resources/test-storage.html @@ -0,0 +1,18 @@ + + + + + Storage page + + +

Storage page

+

+ + + diff --git a/test-it/semaphore.ts b/test-it/semaphore.ts index 2a4e28b..5356fa5 100644 --- a/test-it/semaphore.ts +++ b/test-it/semaphore.ts @@ -49,4 +49,49 @@ describe('semaphore', () => { assert.equal(await p2, 'next') assert.equal(sem.pending(), 0) }) + + it('rejects beyond maxWaiting, without disturbing the queue', async () => { + const sem = createSemaphore(1, { maxWaiting: 1 }) + const first = deferred() + const p1 = sem.run(() => first.promise) + const p2 = sem.run(async () => 'second') + await assert.rejects(sem.run(async () => 'third'), (err: any) => err.code === 'queue-full') + assert.equal(sem.pending(), 1) + first.resolve('first') + assert.equal(await p1, 'first') + assert.equal(await p2, 'second') + }) + + it('rejects a call that waited longer than maxWaitTime, and takes it out of the queue', async () => { + const sem = createSemaphore(1, { maxWaitTime: 50 }) + const first = deferred() + const p1 = sem.run(() => first.promise) + let lateStarted = false + await assert.rejects(sem.run(async () => { lateStarted = true }), (err: any) => err.code === 'queue-timeout') + assert.equal(sem.pending(), 0) + first.resolve('first') + await p1 + await tick() + assert.equal(lateStarted, false) + assert.equal(sem.active(), 0) + }) + + it('rejects an aborted call, before or while it waits', async () => { + const sem = createSemaphore(1) + const first = deferred() + const p1 = sem.run(() => first.promise) + const controller = new AbortController() + let abortedStarted = false + const p2 = sem.run(async () => { abortedStarted = true }, controller.signal) + await tick() + controller.abort() + await assert.rejects(p2, (err: any) => err.code === 'aborted') + assert.equal(sem.pending(), 0) + await assert.rejects(sem.run(async () => {}, controller.signal), (err: any) => err.code === 'aborted') + first.resolve('first') + await p1 + await tick() + assert.equal(abortedStarted, false) + assert.equal(sem.active(), 0) + }) })