From af6bce5d44aa9896b80266e9b17f6d53ab03e977 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Tue, 10 Nov 2026 19:46:50 +0100 Subject: [PATCH 1/3] ui: add @ggml-org/llama-harness, the agentic loop as a package, in ui/lib/harness Llama Harness is the agentic loop as headless TypeScript with no UI, no DOM and no dependencies. It streams a turn, runs the tools the model called, gives it the results, and repeats until the model answers without a tool call. An app gives it an inference client and tools, and reads the run as a stream of events: run(messages, tools, signal). It ships a client for any OpenAI-compatible endpoint, so it also runs on its own in Node. The UI workflow type-checks, tests and packs it. Assisted-by: Claude (Fable 5.2) --- .github/workflows/ui.yml | 31 + ui/lib/harness/.gitignore | 6 + ui/lib/harness/.npmrc | 1 + ui/lib/harness/LICENSE | 21 + ui/lib/harness/README.md | 64 + ui/lib/harness/package-lock.json | 1255 ++++++++++++++++++++ ui/lib/harness/package.json | 39 + ui/lib/harness/src/abort.ts | 17 + ui/lib/harness/src/harness.ts | 375 ++++++ ui/lib/harness/src/index.ts | 4 + ui/lib/harness/src/openai-client.ts | 143 +++ ui/lib/harness/src/types.ts | 209 ++++ ui/lib/harness/tests/harness.test.ts | 316 +++++ ui/lib/harness/tests/openai-client.test.ts | 130 ++ ui/lib/harness/tsconfig.build.json | 10 + ui/lib/harness/tsconfig.json | 13 + 16 files changed, 2634 insertions(+) create mode 100644 ui/lib/harness/.gitignore create mode 100644 ui/lib/harness/.npmrc create mode 100644 ui/lib/harness/LICENSE create mode 100644 ui/lib/harness/README.md create mode 100644 ui/lib/harness/package-lock.json create mode 100644 ui/lib/harness/package.json create mode 100644 ui/lib/harness/src/abort.ts create mode 100644 ui/lib/harness/src/harness.ts create mode 100644 ui/lib/harness/src/index.ts create mode 100644 ui/lib/harness/src/openai-client.ts create mode 100644 ui/lib/harness/src/types.ts create mode 100644 ui/lib/harness/tests/harness.test.ts create mode 100644 ui/lib/harness/tests/openai-client.test.ts create mode 100644 ui/lib/harness/tsconfig.build.json create mode 100644 ui/lib/harness/tsconfig.json diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 10d7ed0ed5e4..11875ba53c5c 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -150,6 +150,37 @@ jobs: run: npm run prepack working-directory: ui/lib/ui + harness-checks: + name: Harness checks + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Install dependencies + run: npm ci + working-directory: ui/lib/harness + + - name: Run type checking + run: npm run check + working-directory: ui/lib/harness + + - name: Run tests + run: npm test + working-directory: ui/lib/harness + + - name: Build the package + run: npm run prepack + working-directory: ui/lib/harness + e2e-tests: name: E2E Tests needs: ui-build diff --git a/ui/lib/harness/.gitignore b/ui/lib/harness/.gitignore new file mode 100644 index 000000000000..16339ab2e87f --- /dev/null +++ b/ui/lib/harness/.gitignore @@ -0,0 +1,6 @@ +node_modules +/dist + +# OS +.DS_Store +Thumbs.db diff --git a/ui/lib/harness/.npmrc b/ui/lib/harness/.npmrc new file mode 100644 index 000000000000..b6f27f135954 --- /dev/null +++ b/ui/lib/harness/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/ui/lib/harness/LICENSE b/ui/lib/harness/LICENSE new file mode 100644 index 000000000000..50c59c51c158 --- /dev/null +++ b/ui/lib/harness/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) ggml + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ui/lib/harness/README.md b/ui/lib/harness/README.md new file mode 100644 index 000000000000..9994a0caa058 --- /dev/null +++ b/ui/lib/harness/README.md @@ -0,0 +1,64 @@ +# @ggml-org/llama-harness + +Llama Harness, the agentic loop of Llama Chat, as a package: it streams a turn, runs the tools the model called, gives it the results, and repeats until the model answers without a tool call. It is headless TypeScript with no UI, no DOM and no dependencies, so the same loop runs in a browser, in Node and in a worker. It lives in llama.cpp at `ui/lib/harness`. + +It needs an OpenAI-compatible chat completions endpoint, not llama-server specifically. What llama-server adds (timings, prompt progress) it reads when it is there. + +## Use + +```ts +import { createHarness, createOpenAIClient, type Tool } from "@ggml-org/llama-harness"; + +const harness = createHarness(createOpenAIClient({ baseUrl: "http://localhost:8080" })); + +const getTime: Tool = { + definition: { + type: "function", + function: { + name: "get_time", + description: "The current time", + parameters: { type: "object", properties: {} }, + }, + }, + execute: async () => ({ content: new Date().toISOString() }), +}; + +for await (const event of harness.run([{ role: "user", content: "What time is it?" }], [getTime])) { + if (event.type === "text") process.stdout.write(event.delta); +} +``` + +## The contract + +```ts +interface Harness { + run(messages, tools, signal?, options?): AsyncIterable; +} +``` + +- **Messages and tools** use the OpenAI shapes. A tool is its definition plus an `execute` function. +- **Events:** `turn_start`, `text`, `reasoning`, `tool_call_delta`, `timings`, `model`, `completion_id`, `message` (a turn's assistant message), `tool_call`, `tool_update` (a streaming tool's output so far), `tool_result`, `turn_end`, and last `done` or `error`. `run` never throws. +- **The loop waits for you.** It does not go on until you ask for the next event, so you can store each message before the next step runs. +- **It keeps nothing.** No conversation state, no storage: the app owns the history and passes it in. +- **Stopping:** `signal` stops at once. `options.interrupted` ends the run at the next safe point, e.g. when the user sent a new message. `options.maxTurns` with `options.onTurnLimit` asks before it goes past a turn limit. + +An app can put another loop behind the same `Harness` interface. What sits above the loop is the same for every loop: the tools, and whatever an app wraps around `execute`, such as a permission prompt. + +## What the app keeps + +Llama Chat (`ui/apps/chat`) is the first app on it. Its agentic store gives the harness: + +- **An inference client:** its own chat service, which knows the servers, the settings and how to resume a stream. +- **The tools:** built-in tools, llama-server tools and MCP tools, each behind the permission gate. +- **Storage:** it turns the events into messages in its database. + +## Develop + +```sh +npm ci +npm run check # type check +npm test # unit tests +npm run prepack # build dist/ and lint the package +``` + +Llama Chat does not need a build of this package: it compiles `src/` directly, through the `@ggml-org/llama-harness` alias in its `svelte.config.js`. diff --git a/ui/lib/harness/package-lock.json b/ui/lib/harness/package-lock.json new file mode 100644 index 000000000000..952987aff8df --- /dev/null +++ b/ui/lib/harness/package-lock.json @@ -0,0 +1,1255 @@ +{ + "name": "@ggml-org/llama-harness", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@ggml-org/llama-harness", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^24", + "publint": "^0.3.21", + "typescript": "^6.0.3", + "vitest": "5.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.156.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.156.0.tgz", + "integrity": "sha512-1Duyg48TdJNdyhEB8Xn1L36bT9Pv6GUtlKjHxOys8xkaF/rQq3XJE+bAurSK/AgIz5mGFsb77cn9Pb20+WCScw==", + "dev": true, + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@publint/pack": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/@publint/pack/-/pack-0.1.7.tgz", + "integrity": "sha512-4EDEmvxWtgsCnnVeBvtFIFZtUhPPt1+bA9JrSwU4Sa//6oKtzCSlGGXYJr44OD9aGISymbieJ4mCKHUygUDU+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyexec": "^1.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://bjornlu.com/sponsor" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.18.tgz", + "integrity": "sha512-ubTG+4KlidvioxjOFdsdEa2F6JVGZb8m2ByJUqQ7fYzUeT1cEn194POcsQ10PKH4p//q0YQn4SB3YpSuzcA1Yg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.18.tgz", + "integrity": "sha512-2Xa1t/JeFGTZn5TbzgPppcCftF8sCVnTNvrIvoSdcZhoFzAuVy1L3xxQUQ4sfWDtwVzg1u6Yi1R3SlJugRNV2Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.18.tgz", + "integrity": "sha512-iUzw9HshVvrWci9gVXjxWS7/M/IcLvNo67vu2c/w9zKBz4hIvPRixdUpIW/iCfhldLaq9ER2hecsI9UY5z0/yA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.18.tgz", + "integrity": "sha512-XuDHFzbFEtZl0QGJi92+fmG67tfnp1iZ9VOeDkwasGyTq2VG3/5PnJnvAr+MTzFh/rNScJ3VhN+XpZyyGsN7Jw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.18.tgz", + "integrity": "sha512-MM8opLfsLK15oEXDz6iBc0t3ahu+ew9jsAmha+ovZ33hSwYgqo7YmpRITqFTfW7xA71BLEQ3jJdTsaCmhB1c9w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.18.tgz", + "integrity": "sha512-J6EoCWiy5RnFQzHrKyHXXevZ9lukQfGpnchX/zG/k/WC2nc9O9XyS1JgA8kI9VxMFggMbCPjpv7LcikriXfehw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.18.tgz", + "integrity": "sha512-HIkFuLF7CdDeki+1aCZLLbLq0hk/PHntb+z4/YCTo47JbksTD6jJEELIz92tYROmDmvMrNqUMqBi71BkJLDR+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.18.tgz", + "integrity": "sha512-4oqWItLLYt7IceOnElCct8T2ek5p7PmD7LN04TRJRXGo5f3HtKTgGH6PhYRFK4C2ICkq/LMUU0BVPvvlAKhm8g==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.18.tgz", + "integrity": "sha512-+AkRly5SrCEgnPhGRfub7pucASYEC2fTsxZg7VCcnozpA2+Wn/gCYy1t7LuJS67/xyJu/7noYBJncDUkbwgw2A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.18.tgz", + "integrity": "sha512-Ttg1r9vVykMnQrE/GwYO9icKtXiuiMfRe/w74xJkeoVuvhRG+WtT6LEMOf58Yl63yd77lhHzO8Y55R8CC3XALA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.18.tgz", + "integrity": "sha512-xX1Tt4o5jUMma0vJsXp5eRZA9nOuyigb2dKJMMzZRmzMxLYGE9PaKKSqHPzPvSQgM8ZOEacSTQwDQcwWfh0xmQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.18.tgz", + "integrity": "sha512-4VVlGI5K2kociZlW3lSN8VjG3TW5yQIlAcfCdBXMgGjJxZll15Fa4ocRHXHvIVVnwlibCw/NQajJrNDiqqpiiw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.18.tgz", + "integrity": "sha512-PApTsgH1jLYuyy3tSCz9De82Gq5qXwi7vImVZhEzTgX1C4150Cp5M2ZfL3pLvbCvbEHL0oJeC6pd6podtv8Y5w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.18.tgz", + "integrity": "sha512-J2XjWjx/SP+4x0H4r6dz3Vzb5WVkg7re40Do9K7HJ3sK2/xcmZrWwakXlSwaMvOkvruLJCD4CL/3wqsIR/thgg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.18.tgz", + "integrity": "sha512-XwRJxRQvkuEtS0hIIt29QrqPJ+JzMJSNYHJw4ae/mc6uEarfRG1ceyg4HmmWiBdL/YCb6g7N5BLbSvmRJRYZHw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.1.0.tgz", + "integrity": "sha512-K+NU2HDCEem4sClgHH4ZZXwdJeNFkJo+hRk479DIwdIviAPN2hFJH1KHHwg8CxHQXvi0M5XY4deTXINfIiSG6w==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@vitest/mocker": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.5.tgz", + "integrity": "sha512-xLL3lkDegJZw+MGoGaCCsFayDZthGgpAtr3Vziyfc4p7xH89K7KRPcRBBh/kEYm0Aqkt5DtTmkYpBtRV/FrHZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", + "@vitest/spy": "5.0.5", + "estree-walker": "^3.0.3", + "magic-string": "^1.4.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9 || ^3.0.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/spy": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.5.tgz", + "integrity": "sha512-lwJRExuH417257Q5wv9QYugN9IMJkoxcrKrYxGPQkBzl7wlNpTUxD/5yNX4ESKVUIShKPgkGchYPFMWOOk0wzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz", + "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "peer": true, + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.5.tgz", + "integrity": "sha512-oLLC2+YvQUfAa+vxoufE/yc1icb87M9iXIRqAoR/2EhE/8cb3SF/+Gi0c2By1B+5Vz2e2vQtUvHiu1+63Znh+w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/obug/-/obug-3.0.0.tgz", + "integrity": "sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/package-manager-detector": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.9.0.tgz", + "integrity": "sha512-zAMfbta9nPA8AKkA4WdmNAT8R8s6Dk0/xwPsqNnRidMFcLo1YkLuPDTgs2f64XdzS8vbLNnoJ+uUIkgbz7oRHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.32.tgz", + "integrity": "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/publint": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/publint/-/publint-0.3.25.tgz", + "integrity": "sha512-WMMbn/d7JqRlfXQL/JUI4n5skC8cJ5WVr8/wT5mmiYWkUdENkZr1I5rgs2+dQPVRgoJd+JRYtB46MTsuDylw9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@publint/pack": "^0.1.7", + "package-manager-detector": "^1.8.0", + "picocolors": "^1.1.1", + "sade": "^1.8.1" + }, + "bin": { + "publint": "src/cli.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://bjornlu.com/sponsor" + } + }, + "node_modules/rolldown": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.18.tgz", + "integrity": "sha512-C1uuT3UR60SShuCaPQFAAZyefJCWZlAKB02BV3BuPE2Rs4fevRQltEKoFO2tV5W9/eJhU5RieV6vJj3euulfNQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@oxc-project/types": "=0.156.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.18", + "@rolldown/binding-android-arm64": "1.2.18", + "@rolldown/binding-darwin-arm64": "1.2.18", + "@rolldown/binding-darwin-x64": "1.2.18", + "@rolldown/binding-freebsd-x64": "1.2.18", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.18", + "@rolldown/binding-linux-arm64-gnu": "1.2.18", + "@rolldown/binding-linux-arm64-musl": "1.2.18", + "@rolldown/binding-linux-ppc64-gnu": "1.2.18", + "@rolldown/binding-linux-s390x-gnu": "1.2.18", + "@rolldown/binding-linux-x64-gnu": "1.2.18", + "@rolldown/binding-linux-x64-musl": "1.2.18", + "@rolldown/binding-openharmony-arm64": "1.2.18", + "@rolldown/binding-win32-arm64-msvc": "1.2.18", + "@rolldown/binding-win32-x64-msvc": "1.2.18" + } + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/std-env": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.3.0.tgz", + "integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.3.tgz", + "integrity": "sha512-Al01C2JIBnO2CXCAPbpAl6sUiOZHeThGtHYNTa51JRw9PyWOCdr+9Ong05R4qg/Sk3+H645E8ov1h2c58ityNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "node_modules/vite": { + "version": "8.3.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.7.tgz", + "integrity": "sha512-oxh0dMxrayBr33ScJvvBJ2HBQ2jc1TBrXKR2NyL/MJ9WPuX19mN2DXWUmywKdyw5RFVQBo8hmtb2FMC0LHrnng==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.32", + "rolldown": "~1.2.18", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1 || ^0.8.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.5.tgz", + "integrity": "sha512-cPODOMF8YnkiiYtDdHo6n4wWmBrOBBbOIYH2d33M4l3CdNn72h+qMmO6uXffUN6A5AZFXNND/d+krS9mgkXRfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/mocker": "5.0.5", + "chai": "^6.3.0", + "es-module-lexer": "^2.3.2", + "expect-type": "^1.4.0", + "magic-string": "^1.4.3", + "obug": "^3.0.0", + "picomatch": "^4.0.7", + "std-env": "^4.3.0", + "tinybench": "^6.2.2", + "tinyexec": "^1.3.1", + "tinyglobby": "^0.2.17", + "why-is-node-running": "3.2.1" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^22.12.0 || ^24.0.0 || >=26.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "5.0.5", + "@vitest/browser-preview": "5.0.5", + "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", + "@vitest/coverage-istanbul": "5.0.5", + "@vitest/coverage-v8": "5.0.5", + "@vitest/ui": "5.0.5", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-3.2.1.tgz", + "integrity": "sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==", + "dev": true, + "license": "MIT", + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=20.11" + } + } + } +} diff --git a/ui/lib/harness/package.json b/ui/lib/harness/package.json new file mode 100644 index 000000000000..b93deffb207d --- /dev/null +++ b/ui/lib/harness/package.json @@ -0,0 +1,39 @@ +{ + "name": "@ggml-org/llama-harness", + "version": "0.1.0", + "description": "Headless agentic loop for llama.cpp frontend apps.", + "homepage": "https://github.com/ggml-org/llama.cpp/tree/master/ui/lib/harness", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/ggml-org/llama.cpp.git", + "directory": "ui/lib/harness" + }, + "files": [ + "dist" + ], + "type": "module", + "sideEffects": false, + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "prepack": "npm run build && publint", + "test": "vitest --run" + }, + "devDependencies": { + "@types/node": "^24", + "publint": "^0.3.21", + "typescript": "^6.0.3", + "vitest": "5.0.5" + }, + "engines": { + "node": ">=20" + } +} diff --git a/ui/lib/harness/src/abort.ts b/ui/lib/harness/src/abort.ts new file mode 100644 index 000000000000..636dd21ed718 --- /dev/null +++ b/ui/lib/harness/src/abort.ts @@ -0,0 +1,17 @@ +const ABORT_ERROR_NAME = "AbortError"; +// TypeErrors a browser throws when a fetch stream is cut by page unload, navigation or a network drop +const ABORT_LIKE_MESSAGE_PATTERNS = [ + /input stream/i, + /network connection was lost/i, + /load failed/i, +]; + +export function isAbortError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + if (error.name === ABORT_ERROR_NAME) return true; + + return ( + error instanceof TypeError && ABORT_LIKE_MESSAGE_PATTERNS.some((re) => re.test(error.message)) + ); +} diff --git a/ui/lib/harness/src/harness.ts b/ui/lib/harness/src/harness.ts new file mode 100644 index 000000000000..e58e7ea25188 --- /dev/null +++ b/ui/lib/harness/src/harness.ts @@ -0,0 +1,375 @@ +import { isAbortError } from "./abort.js"; +import type { + DoneReason, + Harness, + HarnessEvent, + InferenceClient, + LlmTimings, + LlmTotals, + Message, + PartialToolCall, + RunOptions, + RunTimings, + Timings, + Tool, + ToolCall, + ToolResult, + TurnStats, +} from "./types.js"; + +/** The result of a tool call that did not run because the run was interrupted. */ +export const INTERRUPTED_TOOL_RESULT = "Tool execution was interrupted by a new user message."; + +function emptyTotals(): LlmTotals { + return { predicted_ms: 0, predicted_n: 0, prompt_ms: 0, prompt_n: 0 }; +} + +function addTimings(totals: LlmTotals, timings: LlmTimings): LlmTotals { + return { + predicted_ms: totals.predicted_ms + (timings.predicted_ms ?? 0), + predicted_n: totals.predicted_n + (timings.predicted_n ?? 0), + prompt_ms: totals.prompt_ms + (timings.prompt_ms ?? 0), + prompt_n: totals.prompt_n + (timings.prompt_n ?? 0), + }; +} + +function toToolCalls(partial: PartialToolCall[]): ToolCall[] { + return partial.map((call, index) => ({ + function: { + arguments: call?.function?.arguments ?? "", + name: call?.function?.name ?? "", + }, + id: call?.id ?? `tool_${index}`, + type: "function", + })); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * The default loop: stream a turn, run the tools the model called, give it the results, and repeat until it answers + * without a tool call. It keeps no state between runs and stores nothing: the app owns the conversation. + */ +export function createHarness(client: InferenceClient): Harness { + return { + run: (messages, tools, signal, options) => run(client, messages, tools, signal, options), + }; +} + +async function* run( + client: InferenceClient, + messages: Message[], + tools: Tool[], + signal?: AbortSignal, + options: RunOptions = {}, +): AsyncGenerator { + const { interrupted = () => false, maxTurns = Infinity, onTurnLimit } = options; + const history = [...messages]; + const definitions = tools.map((tool) => tool.definition); + const toolsByName = new Map(tools.map((tool) => [tool.definition.function.name, tool])); + const totals: RunTimings = { + llm: emptyTotals(), + perTurn: [], + toolCalls: [], + toolCallsCount: 0, + toolsMs: 0, + turns: 0, + }; + + let lastTimings: LlmTimings | undefined; + let turn = 0; + + const runTimings = (): Timings | undefined => { + if (totals.toolCallsCount === 0) return lastTimings; + + return { + agentic: totals, + cache_n: lastTimings?.cache_n, + predicted_ms: lastTimings?.predicted_ms, + predicted_n: lastTimings?.predicted_n, + prompt_ms: lastTimings?.prompt_ms, + prompt_n: lastTimings?.prompt_n, + }; + }; + const done = (reason: DoneReason): HarnessEvent => ({ + reason, + timings: runTimings(), + type: "done", + }); + + while (true) { + if (turn >= maxTurns) { + const more = onTurnLimit ? await onTurnLimit(signal) : false; + + if (!more || signal?.aborted) { + yield done(signal?.aborted ? "aborted" : "turn_limit"); + + return; + } + + turn = 0; + } + + totals.turns = turn + 1; + + if (signal?.aborted) { + yield done("aborted"); + + return; + } + + yield { turn: turn + 1, type: "turn_start" }; + + let content = ""; + let reasoning = ""; + let partialCalls: PartialToolCall[] = []; + let turnTimings: LlmTimings | undefined; + + const stats: TurnStats = { llm: emptyTotals(), toolCalls: [], toolsMs: 0, turn: turn + 1 }; + // the turn's message when the run ends after it: what streamed so far, with the run's timings + const lastMessage = (toolCalls?: ToolCall[]): HarnessEvent => ({ + content, + final: true, + reasoning: reasoning || undefined, + timings: runTimings(), + toolCalls, + type: "message", + }); + + try { + const stream = client.stream({ + messages: history, + signal, + tools: definitions.length > 0 ? definitions : undefined, + }); + + for await (const event of stream) { + if (event.type === "text") { + content += event.delta; + yield event; + } else if (event.type === "reasoning") { + reasoning += event.delta; + yield event; + } else if (event.type === "tool_calls") { + partialCalls = event.toolCalls; + yield { toolCalls: partialCalls, type: "tool_call_delta" }; + } else if (event.type === "timings") { + if (event.timings) { + lastTimings = event.timings; + turnTimings = event.timings; + } + + yield { ...event, llm: event.timings && addTimings(totals.llm, event.timings) }; + } else { + yield event; + } + } + + if (turnTimings) { + totals.llm = addTimings(totals.llm, turnTimings); + stats.llm = addTimings(stats.llm, turnTimings); + } + } catch (error) { + // keep the partial output in both cases + yield lastMessage(); + + if (signal?.aborted) { + yield done("aborted"); + } else { + yield { + error: error instanceof Error ? error : new Error("LLM stream error"), + timings: runTimings(), + type: "error", + }; + } + + return; + } + + // a client can swallow the abort and end the stream normally: do not take a cut tool call for a complete one + if (signal?.aborted) { + yield lastMessage(); + yield done("aborted"); + + return; + } + + if (interrupted()) { + yield lastMessage(partialCalls.length > 0 ? toToolCalls(partialCalls) : undefined); + yield done("interrupted"); + + return; + } + + if (partialCalls.length === 0) { + totals.perTurn.push(stats); + + const timings = runTimings(); + + yield lastMessage(); + + if (timings) yield { timings, type: "turn_end" }; + + yield done("complete"); + + return; + } + + const toolCalls = toToolCalls(partialCalls); + + yield { + content, + final: false, + reasoning: reasoning || undefined, + timings: turnTimings, + toolCalls, + type: "message", + }; + + history.push({ + content: content || undefined, + reasoning_content: reasoning || undefined, + role: "assistant", + tool_calls: toolCalls, + }); + + for (let i = 0; i < toolCalls.length; i++) { + const toolCall = toolCalls[i]; + + if (signal?.aborted) { + yield done("aborted"); + + return; + } + + if (interrupted()) { + for (const skippedCall of toolCalls.slice(i)) { + yield { + result: { content: INTERRUPTED_TOOL_RESULT }, + skipped: true, + toolCall: skippedCall, + type: "tool_result", + }; + + history.push({ + content: INTERRUPTED_TOOL_RESULT, + role: "tool", + tool_call_id: skippedCall.id, + }); + } + + break; + } + + yield { toolCall, type: "tool_call" }; + + const tool = toolsByName.get(toolCall.function.name); + const startTime = performance.now(); + + let result: ToolResult; + + if (!tool) { + result = { content: `Error: Unknown tool: ${toolCall.function.name}`, isError: true }; + } else { + // only the latest update matters, so a slow consumer skips the ones in between + let pendingUpdate: string | undefined; + let settled = false; + let wake: (() => void) | undefined; + + const notify = () => { + wake?.(); + wake = undefined; + }; + const execution = Promise.resolve() + .then(() => + tool.execute(toolCall, { + signal, + update: (updated) => { + pendingUpdate = updated; + notify(); + }, + }), + ) + .finally(() => { + settled = true; + notify(); + }); + + // the rejection is handled below, after the updates are drained + execution.catch(() => {}); + + while (true) { + if (pendingUpdate !== undefined) { + const updated = pendingUpdate; + + pendingUpdate = undefined; + yield { content: updated, toolCall, type: "tool_update" }; + } else if (settled) { + break; + } else { + await new Promise((resolve) => (wake = resolve)); + } + } + + try { + result = await execution; + } catch (error) { + if (isAbortError(error)) { + yield done("aborted"); + + return; + } + + result = { content: `Error: ${errorMessage(error)}`, isError: true }; + } + } + + const durationMs = Math.round(result.durationMs ?? performance.now() - startTime); + const timing = { + duration_ms: durationMs, + name: toolCall.function.name, + success: !result.isError, + }; + + totals.toolCalls.push(timing); + totals.toolCallsCount++; + totals.toolsMs += durationMs; + stats.toolCalls.push(timing); + stats.toolsMs += durationMs; + + if (signal?.aborted) { + yield done("aborted"); + + return; + } + + yield { result, skipped: false, toolCall, type: "tool_result" }; + + history.push({ + content: result.parts?.length + ? [{ text: result.content, type: "text" }, ...result.parts] + : result.content, + role: "tool", + tool_call_id: toolCall.id, + }); + } + + if (stats.toolCalls.length > 0) { + totals.perTurn.push(stats); + + const timings = runTimings(); + + if (timings) yield { timings, type: "turn_end" }; + } + + if (interrupted()) { + yield done("interrupted"); + + return; + } + + turn++; + } +} diff --git a/ui/lib/harness/src/index.ts b/ui/lib/harness/src/index.ts new file mode 100644 index 000000000000..e3096ef332bb --- /dev/null +++ b/ui/lib/harness/src/index.ts @@ -0,0 +1,4 @@ +export { isAbortError } from "./abort.js"; +export { createHarness, INTERRUPTED_TOOL_RESULT } from "./harness.js"; +export { createOpenAIClient, type OpenAIClientOptions } from "./openai-client.js"; +export type * from "./types.js"; diff --git a/ui/lib/harness/src/openai-client.ts b/ui/lib/harness/src/openai-client.ts new file mode 100644 index 000000000000..48413af26cfd --- /dev/null +++ b/ui/lib/harness/src/openai-client.ts @@ -0,0 +1,143 @@ +import type { InferenceClient, InferenceEvent, PartialToolCall } from "./types.js"; + +const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; +const SSE_DATA_PREFIX = "data:"; +const SSE_DONE = "[DONE]"; + +export interface OpenAIClientOptions { + /** Server root, e.g. `http://localhost:8080`. */ + baseUrl: string; + apiKey?: string; + model?: string; + /** Extra fields of the request body, e.g. `temperature`. */ + params?: Record; + headers?: Record; + /** Default: the global fetch. */ + fetch?: typeof fetch; +} + +interface ToolCallDelta extends PartialToolCall { + index?: number; +} + +function mergeToolCallDeltas(toolCalls: PartialToolCall[], deltas: ToolCallDelta[]): void { + for (const delta of deltas) { + // without an index, a delta with an id opens a call and any other one continues the last call + const index = + delta.index ?? (delta.id || toolCalls.length === 0 ? toolCalls.length : toolCalls.length - 1); + const call = (toolCalls[index] ??= { function: { arguments: "", name: "" } }); + + if (delta.id) call.id = delta.id; + + if (delta.type) call.type = delta.type; + + call.function = { + arguments: (call.function?.arguments ?? "") + (delta.function?.arguments ?? ""), + name: (call.function?.name ?? "") + (delta.function?.name ?? ""), + }; + } +} + +async function errorFromResponse(response: Response): Promise { + const body = await response.text().catch(() => ""); + + let message = body; + + try { + message = JSON.parse(body).error?.message ?? body; + } catch { + // not JSON: keep the body text + } + + return new Error(`HTTP ${response.status}${message ? `: ${message}` : ""}`); +} + +/** + * An inference client for any OpenAI-compatible endpoint, llama-server included. It reads the `timings` and + * `prompt_progress` fields that llama-server adds and works without them. + */ +export function createOpenAIClient(options: OpenAIClientOptions): InferenceClient { + const { apiKey, baseUrl, headers, model, params } = options; + const url = baseUrl.replace(/\/+$/, "") + CHAT_COMPLETIONS_PATH; + + return { + async *stream({ messages, signal, tools }): AsyncGenerator { + const response = await (options.fetch ?? fetch)(url, { + body: JSON.stringify({ ...params, messages, model, stream: true, tools }), + headers: { + "Content-Type": "application/json", + ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + ...headers, + }, + method: "POST", + signal, + }); + + if (!response.ok) throw await errorFromResponse(response); + + if (!response.body) throw new Error("The response has no body"); + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const toolCalls: PartialToolCall[] = []; + + let buffer = ""; + let completionId: string | undefined; + let modelName: string | undefined; + + try { + while (true) { + const { done, value } = await reader.read(); + + if (done) return; + + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split("\n"); + + buffer = lines.pop() ?? ""; + + for (const line of lines) { + if (!line.startsWith(SSE_DATA_PREFIX)) continue; + + const data = line.slice(SSE_DATA_PREFIX.length).trim(); + + if (data === SSE_DONE) return; + + const chunk = JSON.parse(data); + + if (chunk.error) throw new Error(chunk.error.message ?? "Stream error"); + + if (chunk.id && chunk.id !== completionId) { + completionId = chunk.id; + yield { id: chunk.id, type: "completion_id" }; + } + + if (chunk.model && chunk.model !== modelName) { + modelName = chunk.model; + yield { model: chunk.model, type: "model" }; + } + + const delta = chunk.choices?.[0]?.delta; + const reasoning = delta?.reasoning_content ?? delta?.reasoning; + + if (reasoning) yield { delta: reasoning, type: "reasoning" }; + + if (delta?.content) yield { delta: delta.content, type: "text" }; + + if (delta?.tool_calls?.length) { + mergeToolCallDeltas(toolCalls, delta.tool_calls); + yield { toolCalls: toolCalls.map((call) => ({ ...call })), type: "tool_calls" }; + } + + if (chunk.timings || chunk.prompt_progress) { + yield { progress: chunk.prompt_progress, timings: chunk.timings, type: "timings" }; + } + } + } + } finally { + reader.cancel().catch(() => {}); + } + }, + }; +} diff --git a/ui/lib/harness/src/types.ts b/ui/lib/harness/src/types.ts new file mode 100644 index 000000000000..7050ce0ba329 --- /dev/null +++ b/ui/lib/harness/src/types.ts @@ -0,0 +1,209 @@ +// Messages and tools use the OpenAI chat-completions shapes, so they go to the endpoint as they are. + +export interface TextPart { + type: "text"; + text: string; +} + +export interface ImagePart { + type: "image_url"; + image_url: { url: string }; +} + +export interface AudioPart { + type: "input_audio"; + input_audio: { data: string; format: string }; +} + +export type ContentPart = TextPart | ImagePart | AudioPart; + +export interface ToolCall { + id: string; + type: "function"; + function: { name: string; arguments: string }; +} + +/** A tool call as it streams in: any field can still be missing. */ +export interface PartialToolCall { + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; +} + +export type Message = + | { role: "system" | "user"; content: string | ContentPart[] } + | { + role: "assistant"; + content?: string | ContentPart[]; + reasoning_content?: string; + tool_calls?: ToolCall[]; + } + | { role: "tool"; tool_call_id: string; content: string | ContentPart[] }; + +export interface ToolDefinition { + type: "function"; + function: { + name: string; + description?: string; + parameters?: Record; + }; +} + +export interface ToolContext { + signal?: AbortSignal; + /** Report the output so far of a tool that streams. Each call replaces the previous content. */ + update(content: string): void; +} + +export interface ToolResult { + /** Text the model reads. */ + content: string; + /** Media the model reads after the text, e.g. an image the tool produced. */ + parts?: ContentPart[]; + isError?: boolean; + /** Set it when the tool times itself, e.g. to leave out a wait for approval. Default: the time execute took. */ + durationMs?: number; + /** App data the loop does not read. It comes back on the tool_result event. */ + details?: unknown; +} + +export interface Tool { + definition: ToolDefinition; + /** Run the call. Reject with an AbortError to end the run, with any other error to report it to the model. */ + execute(call: ToolCall, context: ToolContext): Promise; +} + +/** Token counts and times of one request, as llama-server reports them. */ +export interface LlmTimings { + cache_n?: number; + predicted_ms?: number; + predicted_n?: number; + prompt_ms?: number; + prompt_n?: number; +} + +export interface LlmTotals { + predicted_ms: number; + predicted_n: number; + prompt_ms: number; + prompt_n: number; +} + +export interface ToolCallTiming { + name: string; + duration_ms: number; + success: boolean; +} + +export interface TurnStats { + turn: number; + llm: LlmTotals; + toolCalls: ToolCallTiming[]; + toolsMs: number; +} + +/** Totals of a run: every turn's requests and tool calls. */ +export interface RunTimings { + turns: number; + toolCallsCount: number; + toolsMs: number; + toolCalls: ToolCallTiming[]; + perTurn: TurnStats[]; + llm: LlmTotals; +} + +/** The last request's timings, with the run's totals once a tool has run. */ +export interface Timings extends LlmTimings { + agentic?: RunTimings; +} + +export interface PromptProgress { + cache: number; + processed: number; + time_ms: number; + total: number; +} + +export interface InferenceRequest { + messages: Message[]; + tools?: ToolDefinition[]; + signal?: AbortSignal; +} + +export type InferenceEvent = + | { type: "text"; delta: string } + | { type: "reasoning"; delta: string } + /** Every tool call of the response so far, not only the new part. */ + | { type: "tool_calls"; toolCalls: PartialToolCall[] } + | { type: "timings"; timings?: LlmTimings; progress?: PromptProgress } + | { type: "model"; model: string } + | { type: "completion_id"; id: string }; + +/** + * One streamed chat completion. The loop needs nothing else from a server, so any OpenAI-compatible endpoint works. + * Timings, model and completion id are optional extras. + */ +export interface InferenceClient { + stream(request: InferenceRequest): AsyncIterable; +} + +export type DoneReason = + /** The model answered without calling a tool. */ + | "complete" + | "aborted" + /** `interrupted` returned true. */ + | "interrupted" + /** The turn limit was reached and `onTurnLimit` did not grant more turns. */ + | "turn_limit"; + +export type HarnessEvent = + /** `turn` starts at 1, and again at 1 once `onTurnLimit` grants more turns. */ + | { type: "turn_start"; turn: number } + | { type: "text"; delta: string } + | { type: "reasoning"; delta: string } + | { type: "tool_call_delta"; toolCalls: PartialToolCall[] } + /** `llm` is the run's totals including the request in flight. */ + | { type: "timings"; timings?: LlmTimings; progress?: PromptProgress; llm?: LlmTotals } + | { type: "model"; model: string } + | { type: "completion_id"; id: string } + /** The assistant message of a turn. `final` is true when the run ends after it and none of its tool calls runs. */ + | { + type: "message"; + content: string; + reasoning?: string; + toolCalls?: ToolCall[]; + timings?: Timings; + final: boolean; + } + | { type: "tool_call"; toolCall: ToolCall } + | { type: "tool_update"; toolCall: ToolCall; content: string } + /** `skipped` is true when the call did not run because the run was interrupted. */ + | { type: "tool_result"; toolCall: ToolCall; result: ToolResult; skipped: boolean } + | { type: "turn_end"; timings: Timings } + | { type: "done"; reason: DoneReason; timings?: Timings } + | { type: "error"; error: Error; timings?: Timings }; + +export interface RunOptions { + /** Turns before `onTurnLimit` is asked. Default: no limit. */ + maxTurns?: number; + /** Resolve true to run `maxTurns` more turns. Default: the run ends. */ + onTurnLimit?: (signal?: AbortSignal) => Promise; + /** + * Return true to end the run at the next safe point, e.g. when the user sent a new message. Tool calls that did + * not run yet get a result that says so. Use `signal` to stop at once. + */ + interrupted?: () => boolean; +} + +/** + * The contract an app runs a loop through. `run` never throws: it ends with a `done` or an `error` event. It waits + * for the consumer between events, so the consumer can store each one before the loop goes on. + */ +export interface Harness { + run( + messages: Message[], + tools: Tool[], + signal?: AbortSignal, + options?: RunOptions, + ): AsyncIterable; +} diff --git a/ui/lib/harness/tests/harness.test.ts b/ui/lib/harness/tests/harness.test.ts new file mode 100644 index 000000000000..4e302a14c5fe --- /dev/null +++ b/ui/lib/harness/tests/harness.test.ts @@ -0,0 +1,316 @@ +import { describe, expect, it } from "vitest"; +import { + createHarness, + INTERRUPTED_TOOL_RESULT, + type HarnessEvent, + type InferenceClient, + type InferenceEvent, + type InferenceRequest, + type Message, + type RunOptions, + type Tool, + type ToolResult, +} from "../src/index.js"; + +const USER: Message[] = [{ content: "hi", role: "user" }]; + +function callEvent(...names: string[]): InferenceEvent { + return { + toolCalls: names.map((name, index) => ({ + function: { arguments: "{}", name }, + id: `call_${index}`, + })), + type: "tool_calls", + }; +} + +// a client that plays one scripted turn per request and records what it was sent +function scriptedClient(turns: Array) { + const requests: InferenceRequest[] = []; + const client: InferenceClient = { + async *stream(request) { + requests.push({ ...request, messages: [...request.messages] }); + + const turn = turns[requests.length - 1] ?? []; + + if (turn instanceof Error) throw turn; + + yield* turn; + }, + }; + + return { client, requests }; +} + +function tool( + name: string, + execute: Tool["execute"] = async () => ({ content: `${name} ran` }), +): Tool { + return { definition: { function: { name }, type: "function" }, execute }; +} + +async function collect( + client: InferenceClient, + tools: Tool[] = [], + options?: RunOptions, + signal?: AbortSignal, +): Promise { + const events: HarnessEvent[] = []; + + for await (const event of createHarness(client).run(USER, tools, signal, options)) + events.push(event); + + return events; +} + +function types(events: HarnessEvent[]): string[] { + return events.map((event) => event.type); +} + +function last(events: HarnessEvent[]): HarnessEvent { + return events[events.length - 1]; +} + +function results(events: HarnessEvent[]) { + return events.filter((event) => event.type === "tool_result"); +} + +describe("createHarness", () => { + it("ends after a turn without tool calls", async () => { + const { client, requests } = scriptedClient([ + [ + { delta: "think", type: "reasoning" }, + { delta: "Hel", type: "text" }, + { delta: "lo", type: "text" }, + ], + ]); + const events = await collect(client); + + expect(types(events)).toEqual(["turn_start", "reasoning", "text", "text", "message", "done"]); + expect(events[4]).toMatchObject({ content: "Hello", final: true, reasoning: "think" }); + expect(last(events)).toMatchObject({ reason: "complete" }); + expect(requests[0].tools).toBeUndefined(); + }); + + it("runs the called tool and sends its result with the next request", async () => { + const { client, requests } = scriptedClient([ + [{ delta: "let me look", type: "text" }, callEvent("read")], + [{ delta: "done", type: "text" }], + ]); + const events = await collect(client, [tool("read")]); + + expect(types(events)).toEqual([ + "turn_start", + "text", + "tool_call_delta", + "message", + "tool_call", + "tool_result", + "turn_end", + "turn_start", + "text", + "message", + "turn_end", + "done", + ]); + expect(events[3]).toMatchObject({ + final: false, + toolCalls: [{ id: "call_0", type: "function" }], + }); + expect(requests[1].messages.slice(1)).toEqual([ + { + content: "let me look", + reasoning_content: undefined, + role: "assistant", + tool_calls: [ + { function: { arguments: "{}", name: "read" }, id: "call_0", type: "function" }, + ], + }, + { content: "read ran", role: "tool", tool_call_id: "call_0" }, + ]); + expect(requests[0].tools).toEqual([{ function: { name: "read" }, type: "function" }]); + }); + + it("sends a result's media parts after its text", async () => { + const image = { image_url: { url: "data:image/png;base64,AAAA" }, type: "image_url" } as const; + const { client, requests } = scriptedClient([[callEvent("shot")], []]); + + await collect(client, [tool("shot", async () => ({ content: "saved", parts: [image] }))]); + + expect(requests[1].messages[2]).toEqual({ + content: [{ text: "saved", type: "text" }, image], + role: "tool", + tool_call_id: "call_0", + }); + }); + + it("reports an unknown tool and a tool that throws to the model", async () => { + const { client } = scriptedClient([[callEvent("missing", "broken")], []]); + const events = await collect(client, [ + tool("broken", async () => { + throw new Error("boom"); + }), + ]); + + expect(results(events).map((event) => event.result)).toEqual([ + { content: "Error: Unknown tool: missing", isError: true }, + { content: "Error: boom", isError: true }, + ]); + expect(last(events)).toMatchObject({ + timings: { + agentic: { toolCalls: [{ success: false }, { success: false }], toolCallsCount: 2 }, + }, + }); + }); + + it("emits a streaming tool's updates before its result", async () => { + const { client } = scriptedClient([[callEvent("shell")], []]); + const events = await collect(client, [ + tool("shell", async (_call, { update }) => { + update(""); + await Promise.resolve(); + update("a"); + update("ab"); + + return { content: "ab" }; + }), + ]); + const updates = events.filter((event) => event.type === "tool_update"); + + // "a" was replaced before the consumer asked for it + expect(updates.map((event) => event.content)).toEqual(["", "ab"]); + expect(types(events).indexOf("tool_result")).toBeGreaterThan( + types(events).lastIndexOf("tool_update"), + ); + }); + + it("uses the duration a tool reports", async () => { + const { client } = scriptedClient([[callEvent("slow")], []]); + const events = await collect(client, [ + tool("slow", async () => ({ content: "", durationMs: 1234 })), + ]); + + expect(last(events)).toMatchObject({ timings: { agentic: { toolsMs: 1234 } } }); + }); + + it("ends without a result when a tool rejects with an AbortError", async () => { + const { client, requests } = scriptedClient([[callEvent("gated")], []]); + const events = await collect(client, [ + tool("gated", async () => { + throw new DOMException("Operation was aborted", "AbortError"); + }), + ]); + + expect(results(events)).toEqual([]); + expect(last(events)).toMatchObject({ reason: "aborted" }); + expect(requests).toHaveLength(1); + }); + + it("keeps the partial output when the signal aborts mid-stream", async () => { + const controller = new AbortController(); + const client: InferenceClient = { + async *stream() { + yield { delta: "par", type: "text" }; + controller.abort(); + throw new DOMException("Operation was aborted", "AbortError"); + }, + }; + const events = await collect(client, [], undefined, controller.signal); + + expect(types(events)).toEqual(["turn_start", "text", "message", "done"]); + expect(events[2]).toMatchObject({ content: "par", final: true }); + expect(last(events)).toMatchObject({ reason: "aborted" }); + }); + + it("ends with an error event when the stream fails", async () => { + const { client } = scriptedClient([new Error("HTTP 500")]); + const events = await collect(client); + + expect(types(events)).toEqual(["turn_start", "message", "error"]); + expect(last(events)).toMatchObject({ error: { message: "HTTP 500" } }); + }); + + it("keeps the tool calls of the turn it is interrupted after, and runs none", async () => { + const { client, requests } = scriptedClient([[callEvent("read")], []]); + const events = await collect(client, [tool("read")], { interrupted: () => true }); + + expect(types(events)).toEqual(["turn_start", "tool_call_delta", "message", "done"]); + expect(events[2]).toMatchObject({ final: true, toolCalls: [{ id: "call_0" }] }); + expect(last(events)).toMatchObject({ reason: "interrupted" }); + expect(requests).toHaveLength(1); + }); + + it("skips the tool calls left when it is interrupted between two", async () => { + let interrupted = false; + + const { client, requests } = scriptedClient([[callEvent("first", "second", "third")], []]); + const events = await collect( + client, + [ + tool("first", async (): Promise => { + interrupted = true; + + return { content: "first ran" }; + }), + tool("second"), + tool("third"), + ], + { interrupted: () => interrupted }, + ); + + expect(results(events).map((event) => [event.result.content, event.skipped])).toEqual([ + ["first ran", false], + [INTERRUPTED_TOOL_RESULT, true], + [INTERRUPTED_TOOL_RESULT, true], + ]); + expect(last(events)).toMatchObject({ reason: "interrupted" }); + expect(requests).toHaveLength(1); + }); + + it("asks at the turn limit and starts the turn count again when it goes on", async () => { + const answers = [true, false]; + const { client } = scriptedClient([[callEvent("read")], [callEvent("read")], []]); + const events = await collect(client, [tool("read")], { + maxTurns: 1, + onTurnLimit: async () => answers.shift() ?? false, + }); + const turns = events.filter((event) => event.type === "turn_start"); + + expect(turns.map((event) => event.turn)).toEqual([1, 1]); + expect(last(events)).toMatchObject({ reason: "turn_limit" }); + }); + + it("adds up the timings of every turn", async () => { + const timings = (n: number): InferenceEvent => ({ + timings: { cache_n: 1, predicted_ms: n, predicted_n: n, prompt_ms: n, prompt_n: n }, + type: "timings", + }); + const { client } = scriptedClient([[timings(10), callEvent("read")], [timings(5)]]); + const events = await collect(client, [tool("read")]); + const live = events + .filter((event) => event.type === "timings") + .map((event) => event.llm?.predicted_n); + + expect(live).toEqual([10, 15]); + expect(last(events)).toMatchObject({ + timings: { + agentic: { + llm: { predicted_ms: 15, predicted_n: 15, prompt_ms: 15, prompt_n: 15 }, + perTurn: [{ turn: 1 }, { turn: 2 }], + toolCallsCount: 1, + turns: 2, + }, + cache_n: 1, + predicted_n: 5, + }, + }); + }); + + it("reports the last request's timings alone when no tool ran", async () => { + const { client } = scriptedClient([[{ timings: { predicted_n: 3 }, type: "timings" }]]); + const events = await collect(client); + + expect(types(events)).toEqual(["turn_start", "timings", "message", "turn_end", "done"]); + expect(last(events)).toEqual({ reason: "complete", timings: { predicted_n: 3 }, type: "done" }); + }); +}); diff --git a/ui/lib/harness/tests/openai-client.test.ts b/ui/lib/harness/tests/openai-client.test.ts new file mode 100644 index 000000000000..fedcb59a0a8c --- /dev/null +++ b/ui/lib/harness/tests/openai-client.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { createOpenAIClient, type InferenceEvent } from "../src/index.js"; + +function sse(...chunks: unknown[]): string { + return chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"; +} + +// answers every request with the given body, cut into pieces to cross read boundaries +function fakeFetch(body: string, init: ResponseInit = {}, pieceLength = 7) { + const calls: Array<{ url: string; init: RequestInit }> = []; + const fetch = (async (url: string, requestInit: RequestInit) => { + calls.push({ init: requestInit, url }); + + const bytes = new TextEncoder().encode(body); + const stream = new ReadableStream({ + start(controller) { + for (let i = 0; i < bytes.length; i += pieceLength) { + controller.enqueue(bytes.slice(i, i + pieceLength)); + } + + controller.close(); + }, + }); + + return new Response(stream, init); + }) as typeof globalThis.fetch; + + return { calls, fetch }; +} + +async function collect(stream: AsyncIterable): Promise { + const events: InferenceEvent[] = []; + + for await (const event of stream) events.push(event); + + return events; +} + +const delta = (fields: object, rest: object = {}) => ({ + choices: [{ delta: fields }], + id: "cmpl-1", + model: "m", + ...rest, +}); + +describe("createOpenAIClient", () => { + it("posts a streaming request to the chat completions endpoint", async () => { + const { calls, fetch } = fakeFetch(sse()); + const client = createOpenAIClient({ + apiKey: "key", + baseUrl: "http://localhost:8080/", + fetch, + model: "m", + params: { temperature: 0 }, + }); + + await collect(client.stream({ messages: [{ content: "hi", role: "user" }] })); + + expect(calls[0].url).toBe("http://localhost:8080/v1/chat/completions"); + expect(calls[0].init.headers).toMatchObject({ Authorization: "Bearer key" }); + expect(JSON.parse(calls[0].init.body as string)).toEqual({ + messages: [{ content: "hi", role: "user" }], + model: "m", + stream: true, + temperature: 0, + }); + }); + + it("turns the chunks into events", async () => { + const timings = { predicted_n: 2 }; + const { fetch } = fakeFetch( + sse( + delta({ reasoning_content: "hm" }), + delta({ content: "Hel" }), + delta({ content: "lo" }), + delta({}, { timings }), + ), + ); + const events = await collect( + createOpenAIClient({ baseUrl: "http://x", fetch }).stream({ messages: [] }), + ); + + expect(events).toEqual([ + { id: "cmpl-1", type: "completion_id" }, + { model: "m", type: "model" }, + { delta: "hm", type: "reasoning" }, + { delta: "Hel", type: "text" }, + { delta: "lo", type: "text" }, + { progress: undefined, timings, type: "timings" }, + ]); + }); + + it("merges tool call deltas by index", async () => { + const { fetch } = fakeFetch( + sse( + delta({ + tool_calls: [ + { function: { arguments: "", name: "read" }, id: "a", index: 0, type: "function" }, + ], + }), + delta({ tool_calls: [{ function: { arguments: '{"path"' }, index: 0 }] }), + delta({ + tool_calls: [ + { function: { arguments: ':"x"}' }, index: 0 }, + { function: { arguments: "{}", name: "list" }, id: "b", index: 1, type: "function" }, + ], + }), + ), + ); + const events = await collect( + createOpenAIClient({ baseUrl: "http://x", fetch }).stream({ messages: [] }), + ); + const toolCalls = events.filter((event) => event.type === "tool_calls"); + + expect(toolCalls).toHaveLength(3); + expect(toolCalls[2].toolCalls).toEqual([ + { function: { arguments: '{"path":"x"}', name: "read" }, id: "a", type: "function" }, + { function: { arguments: "{}", name: "list" }, id: "b", type: "function" }, + ]); + }); + + it("throws the server's error message", async () => { + const { fetch } = fakeFetch(JSON.stringify({ error: { message: "no model" } }), { + status: 400, + }); + const stream = createOpenAIClient({ baseUrl: "http://x", fetch }).stream({ messages: [] }); + + await expect(collect(stream)).rejects.toThrow("HTTP 400: no model"); + }); +}); diff --git a/ui/lib/harness/tsconfig.build.json b/ui/lib/harness/tsconfig.build.json new file mode 100644 index 000000000000..cb6a3121d48a --- /dev/null +++ b/ui/lib/harness/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "declaration": true, + "sourceMap": true + }, + "include": ["src"] +} diff --git a/ui/lib/harness/tsconfig.json b/ui/lib/harness/tsconfig.json new file mode 100644 index 000000000000..caa8927c4fc5 --- /dev/null +++ b/ui/lib/harness/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "tests"] +} From d2e71619eff568dd07f2c8bab52aa9257557fdca Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Tue, 10 Nov 2026 19:47:00 +0100 Subject: [PATCH 2/3] ui: run Llama Chat's agentic loop through @ggml-org/llama-harness The loop leaves the agentic store, which now gives the harness what the app owns: ChatService as its inference client, the enabled tools behind the permission gate, and the callbacks that store each turn. Its API and the flow's callbacks are unchanged. A unit test runs the flow end to end with the server and the tools mocked. Chat compiles the harness from source through an alias, so the builds that copy or isolate ui/apps/chat now bring ui/lib/harness along: CMake and Nix through LLAMA_HARNESS_DIR, the Dockerfiles with a COPY. Assisted-by: Claude (Fable 5.2) --- .devops/cann.Dockerfile | 3 +- .devops/cpu.Dockerfile | 3 +- .devops/cuda.Dockerfile | 3 +- .devops/intel.Dockerfile | 3 +- .devops/musa.Dockerfile | 3 +- .devops/nix/package.nix | 2 +- .devops/openvino.Dockerfile | 3 +- .devops/rocm.Dockerfile | 3 +- .devops/vulkan.Dockerfile | 3 +- .devops/zendnn.Dockerfile | 3 +- .github/workflows/ui-self-hosted.yml | 2 + scripts/ui-assets.cmake | 13 +- ui/README.md | 15 +- ui/apps/chat/README.md | 19 +- ui/apps/chat/sources.cmake | 3 +- ui/apps/chat/src/lib/enums/agentic.enums.ts | 32 + ui/apps/chat/src/lib/enums/index.ts | 2 + .../src/lib/stores/agentic/index.svelte.ts | 910 ++++++++---------- ui/apps/chat/src/lib/utils/async-queue.ts | 50 + ui/apps/chat/tests/unit/agentic-flow.test.ts | 372 +++++++ ui/apps/chat/tests/unit/async-queue.test.ts | 42 + ui/apps/chat/tsconfig.json | 3 +- ui/apps/chat/vite.config.ts | 9 +- 23 files changed, 950 insertions(+), 551 deletions(-) create mode 100644 ui/apps/chat/src/lib/utils/async-queue.ts create mode 100644 ui/apps/chat/tests/unit/agentic-flow.test.ts create mode 100644 ui/apps/chat/tests/unit/async-queue.test.ts diff --git a/.devops/cann.Dockerfile b/.devops/cann.Dockerfile index 83df409506fd..d76bfa69978b 100644 --- a/.devops/cann.Dockerfile +++ b/.devops/cann.Dockerfile @@ -25,9 +25,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM ${CANN_BASE_IMAGE} AS build diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile index e195b5ac8b53..19a6a93b78c0 100644 --- a/.devops/cpu.Dockerfile +++ b/.devops/cpu.Dockerfile @@ -15,9 +15,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/ubuntu:$UBUNTU_VERSION AS build diff --git a/.devops/cuda.Dockerfile b/.devops/cuda.Dockerfile index 20501c1beac2..b4cad90c95c5 100644 --- a/.devops/cuda.Dockerfile +++ b/.devops/cuda.Dockerfile @@ -23,9 +23,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM ${BASE_CUDA_DEV_CONTAINER} AS build diff --git a/.devops/intel.Dockerfile b/.devops/intel.Dockerfile index 16326ebbdea5..9e0025920602 100644 --- a/.devops/intel.Dockerfile +++ b/.devops/intel.Dockerfile @@ -17,9 +17,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build diff --git a/.devops/musa.Dockerfile b/.devops/musa.Dockerfile index dbf9e0b41122..dcfe3a22080f 100644 --- a/.devops/musa.Dockerfile +++ b/.devops/musa.Dockerfile @@ -21,9 +21,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM ${BASE_MUSA_DEV_CONTAINER} AS build diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 4ac844ff959d..887270fb23bd 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -182,7 +182,7 @@ effectiveStdenv.mkDerivation (finalAttrs: { webui = stdenvNoCC.mkDerivation { pname = "webui"; version = llamaVersion; - # the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole + # the app compiles Llama UI and the harness from their sources in ../../lib, so the source is ui/ as a whole src = lib.cleanSource ../../ui; sourceRoot = "source/apps/chat"; diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile index c88cc2bbb5f9..f7461f2eee7f 100644 --- a/.devops/openvino.Dockerfile +++ b/.devops/openvino.Dockerfile @@ -34,9 +34,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build ## Build Image diff --git a/.devops/rocm.Dockerfile b/.devops/rocm.Dockerfile index 9e667660f00e..78a30ef47ae4 100644 --- a/.devops/rocm.Dockerfile +++ b/.devops/rocm.Dockerfile @@ -23,9 +23,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build ### Build image diff --git a/.devops/vulkan.Dockerfile b/.devops/vulkan.Dockerfile index dc8a87667205..740cd3b9cced 100644 --- a/.devops/vulkan.Dockerfile +++ b/.devops/vulkan.Dockerfile @@ -15,9 +15,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/ubuntu:$UBUNTU_VERSION AS build diff --git a/.devops/zendnn.Dockerfile b/.devops/zendnn.Dockerfile index cbc3696ed643..75b2da5a4dd5 100644 --- a/.devops/zendnn.Dockerfile +++ b/.devops/zendnn.Dockerfile @@ -15,9 +15,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/ubuntu:$UBUNTU_VERSION AS build diff --git a/.github/workflows/ui-self-hosted.yml b/.github/workflows/ui-self-hosted.yml index 346f1cfb4b12..e19c4234b827 100644 --- a/.github/workflows/ui-self-hosted.yml +++ b/.github/workflows/ui-self-hosted.yml @@ -19,6 +19,7 @@ on: '.github/workflows/ui-build-self-hosted.yml', 'ui/apps/chat/**.*', 'ui/lib/ui/**.*', + 'ui/lib/harness/**.*', 'tools/server/tests/**.*' ] pull_request: @@ -28,6 +29,7 @@ on: '.github/workflows/ui-build-self-hosted.yml', 'ui/apps/chat/**.*', 'ui/lib/ui/**.*', + 'ui/lib/harness/**.*', 'tools/server/tests/**.*' ] diff --git a/scripts/ui-assets.cmake b/scripts/ui-assets.cmake index 26244b068890..2296c887da4d 100644 --- a/scripts/ui-assets.cmake +++ b/scripts/ui-assets.cmake @@ -20,10 +20,12 @@ set(LLAMA_UI_GZIP "" CACHE STRING "Apply gzip compress to assets to save ban set(DIST_DIR "${UI_BINARY_DIR}/dist") set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist") # The stage keeps the layout of ui/: the app in apps/chat, with the sources of -# Llama UI that it compiles next to it in lib/ui (see ui/apps/chat/scripts/llama-ui.js). +# Llama UI and the harness that it compiles next to it in lib/ (see +# ui/apps/chat/scripts/llama-ui.js and svelte.config.js). set(STAGE_DIR "${UI_BINARY_DIR}/ui-src") set(WORK_DIR "${STAGE_DIR}/apps/chat") set(UI_LIB_DIR "${UI_SOURCE_DIR}/../../lib/ui") +set(HARNESS_DIR "${UI_SOURCE_DIR}/../../lib/harness") set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp") set(EMBED_STAMP "${UI_BINARY_DIR}/.ui-embed.sha256") set(UI_CPP "${UI_BINARY_DIR}/ui.cpp") @@ -320,7 +322,7 @@ function(npm_build_should_skip out_var) endfunction() function(stage_sources) - # everything but the app goes: the library, restaged below, and what a stage + # everything but the app goes: the packages, restaged below, and what a stage # from before this layout left at the top, its node_modules included if(EXISTS "${STAGE_DIR}") file(GLOB staged RELATIVE "${STAGE_DIR}" "${STAGE_DIR}/*") @@ -344,7 +346,8 @@ function(stage_sources) PATTERN "node_modules" EXCLUDE ) - # what the app reads of the library: its sources and the dependency list + # what the app reads of the packages: Llama UI's sources and dependency list, + # and the harness's sources file(COPY "${UI_LIB_DIR}/package.json" DESTINATION "${STAGE_DIR}/lib/ui" NO_SOURCE_PERMISSIONS @@ -353,6 +356,10 @@ function(stage_sources) DESTINATION "${STAGE_DIR}/lib/ui/src" NO_SOURCE_PERMISSIONS ) + file(COPY "${HARNESS_DIR}/src" + DESTINATION "${STAGE_DIR}/lib/harness" + NO_SOURCE_PERMISSIONS + ) endfunction() function(npm_build out_var) diff --git a/ui/README.md b/ui/README.md index b70228d0de7c..af95f83ad93e 100644 --- a/ui/README.md +++ b/ui/README.md @@ -5,20 +5,23 @@ The UI of llama.cpp: the packages in `lib` and the apps built on them. ```text ui/ ├── lib/ -│ └── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components +│ ├── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components +│ └── harness/ @ggml-org/llama-harness: the agentic loop, headless └── apps/ - └── chat/ Llama Chat, the chat app that llama-server serves + └── chat/ Llama Chat, the chat app that llama-server serves ``` - [`lib/ui`](lib/ui/README.md) draws the parts and handles interaction, with no business logic. It is built with `@sveltejs/package` and published to npm as `@ggml-org/llama-ui`. Apps in this repo skip the package: they compile its sources as part of their own build, so they hold no copy of a component, a token or a font. -- [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop. CMake - builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as `@ggml-org/llama-chat` - (with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server. +- [`lib/harness`](lib/harness/README.md) is the agentic loop: plain TypeScript with no UI and no dependencies. An app + gives it an inference client and tools, and reads its events. +- [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop through the + harness. CMake builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as + `@ggml-org/llama-chat` (with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server. Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building, -checking and testing `apps/chat` needs no install in `lib/ui`. +checking and testing `apps/chat` needs no install in `lib/ui` or `lib/harness`. ## Releases diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 5fa37286aadf..c7f4b14465f1 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -84,15 +84,16 @@ Llama Chat is the first app on Llama UI, and the reference for the line between The app imports the library as `@ggml-org/llama-ui/ui/` and compiles its sources in its own build, with no install of the library (see `scripts/vite-plugin-llama-ui.ts`). Data flows one way, through these layers: -| Layer | Lives in | What it holds | -| -------------- | ------------------------- | --------------------------------------------------------------------------------------------------- | -| Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` | -| App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area | -| UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI | -| Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard | -| Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings | -| Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox | -| Storage + API | the browser, llama-server | IndexedDB and localStorage; llama-server's `/v1`, `/props`, `/models`, `/tools`, `/slots` | +| Layer | Lives in | What it holds | +| -------------- | ------------------------- | ---------------------------------------------------------------------------------------------------- | +| Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` | +| App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area | +| UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI | +| Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard | +| Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings | +| Agentic loop | `ui/lib/harness` | Llama Harness: streams a turn, runs the tools the model called, repeats until it answers without one | +| Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox | +| Storage + API | the browser, llama-server | IndexedDB and localStorage; llama-server's `/v1`, `/props`, `/models`, `/tools`, `/slots` | ```mermaid flowchart LR diff --git a/ui/apps/chat/sources.cmake b/ui/apps/chat/sources.cmake index 7e506c0eccf9..817bb9a22918 100644 --- a/ui/apps/chat/sources.cmake +++ b/ui/apps/chat/sources.cmake @@ -3,8 +3,9 @@ set(UI_SOURCE_GLOBS src/* static/* - # Llama UI, which the app compiles from source + # Llama UI and the harness, which the app compiles from source ../../lib/ui/src/lib/* + ../../lib/harness/src/* ) set(UI_SOURCE_FILES diff --git a/ui/apps/chat/src/lib/enums/agentic.enums.ts b/ui/apps/chat/src/lib/enums/agentic.enums.ts index 6dc46b0850ac..31f72bafd992 100644 --- a/ui/apps/chat/src/lib/enums/agentic.enums.ts +++ b/ui/apps/chat/src/lib/enums/agentic.enums.ts @@ -43,3 +43,35 @@ export enum DiffLineKind { CONTEXT = 'context', REMOVE = 'remove' } + +/** + * Events of a harness run. The values mirror `HarnessEvent` of `@ggml-org/llama-harness`. + */ +export enum HarnessEventType { + COMPLETION_ID = 'completion_id', + DONE = 'done', + ERROR = 'error', + MESSAGE = 'message', + MODEL = 'model', + REASONING = 'reasoning', + TEXT = 'text', + TIMINGS = 'timings', + TOOL_CALL = 'tool_call', + TOOL_CALL_DELTA = 'tool_call_delta', + TOOL_RESULT = 'tool_result', + TOOL_UPDATE = 'tool_update', + TURN_END = 'turn_end', + TURN_START = 'turn_start' +} + +/** + * Events of one streamed completion. The values mirror `InferenceEvent` of `@ggml-org/llama-harness`. + */ +export enum InferenceEventType { + COMPLETION_ID = 'completion_id', + MODEL = 'model', + REASONING = 'reasoning', + TEXT = 'text', + TIMINGS = 'timings', + TOOL_CALLS = 'tool_calls' +} diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts index a4eed39d8190..3378504c67ea 100644 --- a/ui/apps/chat/src/lib/enums/index.ts +++ b/ui/apps/chat/src/lib/enums/index.ts @@ -10,6 +10,8 @@ export { AgenticSectionType, ContinueIntentKind, DiffLineKind, + HarnessEventType, + InferenceEventType, ToolResultKind, ToolCallType } from './agentic.enums'; diff --git a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts index 2eb93ad3aa98..df914c5d5327 100644 --- a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts @@ -1,15 +1,26 @@ /** - * AgenticStore - Multi-turn agentic loop orchestration + * AgenticStore - Llama Chat's side of the agentic loop * - * Drives the agentic loop over MCP tools: streams each LLM turn, detects - * tool calls, executes them via mcpStore, and enforces the turn limit. Each - * turn produces one assistant message (with tool_calls) and one tool result - * message per executed call, persisted as separate DB rows. + * The loop itself is `@ggml-org/llama-harness`: it streams each LLM turn, runs + * the tool calls and enforces the turn limit. This store gives it what the + * app owns: ChatService as its inference client, the enabled tools behind + * the permission gate, and the callbacks that persist each turn as one + * assistant message (with tool_calls) and one tool result message per call. * - * Uses ChatService for streaming and mcpStore for tool execution; waits on - * the permission/continue/steering gates owned by {@link AgenticGates}. + * Waits on the permission/continue/steering gates owned by {@link AgenticGates}. */ +import { + type ContentPart, + createHarness, + type InferenceClient, + type InferenceEvent, + type Message, + type Tool, + type ToolCall, + type ToolContext, + type ToolResult +} from '@ggml-org/llama-harness'; import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '#lib/constants/index.ts'; import { AUDIO_MIME_TO_EXTENSION, @@ -24,6 +35,8 @@ import { BuiltInTool, ToolPermissionDecision, ToolSource } from '#lib/enums/inde import { AttachmentType, ContentPartType, + HarnessEventType, + InferenceEventType, MessageRole, MimeTypePrefix, ToolCallType @@ -44,24 +57,12 @@ import type { AgenticFlowCallbacks, AgenticFlowOptions, AgenticMessage, - AgenticToolCallList, AgenticToolPolicy, SteeringMessage } from '#lib/types/agentic.js'; +import type { ApiChatMessageData } from '#lib/types/api.js'; +import type { ChatMessagePromptProgress, ChatMessageTimings } from '#lib/types/chat.js'; import type { - ApiChatCompletionToolCall, - ApiChatMessageContentPart, - ApiChatMessageData -} from '#lib/types/api.js'; -import type { - ChatMessageAgenticTimings, - ChatMessageAgenticTurnStats, - ChatMessagePromptProgress, - ChatMessageTimings, - ChatMessageToolCallTiming -} from '#lib/types/chat.js'; -import type { - DatabaseMessage, DatabaseMessageExtra, DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile @@ -76,11 +77,13 @@ import type { ToolExecutionResult } from '#lib/types/index.ts'; import type { OpenAIToolDefinition } from '#lib/types/mcp.js'; +import { createAsyncQueue } from '#lib/utils/async-queue.ts'; import { executeBrowserInfoTool, executeGetDatetimeTool, getAudioInputFormat, - isAbortError + isAbortError, + throwIfAborted } from '#lib/utils/index.ts'; import { SvelteMap } from 'svelte/reactivity'; @@ -407,22 +410,142 @@ class AgenticStore { return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; } - private buildFinalTimings( - capturedTimings: ChatMessageTimings | undefined, - agenticTimings: ChatMessageAgenticTimings - ): ChatMessageTimings | undefined { - if (agenticTimings.toolCallsCount === 0) return capturedTimings; + /** + * The media of a tool result as content parts, only what the model can read. + */ + private buildContentParts( + attachments: DatabaseMessageExtra[], + effectiveModel: string + ): ContentPart[] { + const contentParts: ContentPart[] = []; + + for (const attachment of attachments) { + if (attachment.type === AttachmentType.AUDIO) { + if (modelsStore.props.modelSupportsAudio(effectiveModel)) { + contentParts.push({ + input_audio: { + data: (attachment as DatabaseMessageExtraAudioFile).base64Data, + format: getAudioInputFormat((attachment as DatabaseMessageExtraAudioFile).mimeType) + }, + type: ContentPartType.INPUT_AUDIO + }); + } + } else if (attachment.type === AttachmentType.IMAGE) { + if (modelsStore.props.modelSupportsVision(effectiveModel)) { + contentParts.push({ + image_url: { + url: (attachment as DatabaseMessageExtraImageFile).base64Url + }, + type: ContentPartType.IMAGE_URL + }); + } else { + console.info( + `[AgenticStore] Skipping image attachment (model "${effectiveModel}" does not support vision)` + ); + } + } + } - return { - agentic: agenticTimings, - cache_n: capturedTimings?.cache_n, - predicted_ms: capturedTimings?.predicted_ms, - predicted_n: capturedTimings?.predicted_n, - prompt_ms: capturedTimings?.prompt_ms, - prompt_n: capturedTimings?.prompt_n + return contentParts; + } + + /** + * Run a tool call where its source runs it: on llama-server, in the + * browser or on an MCP server. Throws when the call fails. + */ + private async dispatchTool(params: { + callbacks: AgenticFlowCallbacks; + context: ToolContext; + conversationId: string; + effectiveModel: string; + toolCall: ToolCall; + }): Promise { + const { callbacks, context, conversationId, effectiveModel, toolCall } = params; + const { signal } = context; + const toolName = toolCall.function.name; + const toolSource = toolsStore.getToolSource(toolName); + + if ( + toolSource === ToolSource.SERVER && + toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND && + callbacks.createToolResultMessage && + callbacks.updateToolResultMessage + ) { + const args = this.parseToolArguments(toolCall.function.arguments); + const cwd = conversationsStore.activeConversation?.cwd; + + // the first update creates the result message, so the output shows live + context.update(''); + + let accumulated = ''; + let isError = false; + + for await (const ev of ToolsService.streamTool(toolName, args, signal, cwd)) { + if (ev.chunk !== null) { + accumulated += ev.chunk; + context.update(accumulated); + } + + if (ev.done) { + if (ev.error) { + accumulated = accumulated ? `${accumulated}\nError: ${ev.error}` : `Error: ${ev.error}`; + context.update(accumulated); + isError = true; + } + + break; + } + } + + return { content: accumulated, isError }; + } + + if (toolSource === ToolSource.SERVER) { + const args = this.parseToolArguments(toolCall.function.arguments); + const cwd = conversationsStore.activeConversation?.cwd; + + return ToolsService.executeTool(toolName, args, signal, cwd); + } + + if (toolSource === ToolSource.BROWSER) { + const args = this.parseToolArguments(toolCall.function.arguments); + + if (toolName === BuiltInTool.BROWSER_GET_DATETIME) return executeGetDatetimeTool(); + + if (toolName === BuiltInTool.SERVER_GET_INFO) return executeBrowserInfoTool(); + + if (toolName === BuiltInTool.BROWSER_SEARCH_CHATS) { + return SearchChatsService.executeTool(args, conversationId); + } + + if (toolName === BuiltInTool.BROWSER_READ_MEDIA) { + return ReadMediaService.executeTool( + args, + { + audio: modelsStore.props.modelSupportsAudio(effectiveModel), + vision: modelsStore.props.modelSupportsVision(effectiveModel) + }, + signal, + conversationsStore.activeConversation?.cwd + ); + } + + return SandboxService.executeTool(toolName, args, signal); + } + + const mcpCall: MCPToolCall = { + function: { arguments: toolCall.function.arguments, name: toolName }, + id: toolCall.id }; + const executionResult = await mcpStore.executeTool(mcpCall, signal); + + return { content: executionResult.content, isError: false }; } + /** + * Run the harness and turn its events into the flow's callbacks and the + * session state the UI reads. + */ private async executeAgenticLoop(params: { conversationId: string; messages: ApiChatMessageData[]; @@ -448,545 +571,252 @@ class AgenticStore { onTurnComplete, updateToolResultMessage } = callbacks; - const sessionMessages: AgenticMessage[] = toAgenticMessages(messages); - - let capturedTimings: ChatMessageTimings | undefined; - let totalToolCallCount = 0; - - const agenticTimings: ChatMessageAgenticTimings = { - llm: { predicted_ms: 0, predicted_n: 0, prompt_ms: 0, prompt_n: 0 }, - perTurn: [], - toolCalls: [], - toolCallsCount: 0, - toolsMs: 0, - turns: 0 - }; - const maxTurns = agenticConfig.maxTurns; const effectiveModel = options.model || modelsStore.models[0]?.model || ''; - - let turn = 0; - - while (true) { - if (turn >= maxTurns) { + const harness = createHarness(this.inferenceClient(options, conversationId)); + const harnessTools: Tool[] = tools.map((definition) => ({ + definition, + execute: (toolCall, context) => + this.executeTool({ callbacks, context, conversationId, effectiveModel, toolCall }) + })); + // result messages of streaming tools, created while the tool runs, by tool call id + const streamedResults = new SvelteMap(); + const events = harness.run(toAgenticMessages(messages) as Message[], harnessTools, signal, { + interrupted: () => this.gates.hasPendingSteeringMessage(conversationId), + maxTurns: agenticConfig.maxTurns, + onTurnLimit: async () => { // Turn limit reached - ask user whether to continue const shouldContinue = await this.gates.requestContinue(conversationId, signal); // Yield to allow Svelte to flush the UI update await new Promise((r) => setTimeout(r, 0)); - if (!shouldContinue || signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); - - return; - } - - // User chose to continue - extend the limit - turn = 0; - } - - this.updateSession(conversationId, { currentTurn: turn + 1 }); - agenticTimings.turns = turn + 1; - - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); - - return; - } - - // For turns > 0, create a new assistant message via callback - if (turn > 0 && createAssistantMessage) { - await createAssistantMessage(); + return shouldContinue; } + }); - let turnContent = ''; - let turnReasoningContent = ''; - let turnToolCalls: ApiChatCompletionToolCall[] = []; - let lastStreamingToolCallName = ''; - let lastStreamingToolCallArgsLength = 0; - let turnTimings: ChatMessageTimings | undefined; - - const turnStats: ChatMessageAgenticTurnStats = { - llm: { predicted_ms: 0, predicted_n: 0, prompt_ms: 0, prompt_n: 0 }, - toolCalls: [], - toolsMs: 0, - turn: turn + 1 - }; - - try { - await ChatService.sendMessage( - sessionMessages as ApiChatMessageData[], - { - ...options, - onChunk: (chunk: string) => { - turnContent += chunk; - onChunk?.(chunk); - }, - onComplete: () => { - /* Completion handled after sendMessage resolves */ - }, - onCompletionId, - onError: (error: Error) => { - throw error; - }, - onModel, - onReasoningChunk: (chunk: string) => { - turnReasoningContent += chunk; - onReasoningChunk?.(chunk); - }, - onTimings: (timings?: ChatMessageTimings, progress?: ChatMessagePromptProgress) => { - onTimings?.(timings, progress); - - if (timings) { - capturedTimings = timings; - turnTimings = timings; - - // completed turns + in-flight turn live counts - this.updateSession(conversationId, { - liveLlm: { - predicted_ms: agenticTimings.llm.predicted_ms + (timings.predicted_ms ?? 0), - predicted_n: agenticTimings.llm.predicted_n + (timings.predicted_n ?? 0), - prompt_ms: agenticTimings.llm.prompt_ms + (timings.prompt_ms ?? 0), - prompt_n: agenticTimings.llm.prompt_n + (timings.prompt_n ?? 0) - } - }); - } - }, - onToolCallChunk: (serialized: string) => { - try { - turnToolCalls = JSON.parse(serialized) as ApiChatCompletionToolCall[]; - - onToolCallsStreaming?.(turnToolCalls); - - if (turnToolCalls.length > 0 && turnToolCalls[0]?.function) { - const name = turnToolCalls[0].function.name || ''; - const args = turnToolCalls[0].function.arguments || ''; - const argsLengthBucket = Math.floor(args.length / 100); - - if ( - name !== lastStreamingToolCallName || - argsLengthBucket !== lastStreamingToolCallArgsLength - ) { - lastStreamingToolCallName = name; - lastStreamingToolCallArgsLength = argsLengthBucket; - this.updateSession(conversationId, { - streamingToolCall: { arguments: args, name } - }); - } - } - } catch { - /* Ignore parse errors during streaming */ - } - }, - stream: true, - tools: tools.length > 0 ? tools : undefined - }, - conversationId, - signal - ); + let totalToolCallCount = 0; + let lastStreamingToolCallName = ''; + let lastStreamingToolCallArgsLength = 0; - this.updateSession(conversationId, { streamingToolCall: null }); - - if (turnTimings) { - agenticTimings.llm.predicted_n += turnTimings.predicted_n || 0; - agenticTimings.llm.predicted_ms += turnTimings.predicted_ms || 0; - agenticTimings.llm.prompt_n += turnTimings.prompt_n || 0; - agenticTimings.llm.prompt_ms += turnTimings.prompt_ms || 0; - turnStats.llm.predicted_n = turnTimings.predicted_n || 0; - turnStats.llm.predicted_ms = turnTimings.predicted_ms || 0; - turnStats.llm.prompt_n = turnTimings.prompt_n || 0; - turnStats.llm.prompt_ms = turnTimings.prompt_ms || 0; - } - } catch (error) { - if (signal?.aborted) { - // Save whatever we have for this turn before exiting - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + for await (const event of events) { + switch (event.type) { + case HarnessEventType.TURN_START: + this.updateSession(conversationId, { currentTurn: event.turn }); + lastStreamingToolCallName = ''; + lastStreamingToolCallArgsLength = 0; - return; - } + // the caller created the first turn's assistant message + if (event.turn > 1 && createAssistantMessage) await createAssistantMessage(); - const normalizedError = error instanceof Error ? error : new Error('LLM stream error'); + break; - // preserve partial output as is, the outer error dialog informs the user separately - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.TEXT: + onChunk?.(event.delta); - throw normalizedError; - } + break; - // If the abort landed while ChatService.sendMessage was still resolving, the - // outer catch above never fires because ChatService swallows the AbortError - // and returns normally. Bail out here so a half-received tool_call (truncated - // arguments JSON) is not persisted as if it were complete. - if (signal?.aborted) { - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.REASONING: + onReasoningChunk?.(event.delta); - return; - } + break; - // === Steering check: if a user message was queued during this turn, exit the flow. - // The caller (chatStore) will consume the pending message and re-send it normally. - if (this.gates.hasPendingSteeringMessage(conversationId)) { - console.log('[AgenticStore] Steering message detected after turn, exiting agentic flow'); - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - turnToolCalls.length > 0 ? this.normalizeToolCalls(turnToolCalls) : undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.TOOL_CALL_DELTA: { + onToolCallsStreaming?.(event.toolCalls); - return; - } + const streamingCall = event.toolCalls[0]?.function; - // No tool calls = final turn, save and complete - if (turnToolCalls.length === 0) { - agenticTimings.perTurn!.push(turnStats); + if (streamingCall) { + const name = streamingCall.name || ''; + const args = streamingCall.arguments || ''; + const argsLengthBucket = Math.floor(args.length / 100); - const finalTimings = this.buildFinalTimings(capturedTimings, agenticTimings); - - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - finalTimings, - undefined - ); + if ( + name !== lastStreamingToolCallName || + argsLengthBucket !== lastStreamingToolCallArgsLength + ) { + lastStreamingToolCallName = name; + lastStreamingToolCallArgsLength = argsLengthBucket; + this.updateSession(conversationId, { + streamingToolCall: { arguments: args, name } + }); + } + } - if (finalTimings) onTurnComplete?.(finalTimings); + break; + } - onFlowComplete?.(finalTimings); + case HarnessEventType.TIMINGS: + onTimings?.(event.timings, event.progress); - return; - } + // completed turns + in-flight turn live counts + if (event.llm) this.updateSession(conversationId, { liveLlm: event.llm }); - // Normalize and save assistant turn with tool calls - const normalizedCalls = this.normalizeToolCalls(turnToolCalls); + break; - if (normalizedCalls.length === 0) { - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.MODEL: + onModel?.(event.model); - return; - } + break; - totalToolCallCount += normalizedCalls.length; - this.updateSession(conversationId, { totalToolCalls: totalToolCallCount }); - - // Save the assistant message with its tool calls - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - turnTimings, - normalizedCalls - ); - - // Add assistant message to session history - sessionMessages.push({ - content: turnContent || undefined, - reasoning_content: turnReasoningContent || undefined, - role: MessageRole.ASSISTANT, - tool_calls: normalizedCalls - }); + case HarnessEventType.COMPLETION_ID: + onCompletionId?.(event.id); - // Execute each tool call and create result messages - for (let i = 0; i < normalizedCalls.length; i++) { - const toolCall = normalizedCalls[i]; + break; - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.MESSAGE: + this.updateSession(conversationId, { streamingToolCall: null }); - return; - } + if (event.toolCalls && !event.final) { + totalToolCallCount += event.toolCalls.length; + this.updateSession(conversationId, { totalToolCalls: totalToolCallCount }); + } - // Check for pending steering message - skip remaining tool calls - if (this.gates.hasPendingSteeringMessage(conversationId)) { - console.log( - `[AgenticStore] Steering message detected, skipping ${normalizedCalls.length - i} remaining tool call(s)` + await onAssistantTurnComplete?.( + event.content, + event.reasoning, + event.timings, + event.toolCalls ); - for (let j = i; j < normalizedCalls.length; j++) { - const remainingCall = normalizedCalls[j]; - const interruptedContent = 'Tool execution was interrupted by a new user message.'; - if (createToolResultMessage) { - await createToolResultMessage(remainingCall.id, interruptedContent); - } + break; - sessionMessages.push({ - content: interruptedContent, - role: MessageRole.TOOL, - tool_call_id: remainingCall.id + case HarnessEventType.TOOL_UPDATE: { + if (!createToolResultMessage || !updateToolResultMessage) break; + + const streamed = streamedResults.get(event.toolCall.id); + + if (streamed) { + streamed.content = event.content; + await updateToolResultMessage(streamed.messageId, event.content); + } else { + const message = await createToolResultMessage( + event.toolCall.id, + event.content, + undefined, + conversationsStore.activeConversation?.cwd + ); + + streamedResults.set(event.toolCall.id, { + content: event.content, + messageId: message.id }); } break; } - const toolName = toolCall.function.name; - const serverLabel = toolsStore.getToolServerLabel(toolName); - // Ask for permission before executing the tool - const permission = await this.gates.requestPermission( - conversationId, - toolName, - serverLabel, - signal - ); + case HarnessEventType.TOOL_RESULT: { + const { content } = event.result; + const attachments = (event.result.details as DatabaseMessageExtra[] | undefined) ?? []; + const extras = attachments.length > 0 ? attachments : undefined; + const streamed = streamedResults.get(event.toolCall.id); - // Yield to allow Svelte to flush the UI update (hide permission dialog) - await new Promise((r) => setTimeout(r, 0)); - - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + let toolResultMessageId: string | undefined; - return; - } + if (streamed) { + toolResultMessageId = streamed.messageId; - const toolStartTime = performance.now(); - const toolSource = toolsStore.getToolSource(toolName); - - let result = ''; - let toolSuccess = true; - let createdToolResultMessageId: string | null = null; - - // Streaming tools (currently only exec_shell_command): mark - // the session so the matching renderer can switch to live mode. - // Cleared unconditionally below. - this.updateSession(conversationId, { executingToolCallId: toolCall.id }); - - if (permission === ToolPermissionDecision.DENY) { - result = 'Tool execution was denied by the user.'; - toolSuccess = false; - } else { - try { - if ( - toolSource === ToolSource.SERVER && - toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND && - createToolResultMessage && - updateToolResultMessage - ) { - const args = this.parseToolArguments(toolCall.function.arguments); - const cwd = conversationsStore.activeConversation?.cwd; - const msg = await createToolResultMessage(toolCall.id, '', undefined, cwd); - - createdToolResultMessageId = msg.id; - - let accumulated = ''; - - for await (const ev of ToolsService.streamTool(toolName, args, signal, cwd)) { - if (ev.chunk !== null) { - accumulated += ev.chunk; - await updateToolResultMessage(msg.id, accumulated); - } - - if (ev.done) { - if (ev.error) { - accumulated = accumulated - ? `${accumulated}\nError: ${ev.error}` - : `Error: ${ev.error}`; - await updateToolResultMessage(msg.id, accumulated); - toolSuccess = false; - } - - break; - } - } - result = accumulated; - } else if (toolSource === ToolSource.SERVER) { - const args = this.parseToolArguments(toolCall.function.arguments); - const cwd = conversationsStore.activeConversation?.cwd; - const executionResult = await ToolsService.executeTool(toolName, args, signal, cwd); - - result = executionResult.content; - - if (executionResult.isError) toolSuccess = false; - } else if (toolSource === ToolSource.BROWSER) { - const args = this.parseToolArguments(toolCall.function.arguments); - - let executionResult: ToolExecutionResult; - - if (toolName === BuiltInTool.BROWSER_GET_DATETIME) { - executionResult = executeGetDatetimeTool(); - } else if (toolName === BuiltInTool.SERVER_GET_INFO) { - executionResult = executeBrowserInfoTool(); - } else if (toolName === BuiltInTool.BROWSER_SEARCH_CHATS) { - executionResult = await SearchChatsService.executeTool(args, conversationId); - } else if (toolName === BuiltInTool.BROWSER_READ_MEDIA) { - executionResult = await ReadMediaService.executeTool( - args, - { - audio: modelsStore.props.modelSupportsAudio(effectiveModel), - vision: modelsStore.props.modelSupportsVision(effectiveModel) - }, - signal, - conversationsStore.activeConversation?.cwd - ); - } else { - executionResult = await SandboxService.executeTool(toolName, args, signal); - } - - result = executionResult.content; - - if (executionResult.isError) toolSuccess = false; - } else { - const mcpCall: MCPToolCall = { - function: { arguments: toolCall.function.arguments, name: toolName }, - id: toolCall.id - }; - const executionResult = await mcpStore.executeTool(mcpCall, signal); - - result = executionResult.content; - } - } catch (error) { - if (isAbortError(error)) { - this.updateSession(conversationId, { executingToolCallId: null }); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); - - return; - } - - // Carry the partial stream contents already mirrored to the UI - - // they show up as live output even if the stream broke off mid-run. - result = result - ? `${result}\nError: ${error instanceof Error ? error.message : String(error)}` - : `Error: ${error instanceof Error ? error.message : String(error)}`; - toolSuccess = false; - - if (createdToolResultMessageId && updateToolResultMessage) { - await updateToolResultMessage(createdToolResultMessageId, result); + // the message already shows the streamed output: rewrite it only when the result differs + if (extras || content !== streamed.content) { + await updateToolResultMessage?.(streamed.messageId, content, extras); } + } else if (createToolResultMessage) { + toolResultMessageId = ( + await createToolResultMessage(event.toolCall.id, content, extras) + ).id; } - } - this.updateSession(conversationId, { executingToolCallId: null }); + if (extras && toolResultMessageId) onAttachments?.(toolResultMessageId, extras); - const toolDurationMs = performance.now() - toolStartTime; - const toolTiming: ChatMessageToolCallTiming = { - duration_ms: Math.round(toolDurationMs), - name: toolCall.function.name, - success: toolSuccess - }; + break; + } - agenticTimings.toolCalls!.push(toolTiming); - agenticTimings.toolCallsCount++; - agenticTimings.toolsMs += Math.round(toolDurationMs); - turnStats.toolCalls.push(toolTiming); - turnStats.toolsMs += Math.round(toolDurationMs); + case HarnessEventType.TURN_END: + onTurnComplete?.(event.timings); - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + break; - return; - } + case HarnessEventType.DONE: + onFlowComplete?.(event.timings); - const { attachments, cleanedResult } = this.extractBase64Attachments(result); + break; - // For streaming tools the result message was created empty - // at the start of execution and updated in place as chunks - // arrived via updateToolResultMessage. Skip the second - // create call - just attach any base64 attachments found in - // the final accumulator (rare, since chunks usually don't - // carry image data URIs) and emit the attachments callback. - let toolResultMessage: DatabaseMessage | undefined; + case HarnessEventType.ERROR: + onFlowComplete?.(event.timings); - if (createdToolResultMessageId) { - toolResultMessage = { id: createdToolResultMessageId } as DatabaseMessage; + throw event.error; + } + } + } - if (attachments.length > 0 && updateToolResultMessage) { - await updateToolResultMessage(createdToolResultMessageId, cleanedResult, attachments); - } - } else if (createToolResultMessage) { - toolResultMessage = await createToolResultMessage( - toolCall.id, - cleanedResult, - attachments.length > 0 ? attachments : undefined - ); - } + /** + * A tool of the harness: the permission gate, then the call, then the + * result with its media taken out of the text. + */ + private async executeTool(params: { + callbacks: AgenticFlowCallbacks; + context: ToolContext; + conversationId: string; + effectiveModel: string; + toolCall: ToolCall; + }): Promise { + const { callbacks, context, conversationId, effectiveModel, toolCall } = params; + const toolName = toolCall.function.name; + const serverLabel = toolsStore.getToolServerLabel(toolName); + // Ask for permission before executing the tool + const permission = await this.gates.requestPermission( + conversationId, + toolName, + serverLabel, + context.signal + ); - if (attachments.length > 0 && toolResultMessage) { - onAttachments?.(toolResultMessage.id, attachments); - } + // Yield to allow Svelte to flush the UI update (hide permission dialog) + await new Promise((r) => setTimeout(r, 0)); - // Build content parts for session history (including images for vision models) - const contentParts: ApiChatMessageContentPart[] = [ - { text: cleanedResult, type: ContentPartType.TEXT } - ]; - - for (const attachment of attachments) { - if (attachment.type === AttachmentType.AUDIO) { - if (modelsStore.props.modelSupportsAudio(effectiveModel)) { - contentParts.push({ - input_audio: { - data: (attachment as DatabaseMessageExtraAudioFile).base64Data, - format: getAudioInputFormat( - (attachment as DatabaseMessageExtraAudioFile).mimeType - ) - }, - type: ContentPartType.INPUT_AUDIO - }); - } - } else if (attachment.type === AttachmentType.IMAGE) { - if (modelsStore.props.modelSupportsVision(effectiveModel)) { - contentParts.push({ - image_url: { - url: (attachment as DatabaseMessageExtraImageFile).base64Url - }, - type: ContentPartType.IMAGE_URL - }); - } else { - console.info( - `[AgenticStore] Skipping image attachment (model "${effectiveModel}" does not support vision)` - ); - } - } - } + throwIfAborted(context.signal); - sessionMessages.push({ - content: contentParts.length === 1 ? cleanedResult : contentParts, - role: MessageRole.TOOL, - tool_call_id: toolCall.id - }); - } + // the time waiting for permission does not count + const toolStartTime = performance.now(); - if (turnStats.toolCalls.length > 0) { - agenticTimings.perTurn!.push(turnStats); + let execution: ToolExecutionResult; - const intermediateTimings = this.buildFinalTimings(capturedTimings, agenticTimings); + // Streaming tools (currently only exec_shell_command): mark + // the session so the matching renderer can switch to live mode. + this.updateSession(conversationId, { executingToolCallId: toolCall.id }); - if (intermediateTimings) onTurnComplete?.(intermediateTimings); - } + try { + execution = + permission === ToolPermissionDecision.DENY + ? { content: 'Tool execution was denied by the user.', isError: true } + : await this.dispatchTool({ + callbacks, + context, + conversationId, + effectiveModel, + toolCall + }); + } catch (error) { + if (isAbortError(error)) throw error; - // If tools were interrupted by a steering message, exit now instead of starting another LLM turn - if (this.gates.hasPendingSteeringMessage(conversationId)) { - console.log( - '[AgenticStore] Steering message detected after tool execution, exiting agentic flow' - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + execution = { + content: `Error: ${error instanceof Error ? error.message : String(error)}`, + isError: true + }; + } finally { + this.updateSession(conversationId, { executingToolCallId: null }); + } - return; - } + const durationMs = performance.now() - toolStartTime; + const { attachments, cleanedResult } = this.extractBase64Attachments(execution.content); - turn++; - } + return { + content: cleanedResult, + details: attachments, + durationMs, + isError: execution.isError, + parts: this.buildContentParts(attachments, effectiveModel) + }; } private extractBase64Attachments(result: string): { @@ -1044,17 +874,57 @@ class AgenticStore { return { attachments, cleanedResult: cleanedLines.join(NEWLINE) }; } - private normalizeToolCalls(toolCalls: ApiChatCompletionToolCall[]): AgenticToolCallList { - if (!toolCalls) return []; + /** + * ChatService as the harness's inference client: one streamed completion + * per turn, its callbacks turned into events. + */ + private inferenceClient(options: AgenticFlowOptions, conversationId: string): InferenceClient { + return { + stream: ({ messages, signal, tools }) => { + const events = createAsyncQueue(); - return toolCalls.map((call, index) => ({ - function: { - arguments: call?.function?.arguments ?? '', - name: call?.function?.name ?? '' - }, - id: call?.id ?? `tool_${index}`, - type: (call?.type as ToolCallType.FUNCTION) ?? ToolCallType.FUNCTION - })); + ChatService.sendMessage( + messages as ApiChatMessageData[], + { + ...options, + onChunk: (chunk: string) => + events.push({ delta: chunk, type: InferenceEventType.TEXT }), + onComplete: () => { + /* Completion handled after sendMessage resolves */ + }, + onCompletionId: (id: string) => + events.push({ id, type: InferenceEventType.COMPLETION_ID }), + onError: (error: Error) => { + throw error; + }, + onModel: (model: string) => events.push({ model, type: InferenceEventType.MODEL }), + onReasoningChunk: (chunk: string) => + events.push({ delta: chunk, type: InferenceEventType.REASONING }), + onTimings: (timings?: ChatMessageTimings, progress?: ChatMessagePromptProgress) => + events.push({ progress, timings, type: InferenceEventType.TIMINGS }), + onToolCallChunk: (serialized: string) => { + try { + events.push({ + toolCalls: JSON.parse(serialized), + type: InferenceEventType.TOOL_CALLS + }); + } catch { + /* Ignore parse errors during streaming */ + } + }, + stream: true, + tools + }, + conversationId, + signal + ).then( + () => events.close(), + (error: unknown) => events.fail(error) + ); + + return events; + } + }; } private parseToolArguments(args: string | Record): Record { diff --git a/ui/apps/chat/src/lib/utils/async-queue.ts b/ui/apps/chat/src/lib/utils/async-queue.ts new file mode 100644 index 000000000000..48988dd93951 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/async-queue.ts @@ -0,0 +1,50 @@ +/** + * A queue read as an async iterable, to turn a callback API into a stream. + * The iteration ends after `close`, and throws after `fail`, once the pushed items are read. + */ +export interface AsyncQueue extends AsyncIterable { + push(item: T): void; + close(): void; + fail(error: unknown): void; +} + +export function createAsyncQueue(): AsyncQueue { + const items: T[] = []; + + let closed = false; + let failure: { error: unknown } | undefined; + let wake: (() => void) | undefined; + + const notify = () => { + wake?.(); + wake = undefined; + }; + + return { + close() { + closed = true; + notify(); + }, + fail(error) { + failure = { error }; + notify(); + }, + push(item) { + items.push(item); + notify(); + }, + async *[Symbol.asyncIterator]() { + while (true) { + if (items.length > 0) { + yield items.shift()!; + } else if (failure) { + throw failure.error; + } else if (closed) { + return; + } else { + await new Promise((resolve) => (wake = resolve)); + } + } + } + }; +} diff --git a/ui/apps/chat/tests/unit/agentic-flow.test.ts b/ui/apps/chat/tests/unit/agentic-flow.test.ts new file mode 100644 index 000000000000..e8992b60c841 --- /dev/null +++ b/ui/apps/chat/tests/unit/agentic-flow.test.ts @@ -0,0 +1,372 @@ +// The agentic flow end to end, with the server and the tools mocked: what the +// model is sent each turn, and the order the flow's callbacks fire in. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('#lib/services/database.service.ts', () => ({ + DatabaseService: {} +})); + +import { BuiltInTool, MessageRole, ToolPermissionDecision, ToolSource } from '#lib/enums/index.ts'; +import { ChatService } from '#lib/services/chat.service.ts'; +import { ToolsService } from '#lib/services/tools.service.ts'; +import { agenticStore } from '#lib/stores/agentic/index.svelte.ts'; +import { modelsStore } from '#lib/stores/models/index.svelte.ts'; +import { permissionsStore } from '#lib/stores/permissions.svelte.ts'; +import { toolsStore } from '#lib/stores/tools.svelte.ts'; +import type { AgenticFlowCallbacks } from '#lib/types/agentic.js'; +import type { ApiChatMessageData } from '#lib/types/api.js'; +import type { ChatMessageTimings } from '#lib/types/chat.js'; +import type { DatabaseMessage } from '#lib/types/database.js'; + +const CONV_ID = 'conv-1'; +const USER_MESSAGES: ApiChatMessageData[] = [{ content: 'hi', role: MessageRole.USER }]; +const IMAGE_DATA_URI = 'data:image/png;base64,AAAA'; + +type StreamOptions = { + onChunk?: (chunk: string) => void; + onReasoningChunk?: (chunk: string) => void; + onTimings?: (timings?: ChatMessageTimings) => void; + onToolCallChunk?: (serialized: string) => void; +}; +type Turn = (options: StreamOptions) => void | Promise; + +function toolCalls(...names: string[]): string { + return JSON.stringify( + names.map((name, index) => ({ function: { arguments: '{}', name }, id: `call_${index}` })) + ); +} + +// each request plays the next scripted turn; returns what every request was sent +function mockTurns(...turns: Turn[]): ApiChatMessageData[][] { + const requests: ApiChatMessageData[][] = []; + + vi.spyOn(ChatService, 'sendMessage').mockImplementation(async (messages, options) => { + requests.push(structuredClone(messages) as ApiChatMessageData[]); + await turns[requests.length - 1]?.(options as StreamOptions); + }); + + return requests; +} + +function mockTools(source: ToolSource, ...names: string[]) { + vi.spyOn(toolsStore, 'getEnabledToolsForLLM').mockReturnValue( + names.map((name) => ({ function: { name, parameters: {} }, type: 'function' })) + ); + vi.spyOn(toolsStore, 'getToolSource').mockReturnValue(source); +} + +// callbacks that log their calls in order +function recorder() { + const calls: unknown[][] = []; + const log = + (name: string) => + (...args: unknown[]) => { + calls.push([name, ...args]); + }; + + let nextMessageId = 0; + + const createMessage = + (name: string) => + async (...args: unknown[]) => { + calls.push([name, ...args]); + + return { id: `msg_${nextMessageId++}` } as DatabaseMessage; + }; + const callbacks: AgenticFlowCallbacks = { + createAssistantMessage: createMessage('createAssistantMessage'), + createToolResultMessage: createMessage('createToolResultMessage'), + onAssistantTurnComplete: async (...args) => { + calls.push(['onAssistantTurnComplete', ...args]); + }, + onAttachments: log('onAttachments'), + onChunk: log('onChunk'), + onError: log('onError'), + onFlowComplete: log('onFlowComplete'), + onReasoningChunk: log('onReasoningChunk'), + onToolCallsStreaming: log('onToolCallsStreaming'), + onTurnComplete: log('onTurnComplete'), + updateToolResultMessage: async (...args) => { + calls.push(['updateToolResultMessage', ...args]); + } + }; + + return { + callbacks, + calls, + named: (name: string) => calls.filter((call) => call[0] === name), + names: () => calls.map((call) => call[0]) + }; +} + +function runFlow(callbacks: AgenticFlowCallbacks, signal?: AbortSignal) { + return agenticStore.runAgenticFlow({ + callbacks, + conversationId: CONV_ID, + messages: USER_MESSAGES, + signal + }); +} + +beforeEach(() => { + vi.restoreAllMocks(); + agenticStore.clearSession(CONV_ID); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(ChatService, 'normalizeMessagesForApi').mockImplementation( + async (messages) => messages as ApiChatMessageData[] + ); + vi.spyOn(toolsStore, 'fetchServerTools').mockResolvedValue(); + vi.spyOn(toolsStore, 'getToolServerLabel').mockReturnValue('Browser'); + vi.spyOn(toolsStore, 'getPermissionKey').mockReturnValue('tool-key'); + vi.spyOn(permissionsStore, 'hasTool').mockReturnValue(true); +}); + +describe('agentic flow', () => { + it('is not handled when no tool is enabled', async () => { + vi.spyOn(toolsStore, 'getEnabledToolsForLLM').mockReturnValue([]); + + const send = vi.spyOn(ChatService, 'sendMessage'); + + expect(await runFlow(recorder().callbacks)).toEqual({ handled: false }); + expect(send).not.toHaveBeenCalled(); + }); + + it('runs a tool call, then lets the model answer', async () => { + mockTools(ToolSource.BROWSER, BuiltInTool.BROWSER_GET_DATETIME); + + const requests = mockTurns( + (options) => { + options.onChunk?.('checking'); + options.onToolCallChunk?.(toolCalls(BuiltInTool.BROWSER_GET_DATETIME)); + options.onTimings?.({ predicted_ms: 10, predicted_n: 4, prompt_ms: 20, prompt_n: 8 }); + }, + (options) => { + options.onReasoningChunk?.('hm'); + options.onChunk?.('it is noon'); + options.onTimings?.({ predicted_ms: 5, predicted_n: 2, prompt_ms: 1, prompt_n: 1 }); + } + ); + const { callbacks, named, names } = recorder(); + + expect(await runFlow(callbacks)).toEqual({ handled: true }); + expect(names()).toEqual([ + 'onChunk', + 'onToolCallsStreaming', + 'onAssistantTurnComplete', + 'createToolResultMessage', + 'onTurnComplete', + 'createAssistantMessage', + 'onReasoningChunk', + 'onChunk', + 'onAssistantTurnComplete', + 'onTurnComplete', + 'onFlowComplete' + ]); + + const [toolTurn, finalTurn] = named('onAssistantTurnComplete'); + const call = { + function: { arguments: '{}', name: BuiltInTool.BROWSER_GET_DATETIME }, + id: 'call_0', + type: 'function' + }; + + expect(toolTurn.slice(1)).toEqual([ + 'checking', + undefined, + { predicted_ms: 10, predicted_n: 4, prompt_ms: 20, prompt_n: 8 }, + [call] + ]); + expect(finalTurn.slice(1, 3)).toEqual(['it is noon', 'hm']); + expect(finalTurn[4]).toBeUndefined(); + + const [, toolCallId, toolResult, extras] = named('createToolResultMessage')[0]; + + expect([toolCallId, extras]).toEqual(['call_0', undefined]); + expect(requests[1].slice(1)).toEqual([ + { content: 'checking', role: 'assistant', tool_calls: [call] }, + { content: toolResult, role: 'tool', tool_call_id: 'call_0' } + ]); + expect(named('onFlowComplete')[0][1]).toMatchObject({ + agentic: { + llm: { predicted_ms: 15, predicted_n: 6, prompt_ms: 21, prompt_n: 9 }, + perTurn: [{ turn: 1 }, { turn: 2 }], + toolCalls: [{ name: BuiltInTool.BROWSER_GET_DATETIME, success: true }], + toolCallsCount: 1, + turns: 2 + }, + predicted_n: 2 + }); + expect(agenticStore.getTotalToolCalls(CONV_ID)).toBe(1); + expect(agenticStore.isRunning(CONV_ID)).toBe(false); + }); + + it('waits for permission and tells the model when it is denied', async () => { + mockTools(ToolSource.BROWSER, BuiltInTool.BROWSER_GET_DATETIME); + vi.mocked(permissionsStore.hasTool).mockReturnValue(false); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls(BuiltInTool.BROWSER_GET_DATETIME)), + (options) => options.onChunk?.('ok') + ); + const { callbacks, named } = recorder(); + const flow = runFlow(callbacks); + + await vi.waitFor(() => + expect(agenticStore.getPendingPermissionRequest(CONV_ID)).toEqual({ + serverLabel: 'Browser', + toolName: BuiltInTool.BROWSER_GET_DATETIME + }) + ); + agenticStore.resolvePermission(CONV_ID, ToolPermissionDecision.DENY); + await flow; + + expect(named('createToolResultMessage')[0].slice(1)).toEqual([ + 'call_0', + 'Tool execution was denied by the user.', + undefined + ]); + expect(requests[1][2]).toMatchObject({ content: 'Tool execution was denied by the user.' }); + expect(named('onFlowComplete')[0][1]).toMatchObject({ + agentic: { toolCalls: [{ success: false }] } + }); + }); + + it('reports a failing tool to the model and goes on', async () => { + mockTools(ToolSource.SERVER, 'read_file'); + vi.spyOn(ToolsService, 'executeTool').mockRejectedValue(new Error('no such file')); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls('read_file')), + (options) => options.onChunk?.('sorry') + ); + const { callbacks } = recorder(); + + expect(await runFlow(callbacks)).toEqual({ handled: true }); + expect(requests[1][2]).toMatchObject({ content: 'Error: no such file', role: 'tool' }); + }); + + it('shows the output of a streaming tool while it runs', async () => { + mockTools(ToolSource.SERVER, BuiltInTool.SERVER_EXEC_SHELL_COMMAND); + vi.spyOn(ToolsService, 'streamTool').mockImplementation(async function* () { + yield { chunk: 'one\n', done: false }; + await new Promise((resolve) => setTimeout(resolve, 0)); + yield { chunk: 'two\n', done: false }; + await new Promise((resolve) => setTimeout(resolve, 0)); + yield { chunk: null, done: true }; + }); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls(BuiltInTool.SERVER_EXEC_SHELL_COMMAND)), + (options) => options.onChunk?.('done') + ); + const { callbacks, named } = recorder(); + + await runFlow(callbacks); + + // created empty when the tool starts, then rewritten in place: never created twice + expect(named('createToolResultMessage').map((call) => call.slice(1))).toEqual([ + ['call_0', '', undefined, undefined] + ]); + expect(named('updateToolResultMessage').map((call) => call.slice(1))).toEqual([ + ['msg_0', 'one\n'], + ['msg_0', 'one\ntwo\n'] + ]); + expect(requests[1][2]).toMatchObject({ content: 'one\ntwo\n', role: 'tool' }); + }); + + it('takes an image out of a tool result and sends it to a model with vision', async () => { + mockTools(ToolSource.SERVER, 'screenshot'); + vi.spyOn(ToolsService, 'executeTool').mockResolvedValue({ + content: `saved\n${IMAGE_DATA_URI}`, + isError: false + }); + vi.spyOn(modelsStore.props, 'modelSupportsVision').mockReturnValue(true); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls('screenshot')), + (options) => options.onChunk?.('here') + ); + const { callbacks, named } = recorder(); + + await runFlow(callbacks); + + const [, , content, extras] = named('createToolResultMessage')[0]; + + expect(content).toMatch(/^saved\n\[Attachment saved: .+\.png\]$/); + expect(extras).toMatchObject([{ base64Url: IMAGE_DATA_URI, type: 'IMAGE' }]); + expect(named('onAttachments')[0].slice(1)).toEqual(['msg_0', extras]); + expect(requests[1][2]).toEqual({ + content: [ + { text: content, type: 'text' }, + { image_url: { url: IMAGE_DATA_URI }, type: 'image_url' } + ], + role: 'tool', + tool_call_id: 'call_0' + }); + }); + + it('leaves the flow when a steering message is queued, skipping the tool calls left', async () => { + mockTools(ToolSource.SERVER, 'first', 'second'); + vi.spyOn(ToolsService, 'executeTool').mockImplementation(async () => { + agenticStore.injectSteeringMessage(CONV_ID, 'wait'); + + return { content: 'first ran', isError: false }; + }); + + const requests = mockTurns((options) => + options.onToolCallChunk?.(toolCalls('first', 'second')) + ); + const { callbacks, named, names } = recorder(); + + expect(await runFlow(callbacks)).toEqual({ handled: true }); + expect(named('createToolResultMessage').map((call) => call.slice(1, 3))).toEqual([ + ['call_0', 'first ran'], + ['call_1', 'Tool execution was interrupted by a new user message.'] + ]); + expect(names().slice(-2)).toEqual(['onTurnComplete', 'onFlowComplete']); + expect(requests).toHaveLength(1); + expect(agenticStore.consumePendingSteeringMessage(CONV_ID)).toMatchObject({ content: 'wait' }); + }); + + it('keeps the partial turn and reports the error when the stream fails', async () => { + mockTools(ToolSource.BROWSER, BuiltInTool.BROWSER_GET_DATETIME); + mockTurns((options) => { + options.onChunk?.('par'); + + throw new Error('HTTP 500'); + }); + + const { callbacks, named, names } = recorder(); + const result = await runFlow(callbacks); + + expect(result).toMatchObject({ error: { message: 'HTTP 500' }, handled: true }); + expect(names()).toEqual(['onChunk', 'onAssistantTurnComplete', 'onFlowComplete', 'onError']); + expect(named('onAssistantTurnComplete')[0].slice(1)).toEqual([ + 'par', + undefined, + undefined, + undefined + ]); + expect(agenticStore.getLastError(CONV_ID)?.message).toBe('HTTP 500'); + }); + + it('stops without a tool result when the user aborts during a tool call', async () => { + const controller = new AbortController(); + + mockTools(ToolSource.SERVER, 'slow'); + vi.spyOn(ToolsService, 'executeTool').mockImplementation(async () => { + controller.abort(); + + return { content: 'late', isError: false }; + }); + + const requests = mockTurns((options) => options.onToolCallChunk?.(toolCalls('slow'))); + const { callbacks, named, names } = recorder(); + + expect(await runFlow(callbacks, controller.signal)).toEqual({ handled: true }); + expect(named('createToolResultMessage')).toEqual([]); + expect(names().slice(-1)).toEqual(['onFlowComplete']); + expect(requests).toHaveLength(1); + }); +}); diff --git a/ui/apps/chat/tests/unit/async-queue.test.ts b/ui/apps/chat/tests/unit/async-queue.test.ts new file mode 100644 index 000000000000..4856f74d3ba6 --- /dev/null +++ b/ui/apps/chat/tests/unit/async-queue.test.ts @@ -0,0 +1,42 @@ +import { createAsyncQueue } from '#lib/utils/async-queue.ts'; +import { describe, expect, it } from 'vitest'; + +async function collect(queue: AsyncIterable): Promise { + const items: T[] = []; + + for await (const item of queue) items.push(item); + + return items; +} + +describe('createAsyncQueue', () => { + it('yields the items pushed before and during the iteration, then ends on close', async () => { + const queue = createAsyncQueue(); + + queue.push(1); + + const collected = collect(queue); + + await Promise.resolve(); + queue.push(2); + queue.push(3); + queue.close(); + + expect(await collected).toEqual([1, 2, 3]); + }); + + it('throws the failure after the pushed items are read', async () => { + const queue = createAsyncQueue(); + const seen: number[] = []; + + queue.push(1); + queue.fail(new Error('cut')); + + await expect( + (async () => { + for await (const item of queue) seen.push(item); + })() + ).rejects.toThrow('cut'); + expect(seen).toEqual([1]); + }); +}); diff --git a/ui/apps/chat/tsconfig.json b/ui/apps/chat/tsconfig.json index 1cee89a7912e..aab815e5fbc4 100644 --- a/ui/apps/chat/tsconfig.json +++ b/ui/apps/chat/tsconfig.json @@ -45,7 +45,8 @@ "tw-animate-css/*": ["./node_modules/tw-animate-css/*"], "vaul-svelte": ["./node_modules/vaul-svelte"], "vaul-svelte/*": ["./node_modules/vaul-svelte/*"], - "@lucide/svelte/icons/*": ["./node_modules/@lucide/svelte/dist/icons/*"] + "@lucide/svelte/icons/*": ["./node_modules/@lucide/svelte/dist/icons/*"], + "@ggml-org/llama-harness": ["../../lib/harness/src/index.ts"] }, "sourceMap": true, "strict": true diff --git a/ui/apps/chat/vite.config.ts b/ui/apps/chat/vite.config.ts index 93a4956fa1e1..c393f7661e90 100644 --- a/ui/apps/chat/vite.config.ts +++ b/ui/apps/chat/vite.config.ts @@ -16,6 +16,8 @@ import { fileURLToPath } from 'url'; import { defineConfig, loadEnv, type ProxyOptions, searchForWorkspaceRoot } from 'vite'; const __dirname = dirname(fileURLToPath(import.meta.url)); +// the harness has no build step in the repo: the app compiles it from source +const LLAMA_HARNESS_ENTRY = resolve(__dirname, '../../lib/harness/src/index.ts'); // eslint-disable-next-line @typescript-eslint/no-explicit-any const browserBaseConfig: any = { enabled: true, @@ -89,13 +91,18 @@ export default defineConfig(({ mode }) => { resolve: { alias: { + '@ggml-org/llama-harness': LLAMA_HARNESS_ENTRY, 'katex-fonts': resolve('node_modules/katex/dist/fonts') } }, server: { fs: { - allow: [searchForWorkspaceRoot(process.cwd()), resolve(__dirname, 'tests')] + allow: [ + searchForWorkspaceRoot(process.cwd()), + resolve(__dirname, 'tests'), + resolve(__dirname, '../../lib/harness') + ] }, headers: { 'Cross-Origin-Embedder-Policy': 'require-corp', From caf61a8ae5399fc1b9f556d0b00d3b90636aa2a2 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Thu, 12 Nov 2026 01:07:36 +0100 Subject: [PATCH 3/3] ui: bring the architecture docs in line with the ui/ folder The layout card no longer calls the move planned, the harness is in the pages, and they say that Llama Chat does not import @ggml-org/llama-ui yet. Assisted-by: Claude (Opus 5.5 Medium) --- ui/README.md | 4 ++++ ui/apps/chat/AGENTS.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ui/README.md b/ui/README.md index af95f83ad93e..55fab208f41d 100644 --- a/ui/README.md +++ b/ui/README.md @@ -23,6 +23,10 @@ ui/ Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building, checking and testing `apps/chat` needs no install in `lib/ui` or `lib/harness`. +The split between library and app is not finished. Llama Chat takes its tokens, fonts, surfaces and base components +from `lib/ui`, but still has its own versions of what the library holds above those (ActionIcon, SearchInput, Logo, +SyntaxHighlightedCode and the Model UI), and `DESIGN.md`, `BRAND.md` and the full Storybook still sit beside the app. + ## Releases The npm packages are versioned by hand, with semver, and published by CI: diff --git a/ui/apps/chat/AGENTS.md b/ui/apps/chat/AGENTS.md index 32e6a009a2c9..0b4884de3167 100644 --- a/ui/apps/chat/AGENTS.md +++ b/ui/apps/chat/AGENTS.md @@ -1,6 +1,6 @@ # Instructions for ui/apps/chat -These add to the repository's [AGENTS.md](../../../AGENTS.md) for work in `ui/apps/chat`: **Llama Chat**, the chat client that llama-server bundles. **Llama UI**, the library, is `ui/lib/ui` (`@ggml-org/llama-ui`): Chat imports its tokens, fonts and base components from there (`@ggml-org/llama-ui/ui/`) and keeps no copy of them. `DESIGN.md` and the Storybook live here; the brand book, `BRAND.md`, and the brand and communications pages are in [ggml-org/llama-brand](https://github.com/ggml-org/llama-brand), with their own Storybook. +These add to the repository's [AGENTS.md](../../../AGENTS.md) for work in `ui/apps/chat`: **Llama Chat**, the chat client that llama-server bundles. **Llama UI**, the library, is `ui/lib/ui` (`@ggml-org/llama-ui`): Chat imports its tokens, fonts and base components from there (`@ggml-org/llama-ui/ui/`) and keeps no copy of them. `DESIGN.md` and the Storybook live here; the brand book, `BRAND.md`, and the brand and communications pages are in [ggml-org/llama-brand](https://github.com/ggml-org/llama-brand), with their own Storybook. The agentic loop is `ui/lib/harness` (`@ggml-org/llama-harness`). - **Design:** [DESIGN.md](DESIGN.md) holds the tokens, roles and rules, written for agents. Follow it for anything visual. Its token sections (`colors`, `typography`, `rounded`) are generated from the CSS with `npm run design:tokens`, and CI checks they are current; the rest is written by hand, and when it and the CSS disagree, the CSS wins and DESIGN.md is updated. - **Stories:** a story lives with the code it shows. Llama UI's (its foundations, base components, app components and Model UI) are in `ui/lib/ui/tests/stories`, titled `Llama UI/...`, and run as the library's tests; this app's Storybook shows them too. Llama Chat's are in `tests/stories/`, one folder per `src/lib/components/app/` (`chat/` has one per Chat folder), titled `Llama Chat//`. An accessibility story sits next to its component's, as `.a11y.stories.svelte`, titled `...//Accessibility`. The story helpers both Storybooks use (the Editorial the prose pages are set in, `StateGallery`) live with the library's stories and are imported here as `$llama-ui-stories/...`; reuse them rather than copying them here.