diff --git a/.devops/cann.Dockerfile b/.devops/cann.Dockerfile index 83df409506fd..d76bfa69978b 100644 --- a/.devops/cann.Dockerfile +++ b/.devops/cann.Dockerfile @@ -25,9 +25,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM ${CANN_BASE_IMAGE} AS build diff --git a/.devops/cpu.Dockerfile b/.devops/cpu.Dockerfile index e195b5ac8b53..19a6a93b78c0 100644 --- a/.devops/cpu.Dockerfile +++ b/.devops/cpu.Dockerfile @@ -15,9 +15,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/ubuntu:$UBUNTU_VERSION AS build diff --git a/.devops/cuda.Dockerfile b/.devops/cuda.Dockerfile index 20501c1beac2..b4cad90c95c5 100644 --- a/.devops/cuda.Dockerfile +++ b/.devops/cuda.Dockerfile @@ -23,9 +23,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM ${BASE_CUDA_DEV_CONTAINER} AS build diff --git a/.devops/intel.Dockerfile b/.devops/intel.Dockerfile index 16326ebbdea5..9e0025920602 100644 --- a/.devops/intel.Dockerfile +++ b/.devops/intel.Dockerfile @@ -17,9 +17,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build diff --git a/.devops/musa.Dockerfile b/.devops/musa.Dockerfile index dbf9e0b41122..dcfe3a22080f 100644 --- a/.devops/musa.Dockerfile +++ b/.devops/musa.Dockerfile @@ -21,9 +21,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM ${BASE_MUSA_DEV_CONTAINER} AS build diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 4ac844ff959d..887270fb23bd 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -182,7 +182,7 @@ effectiveStdenv.mkDerivation (finalAttrs: { webui = stdenvNoCC.mkDerivation { pname = "webui"; version = llamaVersion; - # the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole + # the app compiles Llama UI and the harness from their sources in ../../lib, so the source is ui/ as a whole src = lib.cleanSource ../../ui; sourceRoot = "source/apps/chat"; diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile index c88cc2bbb5f9..f7461f2eee7f 100644 --- a/.devops/openvino.Dockerfile +++ b/.devops/openvino.Dockerfile @@ -34,9 +34,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build ## Build Image diff --git a/.devops/rocm.Dockerfile b/.devops/rocm.Dockerfile index 9e667660f00e..78a30ef47ae4 100644 --- a/.devops/rocm.Dockerfile +++ b/.devops/rocm.Dockerfile @@ -23,9 +23,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build ### Build image diff --git a/.devops/vulkan.Dockerfile b/.devops/vulkan.Dockerfile index dc8a87667205..740cd3b9cced 100644 --- a/.devops/vulkan.Dockerfile +++ b/.devops/vulkan.Dockerfile @@ -15,9 +15,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/ubuntu:$UBUNTU_VERSION AS build diff --git a/.devops/zendnn.Dockerfile b/.devops/zendnn.Dockerfile index cbc3696ed643..75b2da5a4dd5 100644 --- a/.devops/zendnn.Dockerfile +++ b/.devops/zendnn.Dockerfile @@ -15,9 +15,10 @@ COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./ RUN npm ci COPY ui/apps/chat/ ./ -# the app compiles Llama UI from its source next to it +# the app compiles Llama UI and the harness from their sources next to it COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/ +COPY ui/lib/harness/src/ /app/ui/lib/harness/src/ RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build FROM docker.io/ubuntu:$UBUNTU_VERSION AS build diff --git a/.github/workflows/ui-self-hosted.yml b/.github/workflows/ui-self-hosted.yml index 346f1cfb4b12..e19c4234b827 100644 --- a/.github/workflows/ui-self-hosted.yml +++ b/.github/workflows/ui-self-hosted.yml @@ -19,6 +19,7 @@ on: '.github/workflows/ui-build-self-hosted.yml', 'ui/apps/chat/**.*', 'ui/lib/ui/**.*', + 'ui/lib/harness/**.*', 'tools/server/tests/**.*' ] pull_request: @@ -28,6 +29,7 @@ on: '.github/workflows/ui-build-self-hosted.yml', 'ui/apps/chat/**.*', 'ui/lib/ui/**.*', + 'ui/lib/harness/**.*', 'tools/server/tests/**.*' ] diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 10d7ed0ed5e4..11875ba53c5c 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -150,6 +150,37 @@ jobs: run: npm run prepack working-directory: ui/lib/ui + harness-checks: + name: Harness checks + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Install dependencies + run: npm ci + working-directory: ui/lib/harness + + - name: Run type checking + run: npm run check + working-directory: ui/lib/harness + + - name: Run tests + run: npm test + working-directory: ui/lib/harness + + - name: Build the package + run: npm run prepack + working-directory: ui/lib/harness + e2e-tests: name: E2E Tests needs: ui-build diff --git a/scripts/ui-assets.cmake b/scripts/ui-assets.cmake index 26244b068890..2296c887da4d 100644 --- a/scripts/ui-assets.cmake +++ b/scripts/ui-assets.cmake @@ -20,10 +20,12 @@ set(LLAMA_UI_GZIP "" CACHE STRING "Apply gzip compress to assets to save ban set(DIST_DIR "${UI_BINARY_DIR}/dist") set(SRC_DIST_DIR "${UI_SOURCE_DIR}/dist") # The stage keeps the layout of ui/: the app in apps/chat, with the sources of -# Llama UI that it compiles next to it in lib/ui (see ui/apps/chat/scripts/llama-ui.js). +# Llama UI and the harness that it compiles next to it in lib/ (see +# ui/apps/chat/scripts/llama-ui.js and svelte.config.js). set(STAGE_DIR "${UI_BINARY_DIR}/ui-src") set(WORK_DIR "${STAGE_DIR}/apps/chat") set(UI_LIB_DIR "${UI_SOURCE_DIR}/../../lib/ui") +set(HARNESS_DIR "${UI_SOURCE_DIR}/../../lib/harness") set(STAMP_FILE "${UI_BINARY_DIR}/.ui-stamp") set(EMBED_STAMP "${UI_BINARY_DIR}/.ui-embed.sha256") set(UI_CPP "${UI_BINARY_DIR}/ui.cpp") @@ -320,7 +322,7 @@ function(npm_build_should_skip out_var) endfunction() function(stage_sources) - # everything but the app goes: the library, restaged below, and what a stage + # everything but the app goes: the packages, restaged below, and what a stage # from before this layout left at the top, its node_modules included if(EXISTS "${STAGE_DIR}") file(GLOB staged RELATIVE "${STAGE_DIR}" "${STAGE_DIR}/*") @@ -344,7 +346,8 @@ function(stage_sources) PATTERN "node_modules" EXCLUDE ) - # what the app reads of the library: its sources and the dependency list + # what the app reads of the packages: Llama UI's sources and dependency list, + # and the harness's sources file(COPY "${UI_LIB_DIR}/package.json" DESTINATION "${STAGE_DIR}/lib/ui" NO_SOURCE_PERMISSIONS @@ -353,6 +356,10 @@ function(stage_sources) DESTINATION "${STAGE_DIR}/lib/ui/src" NO_SOURCE_PERMISSIONS ) + file(COPY "${HARNESS_DIR}/src" + DESTINATION "${STAGE_DIR}/lib/harness" + NO_SOURCE_PERMISSIONS + ) endfunction() function(npm_build out_var) diff --git a/ui/README.md b/ui/README.md index b70228d0de7c..55fab208f41d 100644 --- a/ui/README.md +++ b/ui/README.md @@ -5,20 +5,27 @@ The UI of llama.cpp: the packages in `lib` and the apps built on them. ```text ui/ ├── lib/ -│ └── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components +│ ├── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components +│ └── harness/ @ggml-org/llama-harness: the agentic loop, headless └── apps/ - └── chat/ Llama Chat, the chat app that llama-server serves + └── chat/ Llama Chat, the chat app that llama-server serves ``` - [`lib/ui`](lib/ui/README.md) draws the parts and handles interaction, with no business logic. It is built with `@sveltejs/package` and published to npm as `@ggml-org/llama-ui`. Apps in this repo skip the package: they compile its sources as part of their own build, so they hold no copy of a component, a token or a font. -- [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop. CMake - builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as `@ggml-org/llama-chat` - (with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server. +- [`lib/harness`](lib/harness/README.md) is the agentic loop: plain TypeScript with no UI and no dependencies. An app + gives it an inference client and tools, and reads its events. +- [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop through the + harness. CMake builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as + `@ggml-org/llama-chat` (with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server. Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building, -checking and testing `apps/chat` needs no install in `lib/ui`. +checking and testing `apps/chat` needs no install in `lib/ui` or `lib/harness`. + +The split between library and app is not finished. Llama Chat takes its tokens, fonts, surfaces and base components +from `lib/ui`, but still has its own versions of what the library holds above those (ActionIcon, SearchInput, Logo, +SyntaxHighlightedCode and the Model UI), and `DESIGN.md`, `BRAND.md` and the full Storybook still sit beside the app. ## Releases diff --git a/ui/apps/chat/AGENTS.md b/ui/apps/chat/AGENTS.md index 32e6a009a2c9..0b4884de3167 100644 --- a/ui/apps/chat/AGENTS.md +++ b/ui/apps/chat/AGENTS.md @@ -1,6 +1,6 @@ # Instructions for ui/apps/chat -These add to the repository's [AGENTS.md](../../../AGENTS.md) for work in `ui/apps/chat`: **Llama Chat**, the chat client that llama-server bundles. **Llama UI**, the library, is `ui/lib/ui` (`@ggml-org/llama-ui`): Chat imports its tokens, fonts and base components from there (`@ggml-org/llama-ui/ui/`) and keeps no copy of them. `DESIGN.md` and the Storybook live here; the brand book, `BRAND.md`, and the brand and communications pages are in [ggml-org/llama-brand](https://github.com/ggml-org/llama-brand), with their own Storybook. +These add to the repository's [AGENTS.md](../../../AGENTS.md) for work in `ui/apps/chat`: **Llama Chat**, the chat client that llama-server bundles. **Llama UI**, the library, is `ui/lib/ui` (`@ggml-org/llama-ui`): Chat imports its tokens, fonts and base components from there (`@ggml-org/llama-ui/ui/`) and keeps no copy of them. `DESIGN.md` and the Storybook live here; the brand book, `BRAND.md`, and the brand and communications pages are in [ggml-org/llama-brand](https://github.com/ggml-org/llama-brand), with their own Storybook. The agentic loop is `ui/lib/harness` (`@ggml-org/llama-harness`). - **Design:** [DESIGN.md](DESIGN.md) holds the tokens, roles and rules, written for agents. Follow it for anything visual. Its token sections (`colors`, `typography`, `rounded`) are generated from the CSS with `npm run design:tokens`, and CI checks they are current; the rest is written by hand, and when it and the CSS disagree, the CSS wins and DESIGN.md is updated. - **Stories:** a story lives with the code it shows. Llama UI's (its foundations, base components, app components and Model UI) are in `ui/lib/ui/tests/stories`, titled `Llama UI/...`, and run as the library's tests; this app's Storybook shows them too. Llama Chat's are in `tests/stories/`, one folder per `src/lib/components/app/` (`chat/` has one per Chat folder), titled `Llama Chat//`. An accessibility story sits next to its component's, as `.a11y.stories.svelte`, titled `...//Accessibility`. The story helpers both Storybooks use (the Editorial the prose pages are set in, `StateGallery`) live with the library's stories and are imported here as `$llama-ui-stories/...`; reuse them rather than copying them here. diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 5fa37286aadf..c7f4b14465f1 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -84,15 +84,16 @@ Llama Chat is the first app on Llama UI, and the reference for the line between The app imports the library as `@ggml-org/llama-ui/ui/` and compiles its sources in its own build, with no install of the library (see `scripts/vite-plugin-llama-ui.ts`). Data flows one way, through these layers: -| Layer | Lives in | What it holds | -| -------------- | ------------------------- | --------------------------------------------------------------------------------------------------- | -| Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` | -| App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area | -| UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI | -| Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard | -| Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings | -| Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox | -| Storage + API | the browser, llama-server | IndexedDB and localStorage; llama-server's `/v1`, `/props`, `/models`, `/tools`, `/slots` | +| Layer | Lives in | What it holds | +| -------------- | ------------------------- | ---------------------------------------------------------------------------------------------------- | +| Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` | +| App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area | +| UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI | +| Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard | +| Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings | +| Agentic loop | `ui/lib/harness` | Llama Harness: streams a turn, runs the tools the model called, repeats until it answers without one | +| Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox | +| Storage + API | the browser, llama-server | IndexedDB and localStorage; llama-server's `/v1`, `/props`, `/models`, `/tools`, `/slots` | ```mermaid flowchart LR diff --git a/ui/apps/chat/sources.cmake b/ui/apps/chat/sources.cmake index 7e506c0eccf9..817bb9a22918 100644 --- a/ui/apps/chat/sources.cmake +++ b/ui/apps/chat/sources.cmake @@ -3,8 +3,9 @@ set(UI_SOURCE_GLOBS src/* static/* - # Llama UI, which the app compiles from source + # Llama UI and the harness, which the app compiles from source ../../lib/ui/src/lib/* + ../../lib/harness/src/* ) set(UI_SOURCE_FILES diff --git a/ui/apps/chat/src/lib/enums/agentic.enums.ts b/ui/apps/chat/src/lib/enums/agentic.enums.ts index 6dc46b0850ac..31f72bafd992 100644 --- a/ui/apps/chat/src/lib/enums/agentic.enums.ts +++ b/ui/apps/chat/src/lib/enums/agentic.enums.ts @@ -43,3 +43,35 @@ export enum DiffLineKind { CONTEXT = 'context', REMOVE = 'remove' } + +/** + * Events of a harness run. The values mirror `HarnessEvent` of `@ggml-org/llama-harness`. + */ +export enum HarnessEventType { + COMPLETION_ID = 'completion_id', + DONE = 'done', + ERROR = 'error', + MESSAGE = 'message', + MODEL = 'model', + REASONING = 'reasoning', + TEXT = 'text', + TIMINGS = 'timings', + TOOL_CALL = 'tool_call', + TOOL_CALL_DELTA = 'tool_call_delta', + TOOL_RESULT = 'tool_result', + TOOL_UPDATE = 'tool_update', + TURN_END = 'turn_end', + TURN_START = 'turn_start' +} + +/** + * Events of one streamed completion. The values mirror `InferenceEvent` of `@ggml-org/llama-harness`. + */ +export enum InferenceEventType { + COMPLETION_ID = 'completion_id', + MODEL = 'model', + REASONING = 'reasoning', + TEXT = 'text', + TIMINGS = 'timings', + TOOL_CALLS = 'tool_calls' +} diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts index a4eed39d8190..3378504c67ea 100644 --- a/ui/apps/chat/src/lib/enums/index.ts +++ b/ui/apps/chat/src/lib/enums/index.ts @@ -10,6 +10,8 @@ export { AgenticSectionType, ContinueIntentKind, DiffLineKind, + HarnessEventType, + InferenceEventType, ToolResultKind, ToolCallType } from './agentic.enums'; diff --git a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts index 2eb93ad3aa98..df914c5d5327 100644 --- a/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/agentic/index.svelte.ts @@ -1,15 +1,26 @@ /** - * AgenticStore - Multi-turn agentic loop orchestration + * AgenticStore - Llama Chat's side of the agentic loop * - * Drives the agentic loop over MCP tools: streams each LLM turn, detects - * tool calls, executes them via mcpStore, and enforces the turn limit. Each - * turn produces one assistant message (with tool_calls) and one tool result - * message per executed call, persisted as separate DB rows. + * The loop itself is `@ggml-org/llama-harness`: it streams each LLM turn, runs + * the tool calls and enforces the turn limit. This store gives it what the + * app owns: ChatService as its inference client, the enabled tools behind + * the permission gate, and the callbacks that persist each turn as one + * assistant message (with tool_calls) and one tool result message per call. * - * Uses ChatService for streaming and mcpStore for tool execution; waits on - * the permission/continue/steering gates owned by {@link AgenticGates}. + * Waits on the permission/continue/steering gates owned by {@link AgenticGates}. */ +import { + type ContentPart, + createHarness, + type InferenceClient, + type InferenceEvent, + type Message, + type Tool, + type ToolCall, + type ToolContext, + type ToolResult +} from '@ggml-org/llama-harness'; import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '#lib/constants/index.ts'; import { AUDIO_MIME_TO_EXTENSION, @@ -24,6 +35,8 @@ import { BuiltInTool, ToolPermissionDecision, ToolSource } from '#lib/enums/inde import { AttachmentType, ContentPartType, + HarnessEventType, + InferenceEventType, MessageRole, MimeTypePrefix, ToolCallType @@ -44,24 +57,12 @@ import type { AgenticFlowCallbacks, AgenticFlowOptions, AgenticMessage, - AgenticToolCallList, AgenticToolPolicy, SteeringMessage } from '#lib/types/agentic.js'; +import type { ApiChatMessageData } from '#lib/types/api.js'; +import type { ChatMessagePromptProgress, ChatMessageTimings } from '#lib/types/chat.js'; import type { - ApiChatCompletionToolCall, - ApiChatMessageContentPart, - ApiChatMessageData -} from '#lib/types/api.js'; -import type { - ChatMessageAgenticTimings, - ChatMessageAgenticTurnStats, - ChatMessagePromptProgress, - ChatMessageTimings, - ChatMessageToolCallTiming -} from '#lib/types/chat.js'; -import type { - DatabaseMessage, DatabaseMessageExtra, DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile @@ -76,11 +77,13 @@ import type { ToolExecutionResult } from '#lib/types/index.ts'; import type { OpenAIToolDefinition } from '#lib/types/mcp.js'; +import { createAsyncQueue } from '#lib/utils/async-queue.ts'; import { executeBrowserInfoTool, executeGetDatetimeTool, getAudioInputFormat, - isAbortError + isAbortError, + throwIfAborted } from '#lib/utils/index.ts'; import { SvelteMap } from 'svelte/reactivity'; @@ -407,22 +410,142 @@ class AgenticStore { return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; } - private buildFinalTimings( - capturedTimings: ChatMessageTimings | undefined, - agenticTimings: ChatMessageAgenticTimings - ): ChatMessageTimings | undefined { - if (agenticTimings.toolCallsCount === 0) return capturedTimings; + /** + * The media of a tool result as content parts, only what the model can read. + */ + private buildContentParts( + attachments: DatabaseMessageExtra[], + effectiveModel: string + ): ContentPart[] { + const contentParts: ContentPart[] = []; + + for (const attachment of attachments) { + if (attachment.type === AttachmentType.AUDIO) { + if (modelsStore.props.modelSupportsAudio(effectiveModel)) { + contentParts.push({ + input_audio: { + data: (attachment as DatabaseMessageExtraAudioFile).base64Data, + format: getAudioInputFormat((attachment as DatabaseMessageExtraAudioFile).mimeType) + }, + type: ContentPartType.INPUT_AUDIO + }); + } + } else if (attachment.type === AttachmentType.IMAGE) { + if (modelsStore.props.modelSupportsVision(effectiveModel)) { + contentParts.push({ + image_url: { + url: (attachment as DatabaseMessageExtraImageFile).base64Url + }, + type: ContentPartType.IMAGE_URL + }); + } else { + console.info( + `[AgenticStore] Skipping image attachment (model "${effectiveModel}" does not support vision)` + ); + } + } + } - return { - agentic: agenticTimings, - cache_n: capturedTimings?.cache_n, - predicted_ms: capturedTimings?.predicted_ms, - predicted_n: capturedTimings?.predicted_n, - prompt_ms: capturedTimings?.prompt_ms, - prompt_n: capturedTimings?.prompt_n + return contentParts; + } + + /** + * Run a tool call where its source runs it: on llama-server, in the + * browser or on an MCP server. Throws when the call fails. + */ + private async dispatchTool(params: { + callbacks: AgenticFlowCallbacks; + context: ToolContext; + conversationId: string; + effectiveModel: string; + toolCall: ToolCall; + }): Promise { + const { callbacks, context, conversationId, effectiveModel, toolCall } = params; + const { signal } = context; + const toolName = toolCall.function.name; + const toolSource = toolsStore.getToolSource(toolName); + + if ( + toolSource === ToolSource.SERVER && + toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND && + callbacks.createToolResultMessage && + callbacks.updateToolResultMessage + ) { + const args = this.parseToolArguments(toolCall.function.arguments); + const cwd = conversationsStore.activeConversation?.cwd; + + // the first update creates the result message, so the output shows live + context.update(''); + + let accumulated = ''; + let isError = false; + + for await (const ev of ToolsService.streamTool(toolName, args, signal, cwd)) { + if (ev.chunk !== null) { + accumulated += ev.chunk; + context.update(accumulated); + } + + if (ev.done) { + if (ev.error) { + accumulated = accumulated ? `${accumulated}\nError: ${ev.error}` : `Error: ${ev.error}`; + context.update(accumulated); + isError = true; + } + + break; + } + } + + return { content: accumulated, isError }; + } + + if (toolSource === ToolSource.SERVER) { + const args = this.parseToolArguments(toolCall.function.arguments); + const cwd = conversationsStore.activeConversation?.cwd; + + return ToolsService.executeTool(toolName, args, signal, cwd); + } + + if (toolSource === ToolSource.BROWSER) { + const args = this.parseToolArguments(toolCall.function.arguments); + + if (toolName === BuiltInTool.BROWSER_GET_DATETIME) return executeGetDatetimeTool(); + + if (toolName === BuiltInTool.SERVER_GET_INFO) return executeBrowserInfoTool(); + + if (toolName === BuiltInTool.BROWSER_SEARCH_CHATS) { + return SearchChatsService.executeTool(args, conversationId); + } + + if (toolName === BuiltInTool.BROWSER_READ_MEDIA) { + return ReadMediaService.executeTool( + args, + { + audio: modelsStore.props.modelSupportsAudio(effectiveModel), + vision: modelsStore.props.modelSupportsVision(effectiveModel) + }, + signal, + conversationsStore.activeConversation?.cwd + ); + } + + return SandboxService.executeTool(toolName, args, signal); + } + + const mcpCall: MCPToolCall = { + function: { arguments: toolCall.function.arguments, name: toolName }, + id: toolCall.id }; + const executionResult = await mcpStore.executeTool(mcpCall, signal); + + return { content: executionResult.content, isError: false }; } + /** + * Run the harness and turn its events into the flow's callbacks and the + * session state the UI reads. + */ private async executeAgenticLoop(params: { conversationId: string; messages: ApiChatMessageData[]; @@ -448,545 +571,252 @@ class AgenticStore { onTurnComplete, updateToolResultMessage } = callbacks; - const sessionMessages: AgenticMessage[] = toAgenticMessages(messages); - - let capturedTimings: ChatMessageTimings | undefined; - let totalToolCallCount = 0; - - const agenticTimings: ChatMessageAgenticTimings = { - llm: { predicted_ms: 0, predicted_n: 0, prompt_ms: 0, prompt_n: 0 }, - perTurn: [], - toolCalls: [], - toolCallsCount: 0, - toolsMs: 0, - turns: 0 - }; - const maxTurns = agenticConfig.maxTurns; const effectiveModel = options.model || modelsStore.models[0]?.model || ''; - - let turn = 0; - - while (true) { - if (turn >= maxTurns) { + const harness = createHarness(this.inferenceClient(options, conversationId)); + const harnessTools: Tool[] = tools.map((definition) => ({ + definition, + execute: (toolCall, context) => + this.executeTool({ callbacks, context, conversationId, effectiveModel, toolCall }) + })); + // result messages of streaming tools, created while the tool runs, by tool call id + const streamedResults = new SvelteMap(); + const events = harness.run(toAgenticMessages(messages) as Message[], harnessTools, signal, { + interrupted: () => this.gates.hasPendingSteeringMessage(conversationId), + maxTurns: agenticConfig.maxTurns, + onTurnLimit: async () => { // Turn limit reached - ask user whether to continue const shouldContinue = await this.gates.requestContinue(conversationId, signal); // Yield to allow Svelte to flush the UI update await new Promise((r) => setTimeout(r, 0)); - if (!shouldContinue || signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); - - return; - } - - // User chose to continue - extend the limit - turn = 0; - } - - this.updateSession(conversationId, { currentTurn: turn + 1 }); - agenticTimings.turns = turn + 1; - - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); - - return; - } - - // For turns > 0, create a new assistant message via callback - if (turn > 0 && createAssistantMessage) { - await createAssistantMessage(); + return shouldContinue; } + }); - let turnContent = ''; - let turnReasoningContent = ''; - let turnToolCalls: ApiChatCompletionToolCall[] = []; - let lastStreamingToolCallName = ''; - let lastStreamingToolCallArgsLength = 0; - let turnTimings: ChatMessageTimings | undefined; - - const turnStats: ChatMessageAgenticTurnStats = { - llm: { predicted_ms: 0, predicted_n: 0, prompt_ms: 0, prompt_n: 0 }, - toolCalls: [], - toolsMs: 0, - turn: turn + 1 - }; - - try { - await ChatService.sendMessage( - sessionMessages as ApiChatMessageData[], - { - ...options, - onChunk: (chunk: string) => { - turnContent += chunk; - onChunk?.(chunk); - }, - onComplete: () => { - /* Completion handled after sendMessage resolves */ - }, - onCompletionId, - onError: (error: Error) => { - throw error; - }, - onModel, - onReasoningChunk: (chunk: string) => { - turnReasoningContent += chunk; - onReasoningChunk?.(chunk); - }, - onTimings: (timings?: ChatMessageTimings, progress?: ChatMessagePromptProgress) => { - onTimings?.(timings, progress); - - if (timings) { - capturedTimings = timings; - turnTimings = timings; - - // completed turns + in-flight turn live counts - this.updateSession(conversationId, { - liveLlm: { - predicted_ms: agenticTimings.llm.predicted_ms + (timings.predicted_ms ?? 0), - predicted_n: agenticTimings.llm.predicted_n + (timings.predicted_n ?? 0), - prompt_ms: agenticTimings.llm.prompt_ms + (timings.prompt_ms ?? 0), - prompt_n: agenticTimings.llm.prompt_n + (timings.prompt_n ?? 0) - } - }); - } - }, - onToolCallChunk: (serialized: string) => { - try { - turnToolCalls = JSON.parse(serialized) as ApiChatCompletionToolCall[]; - - onToolCallsStreaming?.(turnToolCalls); - - if (turnToolCalls.length > 0 && turnToolCalls[0]?.function) { - const name = turnToolCalls[0].function.name || ''; - const args = turnToolCalls[0].function.arguments || ''; - const argsLengthBucket = Math.floor(args.length / 100); - - if ( - name !== lastStreamingToolCallName || - argsLengthBucket !== lastStreamingToolCallArgsLength - ) { - lastStreamingToolCallName = name; - lastStreamingToolCallArgsLength = argsLengthBucket; - this.updateSession(conversationId, { - streamingToolCall: { arguments: args, name } - }); - } - } - } catch { - /* Ignore parse errors during streaming */ - } - }, - stream: true, - tools: tools.length > 0 ? tools : undefined - }, - conversationId, - signal - ); + let totalToolCallCount = 0; + let lastStreamingToolCallName = ''; + let lastStreamingToolCallArgsLength = 0; - this.updateSession(conversationId, { streamingToolCall: null }); - - if (turnTimings) { - agenticTimings.llm.predicted_n += turnTimings.predicted_n || 0; - agenticTimings.llm.predicted_ms += turnTimings.predicted_ms || 0; - agenticTimings.llm.prompt_n += turnTimings.prompt_n || 0; - agenticTimings.llm.prompt_ms += turnTimings.prompt_ms || 0; - turnStats.llm.predicted_n = turnTimings.predicted_n || 0; - turnStats.llm.predicted_ms = turnTimings.predicted_ms || 0; - turnStats.llm.prompt_n = turnTimings.prompt_n || 0; - turnStats.llm.prompt_ms = turnTimings.prompt_ms || 0; - } - } catch (error) { - if (signal?.aborted) { - // Save whatever we have for this turn before exiting - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + for await (const event of events) { + switch (event.type) { + case HarnessEventType.TURN_START: + this.updateSession(conversationId, { currentTurn: event.turn }); + lastStreamingToolCallName = ''; + lastStreamingToolCallArgsLength = 0; - return; - } + // the caller created the first turn's assistant message + if (event.turn > 1 && createAssistantMessage) await createAssistantMessage(); - const normalizedError = error instanceof Error ? error : new Error('LLM stream error'); + break; - // preserve partial output as is, the outer error dialog informs the user separately - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.TEXT: + onChunk?.(event.delta); - throw normalizedError; - } + break; - // If the abort landed while ChatService.sendMessage was still resolving, the - // outer catch above never fires because ChatService swallows the AbortError - // and returns normally. Bail out here so a half-received tool_call (truncated - // arguments JSON) is not persisted as if it were complete. - if (signal?.aborted) { - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.REASONING: + onReasoningChunk?.(event.delta); - return; - } + break; - // === Steering check: if a user message was queued during this turn, exit the flow. - // The caller (chatStore) will consume the pending message and re-send it normally. - if (this.gates.hasPendingSteeringMessage(conversationId)) { - console.log('[AgenticStore] Steering message detected after turn, exiting agentic flow'); - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - turnToolCalls.length > 0 ? this.normalizeToolCalls(turnToolCalls) : undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.TOOL_CALL_DELTA: { + onToolCallsStreaming?.(event.toolCalls); - return; - } + const streamingCall = event.toolCalls[0]?.function; - // No tool calls = final turn, save and complete - if (turnToolCalls.length === 0) { - agenticTimings.perTurn!.push(turnStats); + if (streamingCall) { + const name = streamingCall.name || ''; + const args = streamingCall.arguments || ''; + const argsLengthBucket = Math.floor(args.length / 100); - const finalTimings = this.buildFinalTimings(capturedTimings, agenticTimings); - - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - finalTimings, - undefined - ); + if ( + name !== lastStreamingToolCallName || + argsLengthBucket !== lastStreamingToolCallArgsLength + ) { + lastStreamingToolCallName = name; + lastStreamingToolCallArgsLength = argsLengthBucket; + this.updateSession(conversationId, { + streamingToolCall: { arguments: args, name } + }); + } + } - if (finalTimings) onTurnComplete?.(finalTimings); + break; + } - onFlowComplete?.(finalTimings); + case HarnessEventType.TIMINGS: + onTimings?.(event.timings, event.progress); - return; - } + // completed turns + in-flight turn live counts + if (event.llm) this.updateSession(conversationId, { liveLlm: event.llm }); - // Normalize and save assistant turn with tool calls - const normalizedCalls = this.normalizeToolCalls(turnToolCalls); + break; - if (normalizedCalls.length === 0) { - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - this.buildFinalTimings(capturedTimings, agenticTimings), - undefined - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.MODEL: + onModel?.(event.model); - return; - } + break; - totalToolCallCount += normalizedCalls.length; - this.updateSession(conversationId, { totalToolCalls: totalToolCallCount }); - - // Save the assistant message with its tool calls - await onAssistantTurnComplete?.( - turnContent, - turnReasoningContent || undefined, - turnTimings, - normalizedCalls - ); - - // Add assistant message to session history - sessionMessages.push({ - content: turnContent || undefined, - reasoning_content: turnReasoningContent || undefined, - role: MessageRole.ASSISTANT, - tool_calls: normalizedCalls - }); + case HarnessEventType.COMPLETION_ID: + onCompletionId?.(event.id); - // Execute each tool call and create result messages - for (let i = 0; i < normalizedCalls.length; i++) { - const toolCall = normalizedCalls[i]; + break; - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + case HarnessEventType.MESSAGE: + this.updateSession(conversationId, { streamingToolCall: null }); - return; - } + if (event.toolCalls && !event.final) { + totalToolCallCount += event.toolCalls.length; + this.updateSession(conversationId, { totalToolCalls: totalToolCallCount }); + } - // Check for pending steering message - skip remaining tool calls - if (this.gates.hasPendingSteeringMessage(conversationId)) { - console.log( - `[AgenticStore] Steering message detected, skipping ${normalizedCalls.length - i} remaining tool call(s)` + await onAssistantTurnComplete?.( + event.content, + event.reasoning, + event.timings, + event.toolCalls ); - for (let j = i; j < normalizedCalls.length; j++) { - const remainingCall = normalizedCalls[j]; - const interruptedContent = 'Tool execution was interrupted by a new user message.'; - if (createToolResultMessage) { - await createToolResultMessage(remainingCall.id, interruptedContent); - } + break; - sessionMessages.push({ - content: interruptedContent, - role: MessageRole.TOOL, - tool_call_id: remainingCall.id + case HarnessEventType.TOOL_UPDATE: { + if (!createToolResultMessage || !updateToolResultMessage) break; + + const streamed = streamedResults.get(event.toolCall.id); + + if (streamed) { + streamed.content = event.content; + await updateToolResultMessage(streamed.messageId, event.content); + } else { + const message = await createToolResultMessage( + event.toolCall.id, + event.content, + undefined, + conversationsStore.activeConversation?.cwd + ); + + streamedResults.set(event.toolCall.id, { + content: event.content, + messageId: message.id }); } break; } - const toolName = toolCall.function.name; - const serverLabel = toolsStore.getToolServerLabel(toolName); - // Ask for permission before executing the tool - const permission = await this.gates.requestPermission( - conversationId, - toolName, - serverLabel, - signal - ); + case HarnessEventType.TOOL_RESULT: { + const { content } = event.result; + const attachments = (event.result.details as DatabaseMessageExtra[] | undefined) ?? []; + const extras = attachments.length > 0 ? attachments : undefined; + const streamed = streamedResults.get(event.toolCall.id); - // Yield to allow Svelte to flush the UI update (hide permission dialog) - await new Promise((r) => setTimeout(r, 0)); - - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + let toolResultMessageId: string | undefined; - return; - } + if (streamed) { + toolResultMessageId = streamed.messageId; - const toolStartTime = performance.now(); - const toolSource = toolsStore.getToolSource(toolName); - - let result = ''; - let toolSuccess = true; - let createdToolResultMessageId: string | null = null; - - // Streaming tools (currently only exec_shell_command): mark - // the session so the matching renderer can switch to live mode. - // Cleared unconditionally below. - this.updateSession(conversationId, { executingToolCallId: toolCall.id }); - - if (permission === ToolPermissionDecision.DENY) { - result = 'Tool execution was denied by the user.'; - toolSuccess = false; - } else { - try { - if ( - toolSource === ToolSource.SERVER && - toolName === BuiltInTool.SERVER_EXEC_SHELL_COMMAND && - createToolResultMessage && - updateToolResultMessage - ) { - const args = this.parseToolArguments(toolCall.function.arguments); - const cwd = conversationsStore.activeConversation?.cwd; - const msg = await createToolResultMessage(toolCall.id, '', undefined, cwd); - - createdToolResultMessageId = msg.id; - - let accumulated = ''; - - for await (const ev of ToolsService.streamTool(toolName, args, signal, cwd)) { - if (ev.chunk !== null) { - accumulated += ev.chunk; - await updateToolResultMessage(msg.id, accumulated); - } - - if (ev.done) { - if (ev.error) { - accumulated = accumulated - ? `${accumulated}\nError: ${ev.error}` - : `Error: ${ev.error}`; - await updateToolResultMessage(msg.id, accumulated); - toolSuccess = false; - } - - break; - } - } - result = accumulated; - } else if (toolSource === ToolSource.SERVER) { - const args = this.parseToolArguments(toolCall.function.arguments); - const cwd = conversationsStore.activeConversation?.cwd; - const executionResult = await ToolsService.executeTool(toolName, args, signal, cwd); - - result = executionResult.content; - - if (executionResult.isError) toolSuccess = false; - } else if (toolSource === ToolSource.BROWSER) { - const args = this.parseToolArguments(toolCall.function.arguments); - - let executionResult: ToolExecutionResult; - - if (toolName === BuiltInTool.BROWSER_GET_DATETIME) { - executionResult = executeGetDatetimeTool(); - } else if (toolName === BuiltInTool.SERVER_GET_INFO) { - executionResult = executeBrowserInfoTool(); - } else if (toolName === BuiltInTool.BROWSER_SEARCH_CHATS) { - executionResult = await SearchChatsService.executeTool(args, conversationId); - } else if (toolName === BuiltInTool.BROWSER_READ_MEDIA) { - executionResult = await ReadMediaService.executeTool( - args, - { - audio: modelsStore.props.modelSupportsAudio(effectiveModel), - vision: modelsStore.props.modelSupportsVision(effectiveModel) - }, - signal, - conversationsStore.activeConversation?.cwd - ); - } else { - executionResult = await SandboxService.executeTool(toolName, args, signal); - } - - result = executionResult.content; - - if (executionResult.isError) toolSuccess = false; - } else { - const mcpCall: MCPToolCall = { - function: { arguments: toolCall.function.arguments, name: toolName }, - id: toolCall.id - }; - const executionResult = await mcpStore.executeTool(mcpCall, signal); - - result = executionResult.content; - } - } catch (error) { - if (isAbortError(error)) { - this.updateSession(conversationId, { executingToolCallId: null }); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); - - return; - } - - // Carry the partial stream contents already mirrored to the UI - - // they show up as live output even if the stream broke off mid-run. - result = result - ? `${result}\nError: ${error instanceof Error ? error.message : String(error)}` - : `Error: ${error instanceof Error ? error.message : String(error)}`; - toolSuccess = false; - - if (createdToolResultMessageId && updateToolResultMessage) { - await updateToolResultMessage(createdToolResultMessageId, result); + // the message already shows the streamed output: rewrite it only when the result differs + if (extras || content !== streamed.content) { + await updateToolResultMessage?.(streamed.messageId, content, extras); } + } else if (createToolResultMessage) { + toolResultMessageId = ( + await createToolResultMessage(event.toolCall.id, content, extras) + ).id; } - } - this.updateSession(conversationId, { executingToolCallId: null }); + if (extras && toolResultMessageId) onAttachments?.(toolResultMessageId, extras); - const toolDurationMs = performance.now() - toolStartTime; - const toolTiming: ChatMessageToolCallTiming = { - duration_ms: Math.round(toolDurationMs), - name: toolCall.function.name, - success: toolSuccess - }; + break; + } - agenticTimings.toolCalls!.push(toolTiming); - agenticTimings.toolCallsCount++; - agenticTimings.toolsMs += Math.round(toolDurationMs); - turnStats.toolCalls.push(toolTiming); - turnStats.toolsMs += Math.round(toolDurationMs); + case HarnessEventType.TURN_END: + onTurnComplete?.(event.timings); - if (signal?.aborted) { - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + break; - return; - } + case HarnessEventType.DONE: + onFlowComplete?.(event.timings); - const { attachments, cleanedResult } = this.extractBase64Attachments(result); + break; - // For streaming tools the result message was created empty - // at the start of execution and updated in place as chunks - // arrived via updateToolResultMessage. Skip the second - // create call - just attach any base64 attachments found in - // the final accumulator (rare, since chunks usually don't - // carry image data URIs) and emit the attachments callback. - let toolResultMessage: DatabaseMessage | undefined; + case HarnessEventType.ERROR: + onFlowComplete?.(event.timings); - if (createdToolResultMessageId) { - toolResultMessage = { id: createdToolResultMessageId } as DatabaseMessage; + throw event.error; + } + } + } - if (attachments.length > 0 && updateToolResultMessage) { - await updateToolResultMessage(createdToolResultMessageId, cleanedResult, attachments); - } - } else if (createToolResultMessage) { - toolResultMessage = await createToolResultMessage( - toolCall.id, - cleanedResult, - attachments.length > 0 ? attachments : undefined - ); - } + /** + * A tool of the harness: the permission gate, then the call, then the + * result with its media taken out of the text. + */ + private async executeTool(params: { + callbacks: AgenticFlowCallbacks; + context: ToolContext; + conversationId: string; + effectiveModel: string; + toolCall: ToolCall; + }): Promise { + const { callbacks, context, conversationId, effectiveModel, toolCall } = params; + const toolName = toolCall.function.name; + const serverLabel = toolsStore.getToolServerLabel(toolName); + // Ask for permission before executing the tool + const permission = await this.gates.requestPermission( + conversationId, + toolName, + serverLabel, + context.signal + ); - if (attachments.length > 0 && toolResultMessage) { - onAttachments?.(toolResultMessage.id, attachments); - } + // Yield to allow Svelte to flush the UI update (hide permission dialog) + await new Promise((r) => setTimeout(r, 0)); - // Build content parts for session history (including images for vision models) - const contentParts: ApiChatMessageContentPart[] = [ - { text: cleanedResult, type: ContentPartType.TEXT } - ]; - - for (const attachment of attachments) { - if (attachment.type === AttachmentType.AUDIO) { - if (modelsStore.props.modelSupportsAudio(effectiveModel)) { - contentParts.push({ - input_audio: { - data: (attachment as DatabaseMessageExtraAudioFile).base64Data, - format: getAudioInputFormat( - (attachment as DatabaseMessageExtraAudioFile).mimeType - ) - }, - type: ContentPartType.INPUT_AUDIO - }); - } - } else if (attachment.type === AttachmentType.IMAGE) { - if (modelsStore.props.modelSupportsVision(effectiveModel)) { - contentParts.push({ - image_url: { - url: (attachment as DatabaseMessageExtraImageFile).base64Url - }, - type: ContentPartType.IMAGE_URL - }); - } else { - console.info( - `[AgenticStore] Skipping image attachment (model "${effectiveModel}" does not support vision)` - ); - } - } - } + throwIfAborted(context.signal); - sessionMessages.push({ - content: contentParts.length === 1 ? cleanedResult : contentParts, - role: MessageRole.TOOL, - tool_call_id: toolCall.id - }); - } + // the time waiting for permission does not count + const toolStartTime = performance.now(); - if (turnStats.toolCalls.length > 0) { - agenticTimings.perTurn!.push(turnStats); + let execution: ToolExecutionResult; - const intermediateTimings = this.buildFinalTimings(capturedTimings, agenticTimings); + // Streaming tools (currently only exec_shell_command): mark + // the session so the matching renderer can switch to live mode. + this.updateSession(conversationId, { executingToolCallId: toolCall.id }); - if (intermediateTimings) onTurnComplete?.(intermediateTimings); - } + try { + execution = + permission === ToolPermissionDecision.DENY + ? { content: 'Tool execution was denied by the user.', isError: true } + : await this.dispatchTool({ + callbacks, + context, + conversationId, + effectiveModel, + toolCall + }); + } catch (error) { + if (isAbortError(error)) throw error; - // If tools were interrupted by a steering message, exit now instead of starting another LLM turn - if (this.gates.hasPendingSteeringMessage(conversationId)) { - console.log( - '[AgenticStore] Steering message detected after tool execution, exiting agentic flow' - ); - onFlowComplete?.(this.buildFinalTimings(capturedTimings, agenticTimings)); + execution = { + content: `Error: ${error instanceof Error ? error.message : String(error)}`, + isError: true + }; + } finally { + this.updateSession(conversationId, { executingToolCallId: null }); + } - return; - } + const durationMs = performance.now() - toolStartTime; + const { attachments, cleanedResult } = this.extractBase64Attachments(execution.content); - turn++; - } + return { + content: cleanedResult, + details: attachments, + durationMs, + isError: execution.isError, + parts: this.buildContentParts(attachments, effectiveModel) + }; } private extractBase64Attachments(result: string): { @@ -1044,17 +874,57 @@ class AgenticStore { return { attachments, cleanedResult: cleanedLines.join(NEWLINE) }; } - private normalizeToolCalls(toolCalls: ApiChatCompletionToolCall[]): AgenticToolCallList { - if (!toolCalls) return []; + /** + * ChatService as the harness's inference client: one streamed completion + * per turn, its callbacks turned into events. + */ + private inferenceClient(options: AgenticFlowOptions, conversationId: string): InferenceClient { + return { + stream: ({ messages, signal, tools }) => { + const events = createAsyncQueue(); - return toolCalls.map((call, index) => ({ - function: { - arguments: call?.function?.arguments ?? '', - name: call?.function?.name ?? '' - }, - id: call?.id ?? `tool_${index}`, - type: (call?.type as ToolCallType.FUNCTION) ?? ToolCallType.FUNCTION - })); + ChatService.sendMessage( + messages as ApiChatMessageData[], + { + ...options, + onChunk: (chunk: string) => + events.push({ delta: chunk, type: InferenceEventType.TEXT }), + onComplete: () => { + /* Completion handled after sendMessage resolves */ + }, + onCompletionId: (id: string) => + events.push({ id, type: InferenceEventType.COMPLETION_ID }), + onError: (error: Error) => { + throw error; + }, + onModel: (model: string) => events.push({ model, type: InferenceEventType.MODEL }), + onReasoningChunk: (chunk: string) => + events.push({ delta: chunk, type: InferenceEventType.REASONING }), + onTimings: (timings?: ChatMessageTimings, progress?: ChatMessagePromptProgress) => + events.push({ progress, timings, type: InferenceEventType.TIMINGS }), + onToolCallChunk: (serialized: string) => { + try { + events.push({ + toolCalls: JSON.parse(serialized), + type: InferenceEventType.TOOL_CALLS + }); + } catch { + /* Ignore parse errors during streaming */ + } + }, + stream: true, + tools + }, + conversationId, + signal + ).then( + () => events.close(), + (error: unknown) => events.fail(error) + ); + + return events; + } + }; } private parseToolArguments(args: string | Record): Record { diff --git a/ui/apps/chat/src/lib/utils/async-queue.ts b/ui/apps/chat/src/lib/utils/async-queue.ts new file mode 100644 index 000000000000..48988dd93951 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/async-queue.ts @@ -0,0 +1,50 @@ +/** + * A queue read as an async iterable, to turn a callback API into a stream. + * The iteration ends after `close`, and throws after `fail`, once the pushed items are read. + */ +export interface AsyncQueue extends AsyncIterable { + push(item: T): void; + close(): void; + fail(error: unknown): void; +} + +export function createAsyncQueue(): AsyncQueue { + const items: T[] = []; + + let closed = false; + let failure: { error: unknown } | undefined; + let wake: (() => void) | undefined; + + const notify = () => { + wake?.(); + wake = undefined; + }; + + return { + close() { + closed = true; + notify(); + }, + fail(error) { + failure = { error }; + notify(); + }, + push(item) { + items.push(item); + notify(); + }, + async *[Symbol.asyncIterator]() { + while (true) { + if (items.length > 0) { + yield items.shift()!; + } else if (failure) { + throw failure.error; + } else if (closed) { + return; + } else { + await new Promise((resolve) => (wake = resolve)); + } + } + } + }; +} diff --git a/ui/apps/chat/tests/unit/agentic-flow.test.ts b/ui/apps/chat/tests/unit/agentic-flow.test.ts new file mode 100644 index 000000000000..e8992b60c841 --- /dev/null +++ b/ui/apps/chat/tests/unit/agentic-flow.test.ts @@ -0,0 +1,372 @@ +// The agentic flow end to end, with the server and the tools mocked: what the +// model is sent each turn, and the order the flow's callbacks fire in. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('#lib/services/database.service.ts', () => ({ + DatabaseService: {} +})); + +import { BuiltInTool, MessageRole, ToolPermissionDecision, ToolSource } from '#lib/enums/index.ts'; +import { ChatService } from '#lib/services/chat.service.ts'; +import { ToolsService } from '#lib/services/tools.service.ts'; +import { agenticStore } from '#lib/stores/agentic/index.svelte.ts'; +import { modelsStore } from '#lib/stores/models/index.svelte.ts'; +import { permissionsStore } from '#lib/stores/permissions.svelte.ts'; +import { toolsStore } from '#lib/stores/tools.svelte.ts'; +import type { AgenticFlowCallbacks } from '#lib/types/agentic.js'; +import type { ApiChatMessageData } from '#lib/types/api.js'; +import type { ChatMessageTimings } from '#lib/types/chat.js'; +import type { DatabaseMessage } from '#lib/types/database.js'; + +const CONV_ID = 'conv-1'; +const USER_MESSAGES: ApiChatMessageData[] = [{ content: 'hi', role: MessageRole.USER }]; +const IMAGE_DATA_URI = 'data:image/png;base64,AAAA'; + +type StreamOptions = { + onChunk?: (chunk: string) => void; + onReasoningChunk?: (chunk: string) => void; + onTimings?: (timings?: ChatMessageTimings) => void; + onToolCallChunk?: (serialized: string) => void; +}; +type Turn = (options: StreamOptions) => void | Promise; + +function toolCalls(...names: string[]): string { + return JSON.stringify( + names.map((name, index) => ({ function: { arguments: '{}', name }, id: `call_${index}` })) + ); +} + +// each request plays the next scripted turn; returns what every request was sent +function mockTurns(...turns: Turn[]): ApiChatMessageData[][] { + const requests: ApiChatMessageData[][] = []; + + vi.spyOn(ChatService, 'sendMessage').mockImplementation(async (messages, options) => { + requests.push(structuredClone(messages) as ApiChatMessageData[]); + await turns[requests.length - 1]?.(options as StreamOptions); + }); + + return requests; +} + +function mockTools(source: ToolSource, ...names: string[]) { + vi.spyOn(toolsStore, 'getEnabledToolsForLLM').mockReturnValue( + names.map((name) => ({ function: { name, parameters: {} }, type: 'function' })) + ); + vi.spyOn(toolsStore, 'getToolSource').mockReturnValue(source); +} + +// callbacks that log their calls in order +function recorder() { + const calls: unknown[][] = []; + const log = + (name: string) => + (...args: unknown[]) => { + calls.push([name, ...args]); + }; + + let nextMessageId = 0; + + const createMessage = + (name: string) => + async (...args: unknown[]) => { + calls.push([name, ...args]); + + return { id: `msg_${nextMessageId++}` } as DatabaseMessage; + }; + const callbacks: AgenticFlowCallbacks = { + createAssistantMessage: createMessage('createAssistantMessage'), + createToolResultMessage: createMessage('createToolResultMessage'), + onAssistantTurnComplete: async (...args) => { + calls.push(['onAssistantTurnComplete', ...args]); + }, + onAttachments: log('onAttachments'), + onChunk: log('onChunk'), + onError: log('onError'), + onFlowComplete: log('onFlowComplete'), + onReasoningChunk: log('onReasoningChunk'), + onToolCallsStreaming: log('onToolCallsStreaming'), + onTurnComplete: log('onTurnComplete'), + updateToolResultMessage: async (...args) => { + calls.push(['updateToolResultMessage', ...args]); + } + }; + + return { + callbacks, + calls, + named: (name: string) => calls.filter((call) => call[0] === name), + names: () => calls.map((call) => call[0]) + }; +} + +function runFlow(callbacks: AgenticFlowCallbacks, signal?: AbortSignal) { + return agenticStore.runAgenticFlow({ + callbacks, + conversationId: CONV_ID, + messages: USER_MESSAGES, + signal + }); +} + +beforeEach(() => { + vi.restoreAllMocks(); + agenticStore.clearSession(CONV_ID); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(ChatService, 'normalizeMessagesForApi').mockImplementation( + async (messages) => messages as ApiChatMessageData[] + ); + vi.spyOn(toolsStore, 'fetchServerTools').mockResolvedValue(); + vi.spyOn(toolsStore, 'getToolServerLabel').mockReturnValue('Browser'); + vi.spyOn(toolsStore, 'getPermissionKey').mockReturnValue('tool-key'); + vi.spyOn(permissionsStore, 'hasTool').mockReturnValue(true); +}); + +describe('agentic flow', () => { + it('is not handled when no tool is enabled', async () => { + vi.spyOn(toolsStore, 'getEnabledToolsForLLM').mockReturnValue([]); + + const send = vi.spyOn(ChatService, 'sendMessage'); + + expect(await runFlow(recorder().callbacks)).toEqual({ handled: false }); + expect(send).not.toHaveBeenCalled(); + }); + + it('runs a tool call, then lets the model answer', async () => { + mockTools(ToolSource.BROWSER, BuiltInTool.BROWSER_GET_DATETIME); + + const requests = mockTurns( + (options) => { + options.onChunk?.('checking'); + options.onToolCallChunk?.(toolCalls(BuiltInTool.BROWSER_GET_DATETIME)); + options.onTimings?.({ predicted_ms: 10, predicted_n: 4, prompt_ms: 20, prompt_n: 8 }); + }, + (options) => { + options.onReasoningChunk?.('hm'); + options.onChunk?.('it is noon'); + options.onTimings?.({ predicted_ms: 5, predicted_n: 2, prompt_ms: 1, prompt_n: 1 }); + } + ); + const { callbacks, named, names } = recorder(); + + expect(await runFlow(callbacks)).toEqual({ handled: true }); + expect(names()).toEqual([ + 'onChunk', + 'onToolCallsStreaming', + 'onAssistantTurnComplete', + 'createToolResultMessage', + 'onTurnComplete', + 'createAssistantMessage', + 'onReasoningChunk', + 'onChunk', + 'onAssistantTurnComplete', + 'onTurnComplete', + 'onFlowComplete' + ]); + + const [toolTurn, finalTurn] = named('onAssistantTurnComplete'); + const call = { + function: { arguments: '{}', name: BuiltInTool.BROWSER_GET_DATETIME }, + id: 'call_0', + type: 'function' + }; + + expect(toolTurn.slice(1)).toEqual([ + 'checking', + undefined, + { predicted_ms: 10, predicted_n: 4, prompt_ms: 20, prompt_n: 8 }, + [call] + ]); + expect(finalTurn.slice(1, 3)).toEqual(['it is noon', 'hm']); + expect(finalTurn[4]).toBeUndefined(); + + const [, toolCallId, toolResult, extras] = named('createToolResultMessage')[0]; + + expect([toolCallId, extras]).toEqual(['call_0', undefined]); + expect(requests[1].slice(1)).toEqual([ + { content: 'checking', role: 'assistant', tool_calls: [call] }, + { content: toolResult, role: 'tool', tool_call_id: 'call_0' } + ]); + expect(named('onFlowComplete')[0][1]).toMatchObject({ + agentic: { + llm: { predicted_ms: 15, predicted_n: 6, prompt_ms: 21, prompt_n: 9 }, + perTurn: [{ turn: 1 }, { turn: 2 }], + toolCalls: [{ name: BuiltInTool.BROWSER_GET_DATETIME, success: true }], + toolCallsCount: 1, + turns: 2 + }, + predicted_n: 2 + }); + expect(agenticStore.getTotalToolCalls(CONV_ID)).toBe(1); + expect(agenticStore.isRunning(CONV_ID)).toBe(false); + }); + + it('waits for permission and tells the model when it is denied', async () => { + mockTools(ToolSource.BROWSER, BuiltInTool.BROWSER_GET_DATETIME); + vi.mocked(permissionsStore.hasTool).mockReturnValue(false); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls(BuiltInTool.BROWSER_GET_DATETIME)), + (options) => options.onChunk?.('ok') + ); + const { callbacks, named } = recorder(); + const flow = runFlow(callbacks); + + await vi.waitFor(() => + expect(agenticStore.getPendingPermissionRequest(CONV_ID)).toEqual({ + serverLabel: 'Browser', + toolName: BuiltInTool.BROWSER_GET_DATETIME + }) + ); + agenticStore.resolvePermission(CONV_ID, ToolPermissionDecision.DENY); + await flow; + + expect(named('createToolResultMessage')[0].slice(1)).toEqual([ + 'call_0', + 'Tool execution was denied by the user.', + undefined + ]); + expect(requests[1][2]).toMatchObject({ content: 'Tool execution was denied by the user.' }); + expect(named('onFlowComplete')[0][1]).toMatchObject({ + agentic: { toolCalls: [{ success: false }] } + }); + }); + + it('reports a failing tool to the model and goes on', async () => { + mockTools(ToolSource.SERVER, 'read_file'); + vi.spyOn(ToolsService, 'executeTool').mockRejectedValue(new Error('no such file')); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls('read_file')), + (options) => options.onChunk?.('sorry') + ); + const { callbacks } = recorder(); + + expect(await runFlow(callbacks)).toEqual({ handled: true }); + expect(requests[1][2]).toMatchObject({ content: 'Error: no such file', role: 'tool' }); + }); + + it('shows the output of a streaming tool while it runs', async () => { + mockTools(ToolSource.SERVER, BuiltInTool.SERVER_EXEC_SHELL_COMMAND); + vi.spyOn(ToolsService, 'streamTool').mockImplementation(async function* () { + yield { chunk: 'one\n', done: false }; + await new Promise((resolve) => setTimeout(resolve, 0)); + yield { chunk: 'two\n', done: false }; + await new Promise((resolve) => setTimeout(resolve, 0)); + yield { chunk: null, done: true }; + }); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls(BuiltInTool.SERVER_EXEC_SHELL_COMMAND)), + (options) => options.onChunk?.('done') + ); + const { callbacks, named } = recorder(); + + await runFlow(callbacks); + + // created empty when the tool starts, then rewritten in place: never created twice + expect(named('createToolResultMessage').map((call) => call.slice(1))).toEqual([ + ['call_0', '', undefined, undefined] + ]); + expect(named('updateToolResultMessage').map((call) => call.slice(1))).toEqual([ + ['msg_0', 'one\n'], + ['msg_0', 'one\ntwo\n'] + ]); + expect(requests[1][2]).toMatchObject({ content: 'one\ntwo\n', role: 'tool' }); + }); + + it('takes an image out of a tool result and sends it to a model with vision', async () => { + mockTools(ToolSource.SERVER, 'screenshot'); + vi.spyOn(ToolsService, 'executeTool').mockResolvedValue({ + content: `saved\n${IMAGE_DATA_URI}`, + isError: false + }); + vi.spyOn(modelsStore.props, 'modelSupportsVision').mockReturnValue(true); + + const requests = mockTurns( + (options) => options.onToolCallChunk?.(toolCalls('screenshot')), + (options) => options.onChunk?.('here') + ); + const { callbacks, named } = recorder(); + + await runFlow(callbacks); + + const [, , content, extras] = named('createToolResultMessage')[0]; + + expect(content).toMatch(/^saved\n\[Attachment saved: .+\.png\]$/); + expect(extras).toMatchObject([{ base64Url: IMAGE_DATA_URI, type: 'IMAGE' }]); + expect(named('onAttachments')[0].slice(1)).toEqual(['msg_0', extras]); + expect(requests[1][2]).toEqual({ + content: [ + { text: content, type: 'text' }, + { image_url: { url: IMAGE_DATA_URI }, type: 'image_url' } + ], + role: 'tool', + tool_call_id: 'call_0' + }); + }); + + it('leaves the flow when a steering message is queued, skipping the tool calls left', async () => { + mockTools(ToolSource.SERVER, 'first', 'second'); + vi.spyOn(ToolsService, 'executeTool').mockImplementation(async () => { + agenticStore.injectSteeringMessage(CONV_ID, 'wait'); + + return { content: 'first ran', isError: false }; + }); + + const requests = mockTurns((options) => + options.onToolCallChunk?.(toolCalls('first', 'second')) + ); + const { callbacks, named, names } = recorder(); + + expect(await runFlow(callbacks)).toEqual({ handled: true }); + expect(named('createToolResultMessage').map((call) => call.slice(1, 3))).toEqual([ + ['call_0', 'first ran'], + ['call_1', 'Tool execution was interrupted by a new user message.'] + ]); + expect(names().slice(-2)).toEqual(['onTurnComplete', 'onFlowComplete']); + expect(requests).toHaveLength(1); + expect(agenticStore.consumePendingSteeringMessage(CONV_ID)).toMatchObject({ content: 'wait' }); + }); + + it('keeps the partial turn and reports the error when the stream fails', async () => { + mockTools(ToolSource.BROWSER, BuiltInTool.BROWSER_GET_DATETIME); + mockTurns((options) => { + options.onChunk?.('par'); + + throw new Error('HTTP 500'); + }); + + const { callbacks, named, names } = recorder(); + const result = await runFlow(callbacks); + + expect(result).toMatchObject({ error: { message: 'HTTP 500' }, handled: true }); + expect(names()).toEqual(['onChunk', 'onAssistantTurnComplete', 'onFlowComplete', 'onError']); + expect(named('onAssistantTurnComplete')[0].slice(1)).toEqual([ + 'par', + undefined, + undefined, + undefined + ]); + expect(agenticStore.getLastError(CONV_ID)?.message).toBe('HTTP 500'); + }); + + it('stops without a tool result when the user aborts during a tool call', async () => { + const controller = new AbortController(); + + mockTools(ToolSource.SERVER, 'slow'); + vi.spyOn(ToolsService, 'executeTool').mockImplementation(async () => { + controller.abort(); + + return { content: 'late', isError: false }; + }); + + const requests = mockTurns((options) => options.onToolCallChunk?.(toolCalls('slow'))); + const { callbacks, named, names } = recorder(); + + expect(await runFlow(callbacks, controller.signal)).toEqual({ handled: true }); + expect(named('createToolResultMessage')).toEqual([]); + expect(names().slice(-1)).toEqual(['onFlowComplete']); + expect(requests).toHaveLength(1); + }); +}); diff --git a/ui/apps/chat/tests/unit/async-queue.test.ts b/ui/apps/chat/tests/unit/async-queue.test.ts new file mode 100644 index 000000000000..4856f74d3ba6 --- /dev/null +++ b/ui/apps/chat/tests/unit/async-queue.test.ts @@ -0,0 +1,42 @@ +import { createAsyncQueue } from '#lib/utils/async-queue.ts'; +import { describe, expect, it } from 'vitest'; + +async function collect(queue: AsyncIterable): Promise { + const items: T[] = []; + + for await (const item of queue) items.push(item); + + return items; +} + +describe('createAsyncQueue', () => { + it('yields the items pushed before and during the iteration, then ends on close', async () => { + const queue = createAsyncQueue(); + + queue.push(1); + + const collected = collect(queue); + + await Promise.resolve(); + queue.push(2); + queue.push(3); + queue.close(); + + expect(await collected).toEqual([1, 2, 3]); + }); + + it('throws the failure after the pushed items are read', async () => { + const queue = createAsyncQueue(); + const seen: number[] = []; + + queue.push(1); + queue.fail(new Error('cut')); + + await expect( + (async () => { + for await (const item of queue) seen.push(item); + })() + ).rejects.toThrow('cut'); + expect(seen).toEqual([1]); + }); +}); diff --git a/ui/apps/chat/tsconfig.json b/ui/apps/chat/tsconfig.json index 1cee89a7912e..aab815e5fbc4 100644 --- a/ui/apps/chat/tsconfig.json +++ b/ui/apps/chat/tsconfig.json @@ -45,7 +45,8 @@ "tw-animate-css/*": ["./node_modules/tw-animate-css/*"], "vaul-svelte": ["./node_modules/vaul-svelte"], "vaul-svelte/*": ["./node_modules/vaul-svelte/*"], - "@lucide/svelte/icons/*": ["./node_modules/@lucide/svelte/dist/icons/*"] + "@lucide/svelte/icons/*": ["./node_modules/@lucide/svelte/dist/icons/*"], + "@ggml-org/llama-harness": ["../../lib/harness/src/index.ts"] }, "sourceMap": true, "strict": true diff --git a/ui/apps/chat/vite.config.ts b/ui/apps/chat/vite.config.ts index 93a4956fa1e1..c393f7661e90 100644 --- a/ui/apps/chat/vite.config.ts +++ b/ui/apps/chat/vite.config.ts @@ -16,6 +16,8 @@ import { fileURLToPath } from 'url'; import { defineConfig, loadEnv, type ProxyOptions, searchForWorkspaceRoot } from 'vite'; const __dirname = dirname(fileURLToPath(import.meta.url)); +// the harness has no build step in the repo: the app compiles it from source +const LLAMA_HARNESS_ENTRY = resolve(__dirname, '../../lib/harness/src/index.ts'); // eslint-disable-next-line @typescript-eslint/no-explicit-any const browserBaseConfig: any = { enabled: true, @@ -89,13 +91,18 @@ export default defineConfig(({ mode }) => { resolve: { alias: { + '@ggml-org/llama-harness': LLAMA_HARNESS_ENTRY, 'katex-fonts': resolve('node_modules/katex/dist/fonts') } }, server: { fs: { - allow: [searchForWorkspaceRoot(process.cwd()), resolve(__dirname, 'tests')] + allow: [ + searchForWorkspaceRoot(process.cwd()), + resolve(__dirname, 'tests'), + resolve(__dirname, '../../lib/harness') + ] }, headers: { 'Cross-Origin-Embedder-Policy': 'require-corp', diff --git a/ui/lib/harness/.gitignore b/ui/lib/harness/.gitignore new file mode 100644 index 000000000000..16339ab2e87f --- /dev/null +++ b/ui/lib/harness/.gitignore @@ -0,0 +1,6 @@ +node_modules +/dist + +# OS +.DS_Store +Thumbs.db diff --git a/ui/lib/harness/.npmrc b/ui/lib/harness/.npmrc new file mode 100644 index 000000000000..b6f27f135954 --- /dev/null +++ b/ui/lib/harness/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/ui/lib/harness/LICENSE b/ui/lib/harness/LICENSE new file mode 100644 index 000000000000..50c59c51c158 --- /dev/null +++ b/ui/lib/harness/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) ggml + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ui/lib/harness/README.md b/ui/lib/harness/README.md new file mode 100644 index 000000000000..9994a0caa058 --- /dev/null +++ b/ui/lib/harness/README.md @@ -0,0 +1,64 @@ +# @ggml-org/llama-harness + +Llama Harness, the agentic loop of Llama Chat, as a package: it streams a turn, runs the tools the model called, gives it the results, and repeats until the model answers without a tool call. It is headless TypeScript with no UI, no DOM and no dependencies, so the same loop runs in a browser, in Node and in a worker. It lives in llama.cpp at `ui/lib/harness`. + +It needs an OpenAI-compatible chat completions endpoint, not llama-server specifically. What llama-server adds (timings, prompt progress) it reads when it is there. + +## Use + +```ts +import { createHarness, createOpenAIClient, type Tool } from "@ggml-org/llama-harness"; + +const harness = createHarness(createOpenAIClient({ baseUrl: "http://localhost:8080" })); + +const getTime: Tool = { + definition: { + type: "function", + function: { + name: "get_time", + description: "The current time", + parameters: { type: "object", properties: {} }, + }, + }, + execute: async () => ({ content: new Date().toISOString() }), +}; + +for await (const event of harness.run([{ role: "user", content: "What time is it?" }], [getTime])) { + if (event.type === "text") process.stdout.write(event.delta); +} +``` + +## The contract + +```ts +interface Harness { + run(messages, tools, signal?, options?): AsyncIterable; +} +``` + +- **Messages and tools** use the OpenAI shapes. A tool is its definition plus an `execute` function. +- **Events:** `turn_start`, `text`, `reasoning`, `tool_call_delta`, `timings`, `model`, `completion_id`, `message` (a turn's assistant message), `tool_call`, `tool_update` (a streaming tool's output so far), `tool_result`, `turn_end`, and last `done` or `error`. `run` never throws. +- **The loop waits for you.** It does not go on until you ask for the next event, so you can store each message before the next step runs. +- **It keeps nothing.** No conversation state, no storage: the app owns the history and passes it in. +- **Stopping:** `signal` stops at once. `options.interrupted` ends the run at the next safe point, e.g. when the user sent a new message. `options.maxTurns` with `options.onTurnLimit` asks before it goes past a turn limit. + +An app can put another loop behind the same `Harness` interface. What sits above the loop is the same for every loop: the tools, and whatever an app wraps around `execute`, such as a permission prompt. + +## What the app keeps + +Llama Chat (`ui/apps/chat`) is the first app on it. Its agentic store gives the harness: + +- **An inference client:** its own chat service, which knows the servers, the settings and how to resume a stream. +- **The tools:** built-in tools, llama-server tools and MCP tools, each behind the permission gate. +- **Storage:** it turns the events into messages in its database. + +## Develop + +```sh +npm ci +npm run check # type check +npm test # unit tests +npm run prepack # build dist/ and lint the package +``` + +Llama Chat does not need a build of this package: it compiles `src/` directly, through the `@ggml-org/llama-harness` alias in its `svelte.config.js`. diff --git a/ui/lib/harness/package-lock.json b/ui/lib/harness/package-lock.json new file mode 100644 index 000000000000..952987aff8df --- /dev/null +++ b/ui/lib/harness/package-lock.json @@ -0,0 +1,1255 @@ +{ + "name": "@ggml-org/llama-harness", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@ggml-org/llama-harness", + "version": "0.1.0", + "license": "MIT", + "devDependencies": { + "@types/node": "^24", + "publint": "^0.3.21", + "typescript": "^6.0.3", + "vitest": "5.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.156.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.156.0.tgz", + "integrity": "sha512-1Duyg48TdJNdyhEB8Xn1L36bT9Pv6GUtlKjHxOys8xkaF/rQq3XJE+bAurSK/AgIz5mGFsb77cn9Pb20+WCScw==", + "dev": true, + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@publint/pack": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/@publint/pack/-/pack-0.1.7.tgz", + "integrity": "sha512-4EDEmvxWtgsCnnVeBvtFIFZtUhPPt1+bA9JrSwU4Sa//6oKtzCSlGGXYJr44OD9aGISymbieJ4mCKHUygUDU+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyexec": "^1.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://bjornlu.com/sponsor" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.18.tgz", + "integrity": "sha512-ubTG+4KlidvioxjOFdsdEa2F6JVGZb8m2ByJUqQ7fYzUeT1cEn194POcsQ10PKH4p//q0YQn4SB3YpSuzcA1Yg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.18.tgz", + "integrity": "sha512-2Xa1t/JeFGTZn5TbzgPppcCftF8sCVnTNvrIvoSdcZhoFzAuVy1L3xxQUQ4sfWDtwVzg1u6Yi1R3SlJugRNV2Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.18.tgz", + "integrity": "sha512-iUzw9HshVvrWci9gVXjxWS7/M/IcLvNo67vu2c/w9zKBz4hIvPRixdUpIW/iCfhldLaq9ER2hecsI9UY5z0/yA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.18.tgz", + "integrity": "sha512-XuDHFzbFEtZl0QGJi92+fmG67tfnp1iZ9VOeDkwasGyTq2VG3/5PnJnvAr+MTzFh/rNScJ3VhN+XpZyyGsN7Jw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.18.tgz", + "integrity": "sha512-MM8opLfsLK15oEXDz6iBc0t3ahu+ew9jsAmha+ovZ33hSwYgqo7YmpRITqFTfW7xA71BLEQ3jJdTsaCmhB1c9w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.18.tgz", + "integrity": "sha512-J6EoCWiy5RnFQzHrKyHXXevZ9lukQfGpnchX/zG/k/WC2nc9O9XyS1JgA8kI9VxMFggMbCPjpv7LcikriXfehw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.18.tgz", + "integrity": "sha512-HIkFuLF7CdDeki+1aCZLLbLq0hk/PHntb+z4/YCTo47JbksTD6jJEELIz92tYROmDmvMrNqUMqBi71BkJLDR+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.18.tgz", + "integrity": "sha512-4oqWItLLYt7IceOnElCct8T2ek5p7PmD7LN04TRJRXGo5f3HtKTgGH6PhYRFK4C2ICkq/LMUU0BVPvvlAKhm8g==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.18.tgz", + "integrity": "sha512-+AkRly5SrCEgnPhGRfub7pucASYEC2fTsxZg7VCcnozpA2+Wn/gCYy1t7LuJS67/xyJu/7noYBJncDUkbwgw2A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.18.tgz", + "integrity": "sha512-Ttg1r9vVykMnQrE/GwYO9icKtXiuiMfRe/w74xJkeoVuvhRG+WtT6LEMOf58Yl63yd77lhHzO8Y55R8CC3XALA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.18.tgz", + "integrity": "sha512-xX1Tt4o5jUMma0vJsXp5eRZA9nOuyigb2dKJMMzZRmzMxLYGE9PaKKSqHPzPvSQgM8ZOEacSTQwDQcwWfh0xmQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.18.tgz", + "integrity": "sha512-4VVlGI5K2kociZlW3lSN8VjG3TW5yQIlAcfCdBXMgGjJxZll15Fa4ocRHXHvIVVnwlibCw/NQajJrNDiqqpiiw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.18.tgz", + "integrity": "sha512-PApTsgH1jLYuyy3tSCz9De82Gq5qXwi7vImVZhEzTgX1C4150Cp5M2ZfL3pLvbCvbEHL0oJeC6pd6podtv8Y5w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.18.tgz", + "integrity": "sha512-J2XjWjx/SP+4x0H4r6dz3Vzb5WVkg7re40Do9K7HJ3sK2/xcmZrWwakXlSwaMvOkvruLJCD4CL/3wqsIR/thgg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.18.tgz", + "integrity": "sha512-XwRJxRQvkuEtS0hIIt29QrqPJ+JzMJSNYHJw4ae/mc6uEarfRG1ceyg4HmmWiBdL/YCb6g7N5BLbSvmRJRYZHw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.1.0.tgz", + "integrity": "sha512-K+NU2HDCEem4sClgHH4ZZXwdJeNFkJo+hRk479DIwdIviAPN2hFJH1KHHwg8CxHQXvi0M5XY4deTXINfIiSG6w==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@vitest/mocker": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.5.tgz", + "integrity": "sha512-xLL3lkDegJZw+MGoGaCCsFayDZthGgpAtr3Vziyfc4p7xH89K7KRPcRBBh/kEYm0Aqkt5DtTmkYpBtRV/FrHZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", + "@vitest/spy": "5.0.5", + "estree-walker": "^3.0.3", + "magic-string": "^1.4.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9 || ^3.0.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/spy": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.5.tgz", + "integrity": "sha512-lwJRExuH417257Q5wv9QYugN9IMJkoxcrKrYxGPQkBzl7wlNpTUxD/5yNX4ESKVUIShKPgkGchYPFMWOOk0wzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz", + "integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "peer": true, + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.5.tgz", + "integrity": "sha512-oLLC2+YvQUfAa+vxoufE/yc1icb87M9iXIRqAoR/2EhE/8cb3SF/+Gi0c2By1B+5Vz2e2vQtUvHiu1+63Znh+w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/obug/-/obug-3.0.0.tgz", + "integrity": "sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/package-manager-detector": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.9.0.tgz", + "integrity": "sha512-zAMfbta9nPA8AKkA4WdmNAT8R8s6Dk0/xwPsqNnRidMFcLo1YkLuPDTgs2f64XdzS8vbLNnoJ+uUIkgbz7oRHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.32.tgz", + "integrity": "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/publint": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/publint/-/publint-0.3.25.tgz", + "integrity": "sha512-WMMbn/d7JqRlfXQL/JUI4n5skC8cJ5WVr8/wT5mmiYWkUdENkZr1I5rgs2+dQPVRgoJd+JRYtB46MTsuDylw9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@publint/pack": "^0.1.7", + "package-manager-detector": "^1.8.0", + "picocolors": "^1.1.1", + "sade": "^1.8.1" + }, + "bin": { + "publint": "src/cli.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://bjornlu.com/sponsor" + } + }, + "node_modules/rolldown": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.18.tgz", + "integrity": "sha512-C1uuT3UR60SShuCaPQFAAZyefJCWZlAKB02BV3BuPE2Rs4fevRQltEKoFO2tV5W9/eJhU5RieV6vJj3euulfNQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@oxc-project/types": "=0.156.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.18", + "@rolldown/binding-android-arm64": "1.2.18", + "@rolldown/binding-darwin-arm64": "1.2.18", + "@rolldown/binding-darwin-x64": "1.2.18", + "@rolldown/binding-freebsd-x64": "1.2.18", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.18", + "@rolldown/binding-linux-arm64-gnu": "1.2.18", + "@rolldown/binding-linux-arm64-musl": "1.2.18", + "@rolldown/binding-linux-ppc64-gnu": "1.2.18", + "@rolldown/binding-linux-s390x-gnu": "1.2.18", + "@rolldown/binding-linux-x64-gnu": "1.2.18", + "@rolldown/binding-linux-x64-musl": "1.2.18", + "@rolldown/binding-openharmony-arm64": "1.2.18", + "@rolldown/binding-win32-arm64-msvc": "1.2.18", + "@rolldown/binding-win32-x64-msvc": "1.2.18" + } + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/std-env": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.3.0.tgz", + "integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.3.tgz", + "integrity": "sha512-Al01C2JIBnO2CXCAPbpAl6sUiOZHeThGtHYNTa51JRw9PyWOCdr+9Ong05R4qg/Sk3+H645E8ov1h2c58ityNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "node_modules/vite": { + "version": "8.3.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.7.tgz", + "integrity": "sha512-oxh0dMxrayBr33ScJvvBJ2HBQ2jc1TBrXKR2NyL/MJ9WPuX19mN2DXWUmywKdyw5RFVQBo8hmtb2FMC0LHrnng==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.32", + "rolldown": "~1.2.18", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1 || ^0.8.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.5.tgz", + "integrity": "sha512-cPODOMF8YnkiiYtDdHo6n4wWmBrOBBbOIYH2d33M4l3CdNn72h+qMmO6uXffUN6A5AZFXNND/d+krS9mgkXRfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/mocker": "5.0.5", + "chai": "^6.3.0", + "es-module-lexer": "^2.3.2", + "expect-type": "^1.4.0", + "magic-string": "^1.4.3", + "obug": "^3.0.0", + "picomatch": "^4.0.7", + "std-env": "^4.3.0", + "tinybench": "^6.2.2", + "tinyexec": "^1.3.1", + "tinyglobby": "^0.2.17", + "why-is-node-running": "3.2.1" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^22.12.0 || ^24.0.0 || >=26.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "5.0.5", + "@vitest/browser-preview": "5.0.5", + "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", + "@vitest/coverage-istanbul": "5.0.5", + "@vitest/coverage-v8": "5.0.5", + "@vitest/ui": "5.0.5", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-3.2.1.tgz", + "integrity": "sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==", + "dev": true, + "license": "MIT", + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=20.11" + } + } + } +} diff --git a/ui/lib/harness/package.json b/ui/lib/harness/package.json new file mode 100644 index 000000000000..b93deffb207d --- /dev/null +++ b/ui/lib/harness/package.json @@ -0,0 +1,39 @@ +{ + "name": "@ggml-org/llama-harness", + "version": "0.1.0", + "description": "Headless agentic loop for llama.cpp frontend apps.", + "homepage": "https://github.com/ggml-org/llama.cpp/tree/master/ui/lib/harness", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/ggml-org/llama.cpp.git", + "directory": "ui/lib/harness" + }, + "files": [ + "dist" + ], + "type": "module", + "sideEffects": false, + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "prepack": "npm run build && publint", + "test": "vitest --run" + }, + "devDependencies": { + "@types/node": "^24", + "publint": "^0.3.21", + "typescript": "^6.0.3", + "vitest": "5.0.5" + }, + "engines": { + "node": ">=20" + } +} diff --git a/ui/lib/harness/src/abort.ts b/ui/lib/harness/src/abort.ts new file mode 100644 index 000000000000..636dd21ed718 --- /dev/null +++ b/ui/lib/harness/src/abort.ts @@ -0,0 +1,17 @@ +const ABORT_ERROR_NAME = "AbortError"; +// TypeErrors a browser throws when a fetch stream is cut by page unload, navigation or a network drop +const ABORT_LIKE_MESSAGE_PATTERNS = [ + /input stream/i, + /network connection was lost/i, + /load failed/i, +]; + +export function isAbortError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + if (error.name === ABORT_ERROR_NAME) return true; + + return ( + error instanceof TypeError && ABORT_LIKE_MESSAGE_PATTERNS.some((re) => re.test(error.message)) + ); +} diff --git a/ui/lib/harness/src/harness.ts b/ui/lib/harness/src/harness.ts new file mode 100644 index 000000000000..e58e7ea25188 --- /dev/null +++ b/ui/lib/harness/src/harness.ts @@ -0,0 +1,375 @@ +import { isAbortError } from "./abort.js"; +import type { + DoneReason, + Harness, + HarnessEvent, + InferenceClient, + LlmTimings, + LlmTotals, + Message, + PartialToolCall, + RunOptions, + RunTimings, + Timings, + Tool, + ToolCall, + ToolResult, + TurnStats, +} from "./types.js"; + +/** The result of a tool call that did not run because the run was interrupted. */ +export const INTERRUPTED_TOOL_RESULT = "Tool execution was interrupted by a new user message."; + +function emptyTotals(): LlmTotals { + return { predicted_ms: 0, predicted_n: 0, prompt_ms: 0, prompt_n: 0 }; +} + +function addTimings(totals: LlmTotals, timings: LlmTimings): LlmTotals { + return { + predicted_ms: totals.predicted_ms + (timings.predicted_ms ?? 0), + predicted_n: totals.predicted_n + (timings.predicted_n ?? 0), + prompt_ms: totals.prompt_ms + (timings.prompt_ms ?? 0), + prompt_n: totals.prompt_n + (timings.prompt_n ?? 0), + }; +} + +function toToolCalls(partial: PartialToolCall[]): ToolCall[] { + return partial.map((call, index) => ({ + function: { + arguments: call?.function?.arguments ?? "", + name: call?.function?.name ?? "", + }, + id: call?.id ?? `tool_${index}`, + type: "function", + })); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * The default loop: stream a turn, run the tools the model called, give it the results, and repeat until it answers + * without a tool call. It keeps no state between runs and stores nothing: the app owns the conversation. + */ +export function createHarness(client: InferenceClient): Harness { + return { + run: (messages, tools, signal, options) => run(client, messages, tools, signal, options), + }; +} + +async function* run( + client: InferenceClient, + messages: Message[], + tools: Tool[], + signal?: AbortSignal, + options: RunOptions = {}, +): AsyncGenerator { + const { interrupted = () => false, maxTurns = Infinity, onTurnLimit } = options; + const history = [...messages]; + const definitions = tools.map((tool) => tool.definition); + const toolsByName = new Map(tools.map((tool) => [tool.definition.function.name, tool])); + const totals: RunTimings = { + llm: emptyTotals(), + perTurn: [], + toolCalls: [], + toolCallsCount: 0, + toolsMs: 0, + turns: 0, + }; + + let lastTimings: LlmTimings | undefined; + let turn = 0; + + const runTimings = (): Timings | undefined => { + if (totals.toolCallsCount === 0) return lastTimings; + + return { + agentic: totals, + cache_n: lastTimings?.cache_n, + predicted_ms: lastTimings?.predicted_ms, + predicted_n: lastTimings?.predicted_n, + prompt_ms: lastTimings?.prompt_ms, + prompt_n: lastTimings?.prompt_n, + }; + }; + const done = (reason: DoneReason): HarnessEvent => ({ + reason, + timings: runTimings(), + type: "done", + }); + + while (true) { + if (turn >= maxTurns) { + const more = onTurnLimit ? await onTurnLimit(signal) : false; + + if (!more || signal?.aborted) { + yield done(signal?.aborted ? "aborted" : "turn_limit"); + + return; + } + + turn = 0; + } + + totals.turns = turn + 1; + + if (signal?.aborted) { + yield done("aborted"); + + return; + } + + yield { turn: turn + 1, type: "turn_start" }; + + let content = ""; + let reasoning = ""; + let partialCalls: PartialToolCall[] = []; + let turnTimings: LlmTimings | undefined; + + const stats: TurnStats = { llm: emptyTotals(), toolCalls: [], toolsMs: 0, turn: turn + 1 }; + // the turn's message when the run ends after it: what streamed so far, with the run's timings + const lastMessage = (toolCalls?: ToolCall[]): HarnessEvent => ({ + content, + final: true, + reasoning: reasoning || undefined, + timings: runTimings(), + toolCalls, + type: "message", + }); + + try { + const stream = client.stream({ + messages: history, + signal, + tools: definitions.length > 0 ? definitions : undefined, + }); + + for await (const event of stream) { + if (event.type === "text") { + content += event.delta; + yield event; + } else if (event.type === "reasoning") { + reasoning += event.delta; + yield event; + } else if (event.type === "tool_calls") { + partialCalls = event.toolCalls; + yield { toolCalls: partialCalls, type: "tool_call_delta" }; + } else if (event.type === "timings") { + if (event.timings) { + lastTimings = event.timings; + turnTimings = event.timings; + } + + yield { ...event, llm: event.timings && addTimings(totals.llm, event.timings) }; + } else { + yield event; + } + } + + if (turnTimings) { + totals.llm = addTimings(totals.llm, turnTimings); + stats.llm = addTimings(stats.llm, turnTimings); + } + } catch (error) { + // keep the partial output in both cases + yield lastMessage(); + + if (signal?.aborted) { + yield done("aborted"); + } else { + yield { + error: error instanceof Error ? error : new Error("LLM stream error"), + timings: runTimings(), + type: "error", + }; + } + + return; + } + + // a client can swallow the abort and end the stream normally: do not take a cut tool call for a complete one + if (signal?.aborted) { + yield lastMessage(); + yield done("aborted"); + + return; + } + + if (interrupted()) { + yield lastMessage(partialCalls.length > 0 ? toToolCalls(partialCalls) : undefined); + yield done("interrupted"); + + return; + } + + if (partialCalls.length === 0) { + totals.perTurn.push(stats); + + const timings = runTimings(); + + yield lastMessage(); + + if (timings) yield { timings, type: "turn_end" }; + + yield done("complete"); + + return; + } + + const toolCalls = toToolCalls(partialCalls); + + yield { + content, + final: false, + reasoning: reasoning || undefined, + timings: turnTimings, + toolCalls, + type: "message", + }; + + history.push({ + content: content || undefined, + reasoning_content: reasoning || undefined, + role: "assistant", + tool_calls: toolCalls, + }); + + for (let i = 0; i < toolCalls.length; i++) { + const toolCall = toolCalls[i]; + + if (signal?.aborted) { + yield done("aborted"); + + return; + } + + if (interrupted()) { + for (const skippedCall of toolCalls.slice(i)) { + yield { + result: { content: INTERRUPTED_TOOL_RESULT }, + skipped: true, + toolCall: skippedCall, + type: "tool_result", + }; + + history.push({ + content: INTERRUPTED_TOOL_RESULT, + role: "tool", + tool_call_id: skippedCall.id, + }); + } + + break; + } + + yield { toolCall, type: "tool_call" }; + + const tool = toolsByName.get(toolCall.function.name); + const startTime = performance.now(); + + let result: ToolResult; + + if (!tool) { + result = { content: `Error: Unknown tool: ${toolCall.function.name}`, isError: true }; + } else { + // only the latest update matters, so a slow consumer skips the ones in between + let pendingUpdate: string | undefined; + let settled = false; + let wake: (() => void) | undefined; + + const notify = () => { + wake?.(); + wake = undefined; + }; + const execution = Promise.resolve() + .then(() => + tool.execute(toolCall, { + signal, + update: (updated) => { + pendingUpdate = updated; + notify(); + }, + }), + ) + .finally(() => { + settled = true; + notify(); + }); + + // the rejection is handled below, after the updates are drained + execution.catch(() => {}); + + while (true) { + if (pendingUpdate !== undefined) { + const updated = pendingUpdate; + + pendingUpdate = undefined; + yield { content: updated, toolCall, type: "tool_update" }; + } else if (settled) { + break; + } else { + await new Promise((resolve) => (wake = resolve)); + } + } + + try { + result = await execution; + } catch (error) { + if (isAbortError(error)) { + yield done("aborted"); + + return; + } + + result = { content: `Error: ${errorMessage(error)}`, isError: true }; + } + } + + const durationMs = Math.round(result.durationMs ?? performance.now() - startTime); + const timing = { + duration_ms: durationMs, + name: toolCall.function.name, + success: !result.isError, + }; + + totals.toolCalls.push(timing); + totals.toolCallsCount++; + totals.toolsMs += durationMs; + stats.toolCalls.push(timing); + stats.toolsMs += durationMs; + + if (signal?.aborted) { + yield done("aborted"); + + return; + } + + yield { result, skipped: false, toolCall, type: "tool_result" }; + + history.push({ + content: result.parts?.length + ? [{ text: result.content, type: "text" }, ...result.parts] + : result.content, + role: "tool", + tool_call_id: toolCall.id, + }); + } + + if (stats.toolCalls.length > 0) { + totals.perTurn.push(stats); + + const timings = runTimings(); + + if (timings) yield { timings, type: "turn_end" }; + } + + if (interrupted()) { + yield done("interrupted"); + + return; + } + + turn++; + } +} diff --git a/ui/lib/harness/src/index.ts b/ui/lib/harness/src/index.ts new file mode 100644 index 000000000000..e3096ef332bb --- /dev/null +++ b/ui/lib/harness/src/index.ts @@ -0,0 +1,4 @@ +export { isAbortError } from "./abort.js"; +export { createHarness, INTERRUPTED_TOOL_RESULT } from "./harness.js"; +export { createOpenAIClient, type OpenAIClientOptions } from "./openai-client.js"; +export type * from "./types.js"; diff --git a/ui/lib/harness/src/openai-client.ts b/ui/lib/harness/src/openai-client.ts new file mode 100644 index 000000000000..48413af26cfd --- /dev/null +++ b/ui/lib/harness/src/openai-client.ts @@ -0,0 +1,143 @@ +import type { InferenceClient, InferenceEvent, PartialToolCall } from "./types.js"; + +const CHAT_COMPLETIONS_PATH = "/v1/chat/completions"; +const SSE_DATA_PREFIX = "data:"; +const SSE_DONE = "[DONE]"; + +export interface OpenAIClientOptions { + /** Server root, e.g. `http://localhost:8080`. */ + baseUrl: string; + apiKey?: string; + model?: string; + /** Extra fields of the request body, e.g. `temperature`. */ + params?: Record; + headers?: Record; + /** Default: the global fetch. */ + fetch?: typeof fetch; +} + +interface ToolCallDelta extends PartialToolCall { + index?: number; +} + +function mergeToolCallDeltas(toolCalls: PartialToolCall[], deltas: ToolCallDelta[]): void { + for (const delta of deltas) { + // without an index, a delta with an id opens a call and any other one continues the last call + const index = + delta.index ?? (delta.id || toolCalls.length === 0 ? toolCalls.length : toolCalls.length - 1); + const call = (toolCalls[index] ??= { function: { arguments: "", name: "" } }); + + if (delta.id) call.id = delta.id; + + if (delta.type) call.type = delta.type; + + call.function = { + arguments: (call.function?.arguments ?? "") + (delta.function?.arguments ?? ""), + name: (call.function?.name ?? "") + (delta.function?.name ?? ""), + }; + } +} + +async function errorFromResponse(response: Response): Promise { + const body = await response.text().catch(() => ""); + + let message = body; + + try { + message = JSON.parse(body).error?.message ?? body; + } catch { + // not JSON: keep the body text + } + + return new Error(`HTTP ${response.status}${message ? `: ${message}` : ""}`); +} + +/** + * An inference client for any OpenAI-compatible endpoint, llama-server included. It reads the `timings` and + * `prompt_progress` fields that llama-server adds and works without them. + */ +export function createOpenAIClient(options: OpenAIClientOptions): InferenceClient { + const { apiKey, baseUrl, headers, model, params } = options; + const url = baseUrl.replace(/\/+$/, "") + CHAT_COMPLETIONS_PATH; + + return { + async *stream({ messages, signal, tools }): AsyncGenerator { + const response = await (options.fetch ?? fetch)(url, { + body: JSON.stringify({ ...params, messages, model, stream: true, tools }), + headers: { + "Content-Type": "application/json", + ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + ...headers, + }, + method: "POST", + signal, + }); + + if (!response.ok) throw await errorFromResponse(response); + + if (!response.body) throw new Error("The response has no body"); + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const toolCalls: PartialToolCall[] = []; + + let buffer = ""; + let completionId: string | undefined; + let modelName: string | undefined; + + try { + while (true) { + const { done, value } = await reader.read(); + + if (done) return; + + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split("\n"); + + buffer = lines.pop() ?? ""; + + for (const line of lines) { + if (!line.startsWith(SSE_DATA_PREFIX)) continue; + + const data = line.slice(SSE_DATA_PREFIX.length).trim(); + + if (data === SSE_DONE) return; + + const chunk = JSON.parse(data); + + if (chunk.error) throw new Error(chunk.error.message ?? "Stream error"); + + if (chunk.id && chunk.id !== completionId) { + completionId = chunk.id; + yield { id: chunk.id, type: "completion_id" }; + } + + if (chunk.model && chunk.model !== modelName) { + modelName = chunk.model; + yield { model: chunk.model, type: "model" }; + } + + const delta = chunk.choices?.[0]?.delta; + const reasoning = delta?.reasoning_content ?? delta?.reasoning; + + if (reasoning) yield { delta: reasoning, type: "reasoning" }; + + if (delta?.content) yield { delta: delta.content, type: "text" }; + + if (delta?.tool_calls?.length) { + mergeToolCallDeltas(toolCalls, delta.tool_calls); + yield { toolCalls: toolCalls.map((call) => ({ ...call })), type: "tool_calls" }; + } + + if (chunk.timings || chunk.prompt_progress) { + yield { progress: chunk.prompt_progress, timings: chunk.timings, type: "timings" }; + } + } + } + } finally { + reader.cancel().catch(() => {}); + } + }, + }; +} diff --git a/ui/lib/harness/src/types.ts b/ui/lib/harness/src/types.ts new file mode 100644 index 000000000000..7050ce0ba329 --- /dev/null +++ b/ui/lib/harness/src/types.ts @@ -0,0 +1,209 @@ +// Messages and tools use the OpenAI chat-completions shapes, so they go to the endpoint as they are. + +export interface TextPart { + type: "text"; + text: string; +} + +export interface ImagePart { + type: "image_url"; + image_url: { url: string }; +} + +export interface AudioPart { + type: "input_audio"; + input_audio: { data: string; format: string }; +} + +export type ContentPart = TextPart | ImagePart | AudioPart; + +export interface ToolCall { + id: string; + type: "function"; + function: { name: string; arguments: string }; +} + +/** A tool call as it streams in: any field can still be missing. */ +export interface PartialToolCall { + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; +} + +export type Message = + | { role: "system" | "user"; content: string | ContentPart[] } + | { + role: "assistant"; + content?: string | ContentPart[]; + reasoning_content?: string; + tool_calls?: ToolCall[]; + } + | { role: "tool"; tool_call_id: string; content: string | ContentPart[] }; + +export interface ToolDefinition { + type: "function"; + function: { + name: string; + description?: string; + parameters?: Record; + }; +} + +export interface ToolContext { + signal?: AbortSignal; + /** Report the output so far of a tool that streams. Each call replaces the previous content. */ + update(content: string): void; +} + +export interface ToolResult { + /** Text the model reads. */ + content: string; + /** Media the model reads after the text, e.g. an image the tool produced. */ + parts?: ContentPart[]; + isError?: boolean; + /** Set it when the tool times itself, e.g. to leave out a wait for approval. Default: the time execute took. */ + durationMs?: number; + /** App data the loop does not read. It comes back on the tool_result event. */ + details?: unknown; +} + +export interface Tool { + definition: ToolDefinition; + /** Run the call. Reject with an AbortError to end the run, with any other error to report it to the model. */ + execute(call: ToolCall, context: ToolContext): Promise; +} + +/** Token counts and times of one request, as llama-server reports them. */ +export interface LlmTimings { + cache_n?: number; + predicted_ms?: number; + predicted_n?: number; + prompt_ms?: number; + prompt_n?: number; +} + +export interface LlmTotals { + predicted_ms: number; + predicted_n: number; + prompt_ms: number; + prompt_n: number; +} + +export interface ToolCallTiming { + name: string; + duration_ms: number; + success: boolean; +} + +export interface TurnStats { + turn: number; + llm: LlmTotals; + toolCalls: ToolCallTiming[]; + toolsMs: number; +} + +/** Totals of a run: every turn's requests and tool calls. */ +export interface RunTimings { + turns: number; + toolCallsCount: number; + toolsMs: number; + toolCalls: ToolCallTiming[]; + perTurn: TurnStats[]; + llm: LlmTotals; +} + +/** The last request's timings, with the run's totals once a tool has run. */ +export interface Timings extends LlmTimings { + agentic?: RunTimings; +} + +export interface PromptProgress { + cache: number; + processed: number; + time_ms: number; + total: number; +} + +export interface InferenceRequest { + messages: Message[]; + tools?: ToolDefinition[]; + signal?: AbortSignal; +} + +export type InferenceEvent = + | { type: "text"; delta: string } + | { type: "reasoning"; delta: string } + /** Every tool call of the response so far, not only the new part. */ + | { type: "tool_calls"; toolCalls: PartialToolCall[] } + | { type: "timings"; timings?: LlmTimings; progress?: PromptProgress } + | { type: "model"; model: string } + | { type: "completion_id"; id: string }; + +/** + * One streamed chat completion. The loop needs nothing else from a server, so any OpenAI-compatible endpoint works. + * Timings, model and completion id are optional extras. + */ +export interface InferenceClient { + stream(request: InferenceRequest): AsyncIterable; +} + +export type DoneReason = + /** The model answered without calling a tool. */ + | "complete" + | "aborted" + /** `interrupted` returned true. */ + | "interrupted" + /** The turn limit was reached and `onTurnLimit` did not grant more turns. */ + | "turn_limit"; + +export type HarnessEvent = + /** `turn` starts at 1, and again at 1 once `onTurnLimit` grants more turns. */ + | { type: "turn_start"; turn: number } + | { type: "text"; delta: string } + | { type: "reasoning"; delta: string } + | { type: "tool_call_delta"; toolCalls: PartialToolCall[] } + /** `llm` is the run's totals including the request in flight. */ + | { type: "timings"; timings?: LlmTimings; progress?: PromptProgress; llm?: LlmTotals } + | { type: "model"; model: string } + | { type: "completion_id"; id: string } + /** The assistant message of a turn. `final` is true when the run ends after it and none of its tool calls runs. */ + | { + type: "message"; + content: string; + reasoning?: string; + toolCalls?: ToolCall[]; + timings?: Timings; + final: boolean; + } + | { type: "tool_call"; toolCall: ToolCall } + | { type: "tool_update"; toolCall: ToolCall; content: string } + /** `skipped` is true when the call did not run because the run was interrupted. */ + | { type: "tool_result"; toolCall: ToolCall; result: ToolResult; skipped: boolean } + | { type: "turn_end"; timings: Timings } + | { type: "done"; reason: DoneReason; timings?: Timings } + | { type: "error"; error: Error; timings?: Timings }; + +export interface RunOptions { + /** Turns before `onTurnLimit` is asked. Default: no limit. */ + maxTurns?: number; + /** Resolve true to run `maxTurns` more turns. Default: the run ends. */ + onTurnLimit?: (signal?: AbortSignal) => Promise; + /** + * Return true to end the run at the next safe point, e.g. when the user sent a new message. Tool calls that did + * not run yet get a result that says so. Use `signal` to stop at once. + */ + interrupted?: () => boolean; +} + +/** + * The contract an app runs a loop through. `run` never throws: it ends with a `done` or an `error` event. It waits + * for the consumer between events, so the consumer can store each one before the loop goes on. + */ +export interface Harness { + run( + messages: Message[], + tools: Tool[], + signal?: AbortSignal, + options?: RunOptions, + ): AsyncIterable; +} diff --git a/ui/lib/harness/tests/harness.test.ts b/ui/lib/harness/tests/harness.test.ts new file mode 100644 index 000000000000..4e302a14c5fe --- /dev/null +++ b/ui/lib/harness/tests/harness.test.ts @@ -0,0 +1,316 @@ +import { describe, expect, it } from "vitest"; +import { + createHarness, + INTERRUPTED_TOOL_RESULT, + type HarnessEvent, + type InferenceClient, + type InferenceEvent, + type InferenceRequest, + type Message, + type RunOptions, + type Tool, + type ToolResult, +} from "../src/index.js"; + +const USER: Message[] = [{ content: "hi", role: "user" }]; + +function callEvent(...names: string[]): InferenceEvent { + return { + toolCalls: names.map((name, index) => ({ + function: { arguments: "{}", name }, + id: `call_${index}`, + })), + type: "tool_calls", + }; +} + +// a client that plays one scripted turn per request and records what it was sent +function scriptedClient(turns: Array) { + const requests: InferenceRequest[] = []; + const client: InferenceClient = { + async *stream(request) { + requests.push({ ...request, messages: [...request.messages] }); + + const turn = turns[requests.length - 1] ?? []; + + if (turn instanceof Error) throw turn; + + yield* turn; + }, + }; + + return { client, requests }; +} + +function tool( + name: string, + execute: Tool["execute"] = async () => ({ content: `${name} ran` }), +): Tool { + return { definition: { function: { name }, type: "function" }, execute }; +} + +async function collect( + client: InferenceClient, + tools: Tool[] = [], + options?: RunOptions, + signal?: AbortSignal, +): Promise { + const events: HarnessEvent[] = []; + + for await (const event of createHarness(client).run(USER, tools, signal, options)) + events.push(event); + + return events; +} + +function types(events: HarnessEvent[]): string[] { + return events.map((event) => event.type); +} + +function last(events: HarnessEvent[]): HarnessEvent { + return events[events.length - 1]; +} + +function results(events: HarnessEvent[]) { + return events.filter((event) => event.type === "tool_result"); +} + +describe("createHarness", () => { + it("ends after a turn without tool calls", async () => { + const { client, requests } = scriptedClient([ + [ + { delta: "think", type: "reasoning" }, + { delta: "Hel", type: "text" }, + { delta: "lo", type: "text" }, + ], + ]); + const events = await collect(client); + + expect(types(events)).toEqual(["turn_start", "reasoning", "text", "text", "message", "done"]); + expect(events[4]).toMatchObject({ content: "Hello", final: true, reasoning: "think" }); + expect(last(events)).toMatchObject({ reason: "complete" }); + expect(requests[0].tools).toBeUndefined(); + }); + + it("runs the called tool and sends its result with the next request", async () => { + const { client, requests } = scriptedClient([ + [{ delta: "let me look", type: "text" }, callEvent("read")], + [{ delta: "done", type: "text" }], + ]); + const events = await collect(client, [tool("read")]); + + expect(types(events)).toEqual([ + "turn_start", + "text", + "tool_call_delta", + "message", + "tool_call", + "tool_result", + "turn_end", + "turn_start", + "text", + "message", + "turn_end", + "done", + ]); + expect(events[3]).toMatchObject({ + final: false, + toolCalls: [{ id: "call_0", type: "function" }], + }); + expect(requests[1].messages.slice(1)).toEqual([ + { + content: "let me look", + reasoning_content: undefined, + role: "assistant", + tool_calls: [ + { function: { arguments: "{}", name: "read" }, id: "call_0", type: "function" }, + ], + }, + { content: "read ran", role: "tool", tool_call_id: "call_0" }, + ]); + expect(requests[0].tools).toEqual([{ function: { name: "read" }, type: "function" }]); + }); + + it("sends a result's media parts after its text", async () => { + const image = { image_url: { url: "data:image/png;base64,AAAA" }, type: "image_url" } as const; + const { client, requests } = scriptedClient([[callEvent("shot")], []]); + + await collect(client, [tool("shot", async () => ({ content: "saved", parts: [image] }))]); + + expect(requests[1].messages[2]).toEqual({ + content: [{ text: "saved", type: "text" }, image], + role: "tool", + tool_call_id: "call_0", + }); + }); + + it("reports an unknown tool and a tool that throws to the model", async () => { + const { client } = scriptedClient([[callEvent("missing", "broken")], []]); + const events = await collect(client, [ + tool("broken", async () => { + throw new Error("boom"); + }), + ]); + + expect(results(events).map((event) => event.result)).toEqual([ + { content: "Error: Unknown tool: missing", isError: true }, + { content: "Error: boom", isError: true }, + ]); + expect(last(events)).toMatchObject({ + timings: { + agentic: { toolCalls: [{ success: false }, { success: false }], toolCallsCount: 2 }, + }, + }); + }); + + it("emits a streaming tool's updates before its result", async () => { + const { client } = scriptedClient([[callEvent("shell")], []]); + const events = await collect(client, [ + tool("shell", async (_call, { update }) => { + update(""); + await Promise.resolve(); + update("a"); + update("ab"); + + return { content: "ab" }; + }), + ]); + const updates = events.filter((event) => event.type === "tool_update"); + + // "a" was replaced before the consumer asked for it + expect(updates.map((event) => event.content)).toEqual(["", "ab"]); + expect(types(events).indexOf("tool_result")).toBeGreaterThan( + types(events).lastIndexOf("tool_update"), + ); + }); + + it("uses the duration a tool reports", async () => { + const { client } = scriptedClient([[callEvent("slow")], []]); + const events = await collect(client, [ + tool("slow", async () => ({ content: "", durationMs: 1234 })), + ]); + + expect(last(events)).toMatchObject({ timings: { agentic: { toolsMs: 1234 } } }); + }); + + it("ends without a result when a tool rejects with an AbortError", async () => { + const { client, requests } = scriptedClient([[callEvent("gated")], []]); + const events = await collect(client, [ + tool("gated", async () => { + throw new DOMException("Operation was aborted", "AbortError"); + }), + ]); + + expect(results(events)).toEqual([]); + expect(last(events)).toMatchObject({ reason: "aborted" }); + expect(requests).toHaveLength(1); + }); + + it("keeps the partial output when the signal aborts mid-stream", async () => { + const controller = new AbortController(); + const client: InferenceClient = { + async *stream() { + yield { delta: "par", type: "text" }; + controller.abort(); + throw new DOMException("Operation was aborted", "AbortError"); + }, + }; + const events = await collect(client, [], undefined, controller.signal); + + expect(types(events)).toEqual(["turn_start", "text", "message", "done"]); + expect(events[2]).toMatchObject({ content: "par", final: true }); + expect(last(events)).toMatchObject({ reason: "aborted" }); + }); + + it("ends with an error event when the stream fails", async () => { + const { client } = scriptedClient([new Error("HTTP 500")]); + const events = await collect(client); + + expect(types(events)).toEqual(["turn_start", "message", "error"]); + expect(last(events)).toMatchObject({ error: { message: "HTTP 500" } }); + }); + + it("keeps the tool calls of the turn it is interrupted after, and runs none", async () => { + const { client, requests } = scriptedClient([[callEvent("read")], []]); + const events = await collect(client, [tool("read")], { interrupted: () => true }); + + expect(types(events)).toEqual(["turn_start", "tool_call_delta", "message", "done"]); + expect(events[2]).toMatchObject({ final: true, toolCalls: [{ id: "call_0" }] }); + expect(last(events)).toMatchObject({ reason: "interrupted" }); + expect(requests).toHaveLength(1); + }); + + it("skips the tool calls left when it is interrupted between two", async () => { + let interrupted = false; + + const { client, requests } = scriptedClient([[callEvent("first", "second", "third")], []]); + const events = await collect( + client, + [ + tool("first", async (): Promise => { + interrupted = true; + + return { content: "first ran" }; + }), + tool("second"), + tool("third"), + ], + { interrupted: () => interrupted }, + ); + + expect(results(events).map((event) => [event.result.content, event.skipped])).toEqual([ + ["first ran", false], + [INTERRUPTED_TOOL_RESULT, true], + [INTERRUPTED_TOOL_RESULT, true], + ]); + expect(last(events)).toMatchObject({ reason: "interrupted" }); + expect(requests).toHaveLength(1); + }); + + it("asks at the turn limit and starts the turn count again when it goes on", async () => { + const answers = [true, false]; + const { client } = scriptedClient([[callEvent("read")], [callEvent("read")], []]); + const events = await collect(client, [tool("read")], { + maxTurns: 1, + onTurnLimit: async () => answers.shift() ?? false, + }); + const turns = events.filter((event) => event.type === "turn_start"); + + expect(turns.map((event) => event.turn)).toEqual([1, 1]); + expect(last(events)).toMatchObject({ reason: "turn_limit" }); + }); + + it("adds up the timings of every turn", async () => { + const timings = (n: number): InferenceEvent => ({ + timings: { cache_n: 1, predicted_ms: n, predicted_n: n, prompt_ms: n, prompt_n: n }, + type: "timings", + }); + const { client } = scriptedClient([[timings(10), callEvent("read")], [timings(5)]]); + const events = await collect(client, [tool("read")]); + const live = events + .filter((event) => event.type === "timings") + .map((event) => event.llm?.predicted_n); + + expect(live).toEqual([10, 15]); + expect(last(events)).toMatchObject({ + timings: { + agentic: { + llm: { predicted_ms: 15, predicted_n: 15, prompt_ms: 15, prompt_n: 15 }, + perTurn: [{ turn: 1 }, { turn: 2 }], + toolCallsCount: 1, + turns: 2, + }, + cache_n: 1, + predicted_n: 5, + }, + }); + }); + + it("reports the last request's timings alone when no tool ran", async () => { + const { client } = scriptedClient([[{ timings: { predicted_n: 3 }, type: "timings" }]]); + const events = await collect(client); + + expect(types(events)).toEqual(["turn_start", "timings", "message", "turn_end", "done"]); + expect(last(events)).toEqual({ reason: "complete", timings: { predicted_n: 3 }, type: "done" }); + }); +}); diff --git a/ui/lib/harness/tests/openai-client.test.ts b/ui/lib/harness/tests/openai-client.test.ts new file mode 100644 index 000000000000..fedcb59a0a8c --- /dev/null +++ b/ui/lib/harness/tests/openai-client.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { createOpenAIClient, type InferenceEvent } from "../src/index.js"; + +function sse(...chunks: unknown[]): string { + return chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n"; +} + +// answers every request with the given body, cut into pieces to cross read boundaries +function fakeFetch(body: string, init: ResponseInit = {}, pieceLength = 7) { + const calls: Array<{ url: string; init: RequestInit }> = []; + const fetch = (async (url: string, requestInit: RequestInit) => { + calls.push({ init: requestInit, url }); + + const bytes = new TextEncoder().encode(body); + const stream = new ReadableStream({ + start(controller) { + for (let i = 0; i < bytes.length; i += pieceLength) { + controller.enqueue(bytes.slice(i, i + pieceLength)); + } + + controller.close(); + }, + }); + + return new Response(stream, init); + }) as typeof globalThis.fetch; + + return { calls, fetch }; +} + +async function collect(stream: AsyncIterable): Promise { + const events: InferenceEvent[] = []; + + for await (const event of stream) events.push(event); + + return events; +} + +const delta = (fields: object, rest: object = {}) => ({ + choices: [{ delta: fields }], + id: "cmpl-1", + model: "m", + ...rest, +}); + +describe("createOpenAIClient", () => { + it("posts a streaming request to the chat completions endpoint", async () => { + const { calls, fetch } = fakeFetch(sse()); + const client = createOpenAIClient({ + apiKey: "key", + baseUrl: "http://localhost:8080/", + fetch, + model: "m", + params: { temperature: 0 }, + }); + + await collect(client.stream({ messages: [{ content: "hi", role: "user" }] })); + + expect(calls[0].url).toBe("http://localhost:8080/v1/chat/completions"); + expect(calls[0].init.headers).toMatchObject({ Authorization: "Bearer key" }); + expect(JSON.parse(calls[0].init.body as string)).toEqual({ + messages: [{ content: "hi", role: "user" }], + model: "m", + stream: true, + temperature: 0, + }); + }); + + it("turns the chunks into events", async () => { + const timings = { predicted_n: 2 }; + const { fetch } = fakeFetch( + sse( + delta({ reasoning_content: "hm" }), + delta({ content: "Hel" }), + delta({ content: "lo" }), + delta({}, { timings }), + ), + ); + const events = await collect( + createOpenAIClient({ baseUrl: "http://x", fetch }).stream({ messages: [] }), + ); + + expect(events).toEqual([ + { id: "cmpl-1", type: "completion_id" }, + { model: "m", type: "model" }, + { delta: "hm", type: "reasoning" }, + { delta: "Hel", type: "text" }, + { delta: "lo", type: "text" }, + { progress: undefined, timings, type: "timings" }, + ]); + }); + + it("merges tool call deltas by index", async () => { + const { fetch } = fakeFetch( + sse( + delta({ + tool_calls: [ + { function: { arguments: "", name: "read" }, id: "a", index: 0, type: "function" }, + ], + }), + delta({ tool_calls: [{ function: { arguments: '{"path"' }, index: 0 }] }), + delta({ + tool_calls: [ + { function: { arguments: ':"x"}' }, index: 0 }, + { function: { arguments: "{}", name: "list" }, id: "b", index: 1, type: "function" }, + ], + }), + ), + ); + const events = await collect( + createOpenAIClient({ baseUrl: "http://x", fetch }).stream({ messages: [] }), + ); + const toolCalls = events.filter((event) => event.type === "tool_calls"); + + expect(toolCalls).toHaveLength(3); + expect(toolCalls[2].toolCalls).toEqual([ + { function: { arguments: '{"path":"x"}', name: "read" }, id: "a", type: "function" }, + { function: { arguments: "{}", name: "list" }, id: "b", type: "function" }, + ]); + }); + + it("throws the server's error message", async () => { + const { fetch } = fakeFetch(JSON.stringify({ error: { message: "no model" } }), { + status: 400, + }); + const stream = createOpenAIClient({ baseUrl: "http://x", fetch }).stream({ messages: [] }); + + await expect(collect(stream)).rejects.toThrow("HTTP 400: no model"); + }); +}); diff --git a/ui/lib/harness/tsconfig.build.json b/ui/lib/harness/tsconfig.build.json new file mode 100644 index 000000000000..cb6a3121d48a --- /dev/null +++ b/ui/lib/harness/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "declaration": true, + "sourceMap": true + }, + "include": ["src"] +} diff --git a/ui/lib/harness/tsconfig.json b/ui/lib/harness/tsconfig.json new file mode 100644 index 000000000000..caa8927c4fc5 --- /dev/null +++ b/ui/lib/harness/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src", "tests"] +}