diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 538485fe9086..49c6fcb4f3c5 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -19,21 +19,22 @@ It works with a llama-server in either mode: **MODEL** (one model) or **ROUTER** Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app as static files on localhost, or writes them to a folder for any static file server (see [Self-hosting](#self-hosting)); it brings no server and no models of its own. It needs Node.js 20 or later. ```bash -npx @ggml-org/llama-chat # serve it on http://127.0.0.1:3000 and open the browser -npx llama-chat --host 0.0.0.0 --no-open # the same, by its short name, reachable from other devices -npx llama-chat build # write the static app to ./llama-chat instead +npx @ggml-org/llama-chat --server http://127.0.0.1:9931 # serve it on http://127.0.0.1:3000 with that server added +npx llama-chat --host 0.0.0.0 --no-open # the same, by its short name, reachable from other devices +npx llama-chat build --server https://llm.example.com # write the static app to ./llama-chat instead ``` | Flag | Default | What it does | | --------------------------------- | ----------- | ------------------------------------------------------------------------------------------- | +| `--server ` | | A llama-server the app adds on startup, once; repeatable. Its API key goes in the app | | `--port ` | `3000` | Fixed, so the origin, and the chats the browser stores under it, stay the same between runs | | `--host ` | `127.0.0.1` | `0.0.0.0` to open it to the LAN | | `--open` / `--no-open` | open | Opens the default browser once it listens | | `-v`, `--version`, `-h`, `--help` | | As usual | -The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`) and `--force`, which lets it write into a folder that is not empty. +The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`), `--force`, which lets it write into a folder that is not empty, and `--server`. -How it works: the CLI only serves the files of `dist`, with llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good), and forwards nothing. The browser talks to each llama-server you add in the app by URL directly, so llama-server's own rules apply: start it with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled, and with `--api-key` when it is reachable from the LAN. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot. +How it works: the CLI only serves the files of `dist`, with llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good), and forwards nothing. `--server` only puts the address in `llama-chat.json` next to `index.html`, which the app reads at startup to add the server; one you remove in the app stays removed. The CLI marks `index.html` with ``, and only a page with that tag asks for the file (`build --server` writes both). The browser talks to each llama-server directly, so llama-server's own rules apply: start it with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled, and with `--api-key` when it is reachable from the LAN. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot. ## Self-hosting diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js index 8ef0f557c3a4..70a7a33c80cd 100644 --- a/ui/apps/chat/cli/server.js +++ b/ui/apps/chat/cli/server.js @@ -6,12 +6,15 @@ * * It only serves files and forwards nothing: the browser calls each llama-server the app is * pointed at directly, so llama-server's own rules (--cors-origins, --api-key) apply as they do - * to any other page. + * to any other page. `--server` only tells the app which servers to add: the CLI serves them in + * `llama-chat.json` next to index.html, and `build` writes that file, which the app reads at + * startup. index.html carries a marker for it, so a page served any other way, such as by + * llama-server or the dev server, does not ask for a file that is not there. */ import { spawn } from 'node:child_process'; import { createReadStream, readFileSync } from 'node:fs'; -import { cp, readdir, stat } from 'node:fs/promises'; +import { cp, readdir, readFile, stat, writeFile } from 'node:fs/promises'; import http from 'node:http'; import { extname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -28,6 +31,15 @@ export const DEFAULTS = Object.freeze({ port: 3000 }); +/** The file next to index.html that lists the servers the app adds at startup. */ +export const PRESET_FILE = 'llama-chat.json'; + +/** + * The marker in index.html that sends the app to `llama-chat.json`. It names the file rather + * than the servers: the service worker keeps index.html, so servers in it would outlive a run. + */ +export const PRESET_META = ``; + // The same caching llama-server applies to the UI it embeds (tools/server/server-http.cpp): // index.html and the PWA's update checks are revalidated, hashed assets are cached for good. const NO_CACHE = new Set([ @@ -36,7 +48,8 @@ const NO_CACHE = new Set([ '/sw.js', '/manifest.webmanifest', '/_app/version.json', - '/build.json' + '/build.json', + `/${PRESET_FILE}` ]); const CACHE_IMMUTABLE = 'public, max-age=31536000, immutable'; const CACHE_REVALIDATE = 'no-cache'; @@ -65,13 +78,15 @@ const SERVE_OPTIONS = ['host', 'open', 'port']; const BUILD_OPTIONS = ['force', 'out']; export const USAGE = `Usage: llama-chat [options] - llama-chat build [--out ] [--force] + llama-chat build [--out ] [--force] [--server ] Serves Llama Chat on localhost as static files, or, with build, writes them to a folder for any HTTP server. The browser talks to llama-server directly: start it with --cors-origins set to this page's address when it restricts origins. Options: + --server a llama-server the app adds on startup, repeatable; the API + key, if it needs one, goes in the app --port port to listen on (default: ${DEFAULTS.port}) --host address to listen on, 0.0.0.0 for the LAN (default: ${DEFAULTS.host}) --open open the browser once it listens (default) @@ -82,11 +97,12 @@ Options: Build options: --out folder to write to (default: ./${DEFAULTS.out}) --force write into a folder that is not empty + --server as above, written to ${PRESET_FILE} in the folder Examples: - npx @ggml-org/llama-chat + npx @ggml-org/llama-chat --server http://127.0.0.1:9931 npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open - npx @ggml-org/llama-chat build --out ./public`; + npx @ggml-org/llama-chat build --out ./public --server https://llm.example.com`; /** The version in this package's package.json. */ export function packageVersion() { @@ -97,7 +113,7 @@ export function packageVersion() { * Reads the command line. No verb serves the app, `start` means the same, and `build` writes it * to a folder. * @param {string[]} argv the arguments after the command - * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, version: boolean }} + * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, servers: string[], version: boolean }} */ export function parseCliArgs(argv) { const { positionals, values } = parseArgs({ @@ -111,6 +127,7 @@ export function parseCliArgs(argv) { open: { type: 'boolean' }, out: { type: 'string' }, port: { type: 'string' }, + server: { multiple: true, type: 'string' }, version: { short: 'v', type: 'boolean' } }, strict: true @@ -146,10 +163,54 @@ export function parseCliArgs(argv) { open: values.open ?? true, out: values.out ?? DEFAULTS.out, port, + servers: (values.server ?? []).map(parseServerUrl), version: values.version ?? false }; } +/** + * A llama-server address for the app, with or without `http://`, as the app stores it: no + * trailing slash, a path kept (for `--api-prefix`). + * @param {string} value + */ +export function parseServerUrl(value) { + let url; + + try { + url = new URL(/^[a-z][a-z\d+.-]*:\/\//i.test(value) ? value : `http://${value}`); + } catch { + throw new Error(`invalid --server: ${value}`); + } + + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error(`invalid --server: ${value} (use an http:// or https:// address)`); + } + + if (url.search || url.hash || url.username || url.password) { + throw new Error(`invalid --server: ${value} (no query, fragment or credentials)`); + } + + return url.href.replace(/\/+$/, ''); +} + +/** + * index.html with the marker for `llama-chat.json` at the end of its head, once. + * @param {string} html + */ +export function withPresetMeta(html) { + if (html.includes('name="llama-chat-servers"')) return html; + + return html.replace('', `\t${PRESET_META}\n\t`); +} + +/** + * The contents of `llama-chat.json`. + * @param {string[]} servers + */ +export function presetFile(servers) { + return `${JSON.stringify({ servers: servers.map((url) => ({ url })) }, null, '\t')}\n`; +} + /** * Maps a request path to a file under `root`, or null when it falls outside it. * @param {string} root @@ -181,7 +242,10 @@ function resolveStatic(root, pathname) { */ function sendFile(req, res, file, pathname, stats) { const name = pathname === '/' ? '/index.html' : pathname; - const etag = `W/"${stats.size.toString(16)}-${Math.floor(stats.mtimeMs).toString(16)}"`; + // index.html goes out with the marker for llama-chat.json, and an ETag of its own + const body = + name === '/index.html' ? Buffer.from(withPresetMeta(readFileSync(file, 'utf8'))) : null; + const etag = `W/"${(body?.length ?? stats.size).toString(16)}-${Math.floor(stats.mtimeMs).toString(16)}"`; res.setHeader('ETag', etag); res.setHeader('Cache-Control', NO_CACHE.has(name) ? CACHE_REVALIDATE : CACHE_IMMUTABLE); @@ -201,7 +265,7 @@ function sendFile(req, res, file, pathname, stats) { 'Content-Type', MIME_TYPES[extname(file).toLowerCase()] ?? 'application/octet-stream' ); - res.setHeader('Content-Length', stats.size); + res.setHeader('Content-Length', body?.length ?? stats.size); res.writeHead(200); if (req.method === 'HEAD') { @@ -210,6 +274,12 @@ function sendFile(req, res, file, pathname, stats) { return; } + if (body) { + res.end(body); + + return; + } + createReadStream(file).pipe(res); } @@ -224,11 +294,13 @@ function sendError(res, status, message) { } /** - * The HTTP server: the files of `dist`, read-only. - * @param {{ distDir?: string }} [options] + * The HTTP server: the files of `dist`, read-only, and `llama-chat.json` with the servers given, + * if any: index.html always points the app at it. + * @param {{ distDir?: string, servers?: string[] }} [options] */ -export function createChatServer({ distDir = DIST_DIR } = {}) { +export function createChatServer({ distDir = DIST_DIR, servers = [] } = {}) { const root = normalize(distDir); + const preset = Buffer.from(presetFile(servers)); return http.createServer(async (req, res) => { const { pathname } = new URL(req.url ?? '/', 'http://localhost'); @@ -240,6 +312,17 @@ export function createChatServer({ distDir = DIST_DIR } = {}) { return; } + if (pathname === `/${PRESET_FILE}`) { + res.writeHead(200, { + 'Cache-Control': CACHE_REVALIDATE, + 'Content-Length': preset.length, + 'Content-Type': MIME_TYPES['.json'] + }); + res.end(req.method === 'HEAD' ? undefined : preset); + + return; + } + const file = resolveStatic(root, pathname); const stats = file ? await stat(file).catch(() => null) : null; @@ -255,11 +338,12 @@ export function createChatServer({ distDir = DIST_DIR } = {}) { /** * Writes the app to a folder: the files of `dist` without source maps, ready for any static - * file server. Refuses a folder that is not empty unless `force` is set, and a folder in `dist`. - * @param {{ distDir?: string, force?: boolean, out: string }} options + * file server, and `llama-chat.json` when servers are given, with the marker in index.html. Refuses a folder that is not empty + * unless `force` is set, and a folder in `dist`. + * @param {{ distDir?: string, force?: boolean, out: string, servers?: string[] }} options * @returns {Promise} the absolute path of the folder */ -export async function buildStatic({ distDir = DIST_DIR, force = false, out }) { +export async function buildStatic({ distDir = DIST_DIR, force = false, out, servers = [] }) { const source = resolve(distDir); const target = resolve(out); const inside = relative(source, target); @@ -276,6 +360,13 @@ export async function buildStatic({ distDir = DIST_DIR, force = false, out }) { await cp(source, target, { filter: (file) => !file.endsWith('.map'), recursive: true }); + if (servers.length > 0) { + const index = join(target, 'index.html'); + + await writeFile(join(target, PRESET_FILE), presetFile(servers)); + await writeFile(index, withPresetMeta(await readFile(index, 'utf8'))); + } + return target; } @@ -339,10 +430,17 @@ export async function main(argv) { if (options.command === 'build') { try { - const target = await buildStatic({ force: options.force, out: options.out }); + const target = await buildStatic({ + force: options.force, + out: options.out, + servers: options.servers + }); const shown = relative(process.cwd(), target) || '.'; console.log(`Llama Chat ${packageVersion()} written to ${target}`); + + for (const server of options.servers) console.log(`Adds the llama-server at ${server}`); + console.log(`Serve it with any static file server, e.g.: npx http-server ${shown}`); } catch (error) { console.error(`llama-chat: ${error instanceof Error ? error.message : error}`); @@ -352,7 +450,7 @@ export async function main(argv) { return; } - const server = createChatServer(); + const server = createChatServer({ servers: options.servers }); return new Promise((resolve) => { server.on('error', (error) => { @@ -373,6 +471,9 @@ export async function main(argv) { const url = `http://${shown.includes(':') ? `[${shown}]` : shown}:${address.port}/`; console.log(`Llama Chat ${packageVersion()} on ${url}`); + + for (const server of options.servers) console.log(`Adds the llama-server at ${server}`); + console.log( `If llama-server restricts origins (tools or MCP on), start it with --cors-origins ${url.slice(0, -1)}` ); diff --git a/ui/apps/chat/package-lock.json b/ui/apps/chat/package-lock.json index a0005085ed9a..ed6e762d09ef 100644 --- a/ui/apps/chat/package-lock.json +++ b/ui/apps/chat/package-lock.json @@ -91,6 +91,7 @@ "typescript-eslint": "8.72.1", "unified": "11.0.5", "unist-util-visit": "5.1.0", + "uqr": "0.1.3", "uuid": "13.0.2", "vaul-svelte": "1.0.0-next.7", "vite": "8.3.6", @@ -12450,6 +12451,13 @@ "node": ">=18.12.0" } }, + "node_modules/uqr": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/uqr/-/uqr-0.1.3.tgz", + "integrity": "sha512-0rjE8iEJe4YmT9TOhwsZtqCMRLc5DXZUI2UEYUUg63ikBkqqE5EYWaI0etFe/5KUcmcYwLih2RND1kq+hrUJXA==", + "dev": true, + "license": "MIT" + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/ui/apps/chat/package.json b/ui/apps/chat/package.json index 2e9ef1849236..c38f70e10932 100644 --- a/ui/apps/chat/package.json +++ b/ui/apps/chat/package.json @@ -151,6 +151,7 @@ "typescript-eslint": "8.72.1", "unified": "11.0.5", "unist-util-visit": "5.1.0", + "uqr": "0.1.3", "uuid": "13.0.2", "vaul-svelte": "1.0.0-next.7", "vite": "8.3.6", diff --git a/ui/apps/chat/src/app.d.ts b/ui/apps/chat/src/app.d.ts index 006a37a8cfaa..7b9e60c457f0 100644 --- a/ui/apps/chat/src/app.d.ts +++ b/ui/apps/chat/src/app.d.ts @@ -10,6 +10,7 @@ import type { ApiChatCompletionStreamChunk, ApiChatCompletionToolCall, ApiChatCompletionToolCallDelta, + ApiChatCompletionUsage, ApiChatMessageContentPart, ApiChatMessageData, ApiContextSizeError, @@ -72,6 +73,7 @@ declare global { ApiChatCompletionResponse, ApiChatCompletionStreamChunk, ApiChatCompletionToolCall, + ApiChatCompletionUsage, ApiChatCompletionToolCallDelta, ApiChatMessageData, ApiChatMessageContentPart, diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte index 802dd7e8d081..53c0820e1896 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte @@ -104,7 +104,7 @@ {/snippet} {#snippet body()} - {#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded} + {#if gauge.canLoadActiveModel && gauge.activeModelId !== null && !gauge.isActiveModelLoaded} {:else} {@const source = toolsStore.getToolSource(toolName)} - {@const providerName = + {@const serverName = source === ToolSource.SERVER ? TOOL_SERVER_LABELS[ToolSource.SERVER] : source === ToolSource.CUSTOM ? TOOL_SERVER_LABELS[ToolSource.CUSTOM] : 'MCP Tools'} onDecision(ToolPermissionDecision.ALWAYS_SERVER)}> - Approve all tools from {providerName} + Approve all tools from {serverName} {/if} diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte index 97c74861c62e..4b97863c4229 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte @@ -26,6 +26,7 @@ CHAT_FORM_MOVE_TO_BOTTOM_IN_MS, CHAT_FORM_MOVE_TO_BOTTOM_OUT_MS } from '#lib/constants/index.ts'; + import { useBackendAvailability } from '#lib/hooks/use-backend-availability.svelte.ts'; import { useChatScreenActiveModel } from '#lib/hooks/use-chat-screen-active-model.svelte.ts'; import { useChatScreenDragAndDrop } from '#lib/hooks/use-chat-screen-drag-and-drop.svelte.ts'; import { useChatScreenFileUpload } from '#lib/hooks/use-chat-screen-file-upload.svelte.ts'; @@ -77,8 +78,10 @@ let formWrapper: HTMLElement | undefined = $state(); let chatScreenForm: ChatScreenForm | undefined = $state(); let activeErrorDialog = $derived(chatStore.errorDialogState); - let isServerLoading = $derived(serverStore.loading); - let hasPropsError = $derived(!!serverStore.error); + // no local server in this deployment: never block on its probe + let isServerLoading = $derived(serverStore.loading && !serverStore.localServerMissing); + const availability = useBackendAvailability(); + let hasPropsError = $derived(availability.isOffline); let isCurrentConversationLoading = $derived(chatStore.isLoading || chatStore.isStreaming()); let chatFormBottomPosition = $derived.by(() => { if (!deviceStore.isMobile) return CHAT_FORM_BOTTOM_OFFSET.desktop; diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte index 22bde59ce057..720e397368ca 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte @@ -63,8 +63,9 @@

Hello there

- {serverStore.props?.modalities?.audio ? 'Record audio, type a message ' : 'Type a message'} or upload - files to get started + {serverStore.localProps?.modalities?.audio + ? 'Record audio, type a message ' + : 'Type a message'} or upload files to get started

diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte index 497c1efc2c14..4d8d60e8fb39 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte @@ -1,11 +1,18 @@ {#if hasError || isLoadingModel} @@ -33,7 +40,19 @@ {#if !isLoadingModel} - {serverStore.error} + + {serverStore.error} + + + + {/if} diff --git a/ui/apps/chat/src/lib/components/app/dialogs/DialogExportSettings.svelte b/ui/apps/chat/src/lib/components/app/dialogs/DialogExportSettings.svelte index 9afa1b17d6e6..35750c932e25 100644 --- a/ui/apps/chat/src/lib/components/app/dialogs/DialogExportSettings.svelte +++ b/ui/apps/chat/src/lib/components/app/dialogs/DialogExportSettings.svelte @@ -38,14 +38,14 @@ {#if includeSensitiveData}

- Warning: This export will include sensitive data such as API keys and MCP server custom - headers (e.g., authorization tokens). Do not share this file with anyone you don't - trust. + Warning: This export will include sensitive data such as API keys, remote access codes + and MCP server custom headers (e.g., authorization tokens). Do not share this file with + anyone you don't trust.

{:else}

- Sensitive data (API keys, MCP server custom headers) will not be included in the export - to protect your credentials. + Sensitive data (API keys, remote access codes, MCP server custom headers) will not be + included in the export to protect your credentials.

{/if}
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte index eb20342ce3cd..994101386db8 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte @@ -1,22 +1,29 @@ -{#if orgName && hubEnabled} +{#if useServerIcon} + + + {#snippet fallback()} + {#if isLocal} + + {:else} + + {/if} + {/snippet} + + +{:else if orgName && hubEnabled} (isNearViewport = true))} class={['inline-flex shrink-0', className]} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte b/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte index 023412df889d..cc978351d6e2 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte @@ -1,4 +1,5 @@ +{#snippet serverMark(server: Server)} + {#if server.id === LOCAL_BACKEND_ID} + + {#snippet fallback()} + + {/snippet} + + {:else} + + {/if} +{/snippet} + @@ -98,6 +136,51 @@ gapSize="2" innerClass="items-center" > + {#if availableServers.length > 1} + + + {#snippet child({ props })} + + {/snippet} + + + + + Servers + + {#each availableServers as server (server.id)} + toggleServer(server.id, checked)} + > + {@render serverMark(server)} + + {server.name} + + {serverCounts[server.id] ?? 0} + + {/each} + + + {#if servers.length > 0} + + + (servers = [])}>Show every server + {/if} + + + {/if} + (contextLimit = Number(value))} type="single" diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte index 9f0183e3188f..f9dbab94f505 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte @@ -14,7 +14,7 @@ import { conversationsStore, deviceStore, uiStore } from '#lib/stores/index.ts'; interface Props { - /** A provider that cannot load or unload has no button to offer. */ + /** A server that cannot load or unload has no button to offer. */ canToggleLoad?: boolean; isDownload?: boolean; isDownloadPaused?: boolean; diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte index 1f25c3a5553c..de64c61f042e 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte @@ -55,7 +55,7 @@ } // the "more actions" dropdown and the right-click menu offer the same actions - let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete, download)); + let actions = $derived(modelRowActions(option, { download, favorite, isHidden }, { onDelete })); let contextMenuOpen = $state(false); diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte index e24f4e91b51b..9f66a93d75a9 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte @@ -45,9 +45,10 @@ ModelCapability, ModelDownloadConfirmAction, ModelsTableGroupKind, + ModelsTableServerKind, ModelsTableSortKey } from '#lib/enums/index.ts'; - import { modelsStore, settingsStore } from '#lib/stores/index.ts'; + import { modelsStore, serversModelsStore, settingsStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; import { groupModelFamilies, type ModelFamilyGroup } from '#lib/utils/model-families.ts'; import type { Snippet } from 'svelte'; @@ -64,6 +65,10 @@ onSelect: (option: ModelOption) => void; /** Modalities a model must support at least one of. */ modalities?: ModalityKey[]; + /** Server ids to keep; empty keeps every server. */ + servers?: string[]; + /** Repos each server contributes to the current search, for the filter menu. */ + serverCounts?: Record; selectedId: string | null; /** Rendered at the toolbar's right end, past the filters. */ toolbarEnd?: Snippet; @@ -78,11 +83,15 @@ modalities = $bindable([]), onSelect, selectedId, + serverCounts = {}, + servers = $bindable([]), toolbarEnd }: Props = $props(); let isEmpty = $derived(groups.every((group) => group.items.length === 0)); - let hasFilters = $derived(hasActiveFilters(contextLimit, modalities, capabilities)); + let hasFilters = $derived( + hasActiveFilters(contextLimit, modalities, capabilities) || servers.length > 0 + ); /** Noun the show-more row counts in, per unit of the grouped list. */ const SHOW_MORE_NOUNS: Record = { @@ -95,7 +104,7 @@ /** Repos whose quants are folded away; the rest show them. */ const collapsedQuants = new SvelteSet(); /** Sections that list their models straight, without folding them into families. */ - const FLAT_SECTIONS = new Set([ + const FLAT_SECTIONS = new Set([ ModelsTableGroupKind.DOWNLOADING, ModelsTableGroupKind.FAVORITES, ModelsTableGroupKind.LOADED @@ -244,6 +253,7 @@ {onSelect} option={quant} selected={selectedId === quant.id} + showServer={entry.kind === 'servers'} /> {/each} @@ -348,6 +358,8 @@ bind:contextLimit bind:filter bind:modalities + bind:servers + {serverCounts} {toolbarEnd} /> @@ -382,15 +394,22 @@ {/if} {/snippet} + {@const serverState = group.serverId ? serversModelsStore.get(group.serverId) : null} + diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte index 2099309e5e29..bb3e9b99cba0 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte @@ -10,6 +10,7 @@ import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '#lib/constants/index.ts'; import { modelsStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; + import { getServer } from '#lib/utils/api-base.ts'; import { isAuxSidecar } from '#lib/utils/index.ts'; interface Props { @@ -18,18 +19,28 @@ onDelete: (option: ModelOption) => void; onSelect: (option: ModelOption) => void; selected: boolean; + /** Badge the server name instead of the quant, for a server group. */ + showServer?: boolean; /** Left padding in px, from the nesting depth. */ indent?: number; } - let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props(); + let { + indent = 0, + isFavorite, + onDelete, + onSelect, + option, + selected, + showServer = false + }: Props = $props(); let favorite = $derived(isFavorite(option)); let isHidden = $derived(modelsStore.isHidden(option.id)); let quant = $derived(option.parsedId?.quantization ?? option.model); // the "more actions" dropdown and the right-click menu offer the same actions - let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete)); + let actions = $derived(modelRowActions(option, { favorite, isHidden }, { onDelete })); let contextMenuOpen = $state(false); @@ -59,7 +70,9 @@ style="padding-left: {indent}px" type="button" > - {quant} + + {showServer ? (getServer(option.serverId)?.name ?? quant) : quant} + !isAuxSidecar(badge.kind))} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte index eee8d9ebba44..bad443931284 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte @@ -20,10 +20,13 @@ let { entry, expanded, indent = 0, onToggle }: Props = $props(); + let serverCount = $derived(new Set(entry.quants.map((option) => option.serverId ?? '')).size); let groupLabel = $derived( - entry.kind === ModelGroupKind.VARIANTS - ? `${entry.quants.length} variants` - : `${entry.quants.length} quants available` + entry.kind === ModelGroupKind.SERVERS + ? `${serverCount} server${serverCount === 1 ? '' : 's'}` + : entry.kind === ModelGroupKind.VARIANTS + ? `${entry.quants.length} variants` + : `${entry.quants.length} quants available` ); let anyLoaded = $derived(entry.quants.some((quant) => modelsStore.isModelRunning(quant.model))); // a repo row stands for its quants, so it reports what they agree on diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte new file mode 100644 index 000000000000..896ebf3133bf --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte @@ -0,0 +1,142 @@ + + +
+ void serversModelsStore.loadAll()} + onSaved={() => void serversModelsStore.loadAll()} + server={editing} + /> + + {#if !serverStore.localServerMissing} + handleAdd(ServerKind.REMOTE, code) : undefined} + onToggle={(enabled) => serversStore.setLocalEnabled(enabled)} + server={serversStore.local} + /> + {/if} + + {#each servers as server (server.id)} + void serversModelsStore.connectAndLoad(server.id)} + onDelete={() => { + serversStore.removeServer(server.id); + void serversModelsStore.loadAll(); + }} + onDisconnect={() => remoteAccessStore.disconnect(server.id)} + onEdit={() => handleEdit(server)} + onToggle={(enabled) => { + serversStore.updateServer(server.id, { enabled }); + void serversModelsStore.loadAll(); + }} + {server} + /> + {/each} + +
+ {#if isEmpty} + + +
+

This page has no server of its own

+ +

+ {#if canPair} + Start one with llama-server --connect and paste the access + code it prints, or point at a llama-server by its URL. + {:else} + Point at a llama-server by its URL. + {/if} +

+
+ + {#if canPair} + + {:else} + + {/if} + {:else} + + +

+ {canPair + ? 'Point at another llama-server, or pair one by its access code.' + : 'Point at another llama-server by its URL.'} +

+ {/if} +
+
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte index 7e0eaf5eb746..16d3243d4212 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte @@ -1,6 +1,7 @@ {#if download} -{:else if isRouter} +{:else if canToggleLoad} ; + /** Server ids to keep; empty keeps every server. */ + servers?: string[]; /** Rendered at the toolbar's right end, past the filters. */ toolbarEnd?: Snippet; } @@ -26,10 +30,14 @@ contextLimit = $bindable(0), filter = $bindable(''), modalities = $bindable([]), + serverCounts = {}, + servers = $bindable([]), toolbarEnd }: Props = $props(); - let hasFilters = $derived(hasActiveFilters(contextLimit, modalities, capabilities)); + let hasFilters = $derived( + hasActiveFilters(contextLimit, modalities, capabilities) || servers.length > 0 + ); let filterInput = $state(null); // The search takes the focus when the models page opens: the input mounts a frame @@ -61,12 +69,20 @@ size={deviceStore.isMobile ? 'default' : 'sm'} /> - + {#if hasFilters} + + + diff --git a/ui/apps/chat/src/lib/components/app/servers/DialogServerForm.svelte b/ui/apps/chat/src/lib/components/app/servers/DialogServerForm.svelte new file mode 100644 index 000000000000..9b98f64ebb4b --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/servers/DialogServerForm.svelte @@ -0,0 +1,417 @@ + + +{#snippet fields()} +
+ (showScanner = true)} + server={draft} + {urlError} + /> +
+ + handleChange({ code: scanned })} + /> + + +
+ {#if pairing} + + +
+

Pairing

+ +

Opening the tunnel to that server.

+
+ {:else if testing} + + +
+

Testing connection

+ +

+ Asking {hostOf(draft.baseUrl)} for the models it serves. +

+
+ {:else if pairError} + + +
+

Pairing failed

+ +

{pairError}

+ +

The server is saved; retry from its card.

+
+ {:else if testResult?.ok} + + +
+

Connected

+ +

+ {testResult.modelCount ?? 0} + model{(testResult.modelCount ?? 0) === 1 ? '' : 's'} available. +

+
+ {:else if testResult} + + +
+

Connection failed

+ +

+ {testResult.error ?? 'The endpoint did not answer.'}{testResult.status + ? ` (HTTP ${testResult.status})` + : ''} +

+
+ {:else} + {#if isRemote} + + {:else} + + {/if} + +
+

+ {isRemote ? 'Not paired yet' : 'Connection not tested'} +

+ +

+ {isRemote + ? 'Pairing saves the server and connects to it.' + : 'Test it to list the models this server serves.'} +

+
+ {/if} +
+{/snippet} + +{#snippet actions()} + + + {#if isRemote} + + {:else} + + + + {/if} +{/snippet} + +{#if deviceStore.isMobile} + + + + + {title} + + {description} + + +
+
+ {@render fields()} +
+ + {@render actions()} +
+
+
+{:else} + + + + {title} + + {description} + + +
+ {@render fields()} + + {@render actions()} +
+
+
+{/if} diff --git a/ui/apps/chat/src/lib/components/app/servers/DialogServers.svelte b/ui/apps/chat/src/lib/components/app/servers/DialogServers.svelte new file mode 100644 index 000000000000..1939343a4d2f --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/servers/DialogServers.svelte @@ -0,0 +1,48 @@ + + + + + deviceStore.isMobile && event.preventDefault()} + > + + + + + Servers + + + + {remoteAccessStore.isEnabled + ? 'Pair a remote server by its access code, or point at one by its URL.' + : 'Point at a llama-server by its URL.'} + + + +
+ +
+
+
diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerCard.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerCard.svelte new file mode 100644 index 000000000000..56aa0452875e --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/servers/ServerCard.svelte @@ -0,0 +1,334 @@ + + + +
+
+ + {#snippet fallback()} + {#if isLocal} + + {:else if remote} + + {:else} + + {/if} + {/snippet} + + +
+

{server.name}

+ + {#if remote} +
+ + + +
+ {:else} +

{displayUrl}

+ {/if} +
+
+ + {#if remote} + + + {#snippet child({ props })} + onToggle?.(value)} + /> + {/snippet} + + + +

Disabling also closes this server's tunnel.

+
+
+ {:else} + onToggle?.(value)} + /> + {/if} +
+ + {#if hostCode} +
+
+
+

Remote access: on

+ +

+ This server can be reached through its access code. +

+
+ +
+ + + + + {#if onPair} + + {/if} +
+
+ + {#if showHostQr && hostQr} +
+ Access code QR code + +

Scan it with the other device.

+
+ {/if} +
+ {/if} + + {#if status.detail} +

+ {status.detail} +

+ {/if} + +
+
+ {#if isConnecting || isChecking} + + {:else if isOnline} + + {:else if isOffline} + + {:else if isIdle} + + {/if} + + {#if hasStatus} + {status.label} + + {#if isOffline} + + {/if} + + · + {/if} + + {metaText} +
+ +
+ {#if isIdle} + + {/if} + + {#if remote && tunnel !== TunnelStatus.IDLE} + + {/if} + + {#if !isLocal} + + + + {/if} +
+
+
+ + (showDelete = false)} + onConfirm={() => { + showDelete = false; + onDelete?.(); + }} + title={remote ? 'Forget this server?' : 'Delete server?'} + variant="destructive" +/> diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte index fab3edba4090..75ea802d1784 100644 --- a/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte +++ b/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte @@ -3,10 +3,16 @@ import { Input } from '@ggml-org/llama-ui/ui/input'; import { Label } from '@ggml-org/llama-ui/ui/label'; import { AlertTriangle, CheckCircle, Key, RefreshCw, XCircle } from '@lucide/svelte'; - import { HEADERS, ICON_CLASS_DEFAULT, ROUTES, SETTINGS_KEYS } from '#lib/constants/index.ts'; + import { + HEADERS, + ICON_CLASS_DEFAULT, + LOCAL_BACKEND_ID, + ROUTES, + SETTINGS_KEYS + } from '#lib/constants/index.ts'; import { KeyboardKey } from '#lib/enums/index.ts'; import { serverStore, settingsStore } from '#lib/stores/index.ts'; - import { withBase } from '#lib/utils/base-path.ts'; + import { apiUrl } from '#lib/utils/api-base.ts'; import { goto } from '$app/navigation'; import { fade, fly, scale } from 'svelte/transition'; @@ -67,7 +73,7 @@ settingsStore.updateConfig(SETTINGS_KEYS.API_KEY, apiKeyInput.trim()); // Test the API key by making a real request to the server - const response = await fetch(withBase('/props'), { + const response = await fetch(apiUrl('/props', LOCAL_BACKEND_ID), { headers: { 'Content-Type': 'application/json', [HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKeyInput.trim()}` diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerForm.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerForm.svelte new file mode 100644 index 000000000000..79f01c3513cc --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/servers/ServerForm.svelte @@ -0,0 +1,188 @@ + + +
+ + + {#if kinds.length > 1} + onChange({ kind: value as ServerKind })} + value={server.kind} + > + + {#each kinds as kind (kind)} + + {kind === ServerKind.REMOTE ? 'Remote server' : 'Server URL'} + + {#if kind === ServerKind.REMOTE} + + {/if} + + {/each} + + + {/if} + + {#if isRemote} +
+ + +

+ Experimental. The pass code is currently sent + to whichever peer answers on the public tracker, so someone watching it could capture the code. + Mutual authentication, where neither side sends it, is planned. +

+
+ {/if} + + {#if isRemote} +
+ + +
+ onChange({ code: e.currentTarget.value })} + placeholder="Paste the {CODE_LENGTHS.SHARE}-character code" + spellcheck="false" + type="text" + value={server.code ?? ''} + /> + + {#if canScan} + + {/if} +
+ + {#if codeError} +

{codeError}

+ {:else} +

+ Printed by llama-server --connect. It is a credential: + anyone who has it can use that server. +

+ {/if} +
+ {:else} +
+ + + onChange({ baseUrl: e.currentTarget.value })} + placeholder="https://api.example.com" + type="url" + value={server.baseUrl} + /> + + {#if urlError} +

{urlError}

+ {:else if endpointError} +

{endpointError}

+ {/if} +
+ {/if} + +
+
+ + + onChange({ name: e.currentTarget.value })} + placeholder="Name shown in the model selector" + type="text" + value={server.name} + /> +
+ +
+ + + onChange({ apiKey: e.currentTarget.value || undefined })} + placeholder="Optional" + type="password" + value={server.apiKey ?? ''} + /> + +

+ {#if apiKeyRequired} + Required by this endpoint. + {:else} + Sent as a Bearer token. + {/if} +

+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerIcon.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerIcon.svelte new file mode 100644 index 000000000000..c3e1bcdae466 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/servers/ServerIcon.svelte @@ -0,0 +1,33 @@ + + +{#if faviconUrl && !failedFavicons.has(faviconUrl)} + failedFavicons.add(faviconUrl)} + src={faviconUrl} + /> +{:else} + {@render fallback?.()} +{/if} diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte index 630079fd9a73..590fcc66067f 100644 --- a/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte +++ b/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte @@ -15,7 +15,7 @@ let error = $derived(serverStore.error); let loading = $derived(serverStore.loading); let model = $derived(modelsStore.singleModelName); - let serverData = $derived(serverStore.props); + let serverData = $derived(serverStore.localProps); function getStatusColor() { if (loading) return 'status-lamp orb-warning'; diff --git a/ui/apps/chat/src/lib/components/app/servers/index.ts b/ui/apps/chat/src/lib/components/app/servers/index.ts index 29394fddebb9..c4afc9a28e73 100644 --- a/ui/apps/chat/src/lib/components/app/servers/index.ts +++ b/ui/apps/chat/src/lib/components/app/servers/index.ts @@ -78,3 +78,10 @@ export { default as ServerErrorSplash } from './ServerErrorSplash.svelte'; * ``` */ export { default as ServerLoadingSplash } from './ServerLoadingSplash.svelte'; + +export { default as ServerCard } from './ServerCard.svelte'; +export { default as ServerForm } from './ServerForm.svelte'; +export { default as ServerIcon } from './ServerIcon.svelte'; +export { default as DialogServerCodeScanner } from './DialogServerCodeScanner.svelte'; +export { default as DialogServerForm } from './DialogServerForm.svelte'; +export { default as DialogServers } from './DialogServers.svelte'; diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 75b9ac341e9c..e5582e17f04f 100644 --- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -68,12 +68,17 @@ return false; }); + // Snapshot used to persist only the fields edited in this dialog. Config keys + // that other surfaces write directly (servers, MCP servers) must not be + // clobbered by the dialog's save. + let baseline: SettingsConfigType = $state({ ...settingsStore.config }); + let mobileHeader: { updateCarousel: () => void } | undefined; let fetchInitiated = false; $effect(() => { - if (serverStore.isRouterMode && currentSection.fields?.length && !fetchInitiated) { + if (serverStore.localIsRouter && currentSection.fields?.length && !fetchInitiated) { fetchInitiated = true; void modelsStore @@ -108,6 +113,7 @@ function handleReset() { localConfig = { ...settingsStore.config }; + baseline = { ...settingsStore.config }; setMode(localConfig.theme as ColorMode); applySurfaceTheme(localConfig[SETTINGS_KEYS.SURFACE_THEME]); mobileHeader?.updateCarousel(); @@ -129,7 +135,13 @@ } } - const processedConfig = { ...localConfig }; + const processedConfig: SettingsConfigType = {}; + + for (const [key, value] of Object.entries(localConfig)) { + if (value !== baseline[key]) { + processedConfig[key] = value; + } + } for (const field of NUMERIC_FIELDS) { if (processedConfig[field] !== undefined && processedConfig[field] !== '') { @@ -161,6 +173,7 @@ export function reset() { localConfig = { ...settingsStore.config }; + baseline = { ...settingsStore.config }; } diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte index 9d864ca7cd49..a6ff5355bf12 100644 --- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte +++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte @@ -5,7 +5,7 @@ import * as RadioGroup from '@ggml-org/llama-ui/ui/radio-group'; import * as Select from '@ggml-org/llama-ui/ui/select'; import { Textarea } from '@ggml-org/llama-ui/ui/textarea'; - import { FlaskConical, RotateCcw } from '@lucide/svelte'; + import { FlaskConical, RotateCcw, ShieldAlert } from '@lucide/svelte'; import { SettingsChatParameterSourceIndicator } from '#lib/components/app/settings/index.ts'; import { ICON_CLASS_DEFAULT, @@ -47,7 +47,7 @@ let currentModelParams = $derived.by(() => { void modelsStore.props.cacheVersion; - if (serverStore.isRouterMode) { + if (serverStore.localIsRouter) { const currentModelName = modelsStore.selectedModelName; if (currentModelName) { @@ -349,6 +349,14 @@ {field.help || SETTING_CONFIG_INFO[field.key]}

{/if} + + {#if field.note} +

+ + + {field.note} +

+ {/if} {/if} diff --git a/ui/apps/chat/src/lib/components/app/settings/index.ts b/ui/apps/chat/src/lib/components/app/settings/index.ts index 184a34fc7f51..d2422daf2e85 100644 --- a/ui/apps/chat/src/lib/components/app/settings/index.ts +++ b/ui/apps/chat/src/lib/components/app/settings/index.ts @@ -91,3 +91,8 @@ export { default as SettingsChatFields } from './SettingsChat/SettingsChatFields * server favicons and permission management controls. */ export { default as SettingsChatToolsTab } from './SettingsChat/SettingsChatToolsTab.svelte'; + +/** + * Servers panel for chat settings. Switches between the llama.cpp and + * OpenAI-compatible protocols and lists the endpoints configured for each. + */ diff --git a/ui/apps/chat/src/lib/constants/backend.constants.ts b/ui/apps/chat/src/lib/constants/backend.constants.ts deleted file mode 100644 index 57f5c4213220..000000000000 --- a/ui/apps/chat/src/lib/constants/backend.constants.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Marks the built-in local llama.cpp backend. */ -export const LOCAL_BACKEND_ID = 'local'; diff --git a/ui/apps/chat/src/lib/constants/index.ts b/ui/apps/chat/src/lib/constants/index.ts index 9dfaa5e2bda7..6406b7eb621a 100644 --- a/ui/apps/chat/src/lib/constants/index.ts +++ b/ui/apps/chat/src/lib/constants/index.ts @@ -4,7 +4,8 @@ export * from './agentic.constants'; export * from './api-endpoints.constants'; export * from './app.constants'; -export * from './backend.constants'; +export * from './servers.constants'; +export * from './webrtc.constants'; export * from './chat-tabs.constants'; export * from './database.constants'; export * from './reasoning-effort.constants'; diff --git a/ui/apps/chat/src/lib/constants/routes.constants.ts b/ui/apps/chat/src/lib/constants/routes.constants.ts index 1d7180540155..f01a5ac40b53 100644 --- a/ui/apps/chat/src/lib/constants/routes.constants.ts +++ b/ui/apps/chat/src/lib/constants/routes.constants.ts @@ -19,6 +19,8 @@ export const ROUTES = { PROJECTS: '#/projects', /** Search — mobile-only full-page conversation search. */ SEARCH: '#/search', + /** Servers - a page on a phone, a dialog elsewhere. */ + SERVERS: '#/servers', /** Settings - a page on a phone, a dialog elsewhere. */ SETTINGS: '#/settings', /** Root — start of the app. */ @@ -31,6 +33,9 @@ export const MODELS_ROUTE_ID = '/models'; /** SvelteKit route id of every page an extension contributes, as `page.route.id` reports it. */ export const EXTENSION_PAGE_ROUTE_ID = '/[extensionPage]'; +/** SvelteKit route id of the servers page, as `page.route.id` reports it. */ +export const SERVERS_ROUTE_ID = '/servers'; + /** SvelteKit route id of the settings page, as `page.route.id` reports it. */ export const SETTINGS_ROUTE_ID = '/settings'; diff --git a/ui/apps/chat/src/lib/constants/servers.constants.ts b/ui/apps/chat/src/lib/constants/servers.constants.ts new file mode 100644 index 000000000000..3234bdbe8fa9 --- /dev/null +++ b/ui/apps/chat/src/lib/constants/servers.constants.ts @@ -0,0 +1,27 @@ +/** Prefix for generated ids of user-added servers. */ +export const SERVER_ID_PREFIX = 'server'; + +/** Chat completions path every server serves. */ +export const DEFAULT_SERVER_CHAT_PATH = '/v1/chat/completions'; + +/** Models listing path every server serves. */ +export const DEFAULT_SERVER_MODELS_PATH = '/v1/models'; + +/** Id of the built-in backend that points at the server serving this UI. */ +export const LOCAL_BACKEND_ID = 'local'; + +/** Favicon extract keyed by domain, for servers with no bundled mark. */ +export const FAVICON_SERVICE_URL = 'https://www.google.com/s2/favicons?domain='; + +/** + * File next to index.html listing servers the app adds on startup, as + * `{ "servers": [{ "url": "http://127.0.0.1:9931" }] }`. `llama-chat --server` + * serves it, and `llama-chat build --server` writes it. + */ +export const PRESET_SERVERS_FILE = 'llama-chat.json'; + +/** + * Name of the meta tag in index.html that says {@link PRESET_SERVERS_FILE} is + * there: pages served any other way, such as by llama-server, never ask for it. + */ +export const PRESET_SERVERS_META = 'llama-chat-servers'; diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts index dcaed3da880a..545f5534b6cf 100644 --- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts @@ -10,6 +10,7 @@ export const SETTINGS_KEYS = { ALWAYS_SHOW_TOOL_CALL_CONTENT: 'alwaysShowToolCallContent', API_KEY: 'apiKey', AUTO_MIC_ON_EMPTY: 'autoMicOnEmpty', + // llama.cpp's own flag: "backend" here is the compute backend, not a server BACKEND_SAMPLING: 'backend_sampling', COMPACTION_PRESERVED_STEPS: 'compactionPreservedSteps', COMPACTION_PROMPT: 'compactionPrompt', @@ -29,11 +30,13 @@ export const SETTINGS_KEYS = { DYNATEMP_RANGE: 'dynatemp_range', ENABLE_CONTINUE_GENERATION: 'enableContinueGeneration', ENABLE_DISCOVER_MODELS: 'enableDiscoverModels', + ENABLE_REMOTE_PAIRING: 'enableRemotePairing', EXCLUDE_REASONING_FROM_CONTEXT: 'excludeReasoningFromContext', FREQUENCY_PENALTY: 'frequency_penalty', FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks', GROUP_MODELS_BY_FAMILY: 'groupModelsByFamily', LIVE_MARKDOWN_PREVIEW: 'liveMarkdownPreview', + LOCAL_BACKEND_ENABLED: 'localBackendEnabled', MAX_IMAGE_RESOLUTION: 'maxImageMPixels', MAX_TOKENS: 'max_tokens', MCP_REQUEST_TIMEOUT_SECONDS: 'mcpRequestTimeoutSeconds', @@ -54,6 +57,8 @@ export const SETTINGS_KEYS = { REPEAT_PENALTY: 'repeat_penalty', SAMPLERS: 'samplers', SEND_ON_ENTER: 'sendOnEnter', + // the server list is stored under the key it shipped with, so the value is not its name + SERVERS: 'backends', SHOW_AGENTIC_TURN_STATS: 'showAgenticTurnStats', SHOW_BUILD_VERSION: 'showBuildVersion', SHOW_FULL_PATH_IN_MENTIONS: 'showFullPathInMentions', diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts index 454d00bde562..fb631642d722 100644 --- a/ui/apps/chat/src/lib/constants/settings.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings.constants.ts @@ -188,6 +188,15 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [ label: 'Show MCP', type: SettingsFieldType.CHECKBOX }, + { + defaultValue: false, + help: 'Experimental: pair a llama-server started with --connect by its access code or QR code, and reach it peer-to-peer over WebRTC from any network. Servers added by URL work either way.', + isExperimental: true, + key: SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, + label: 'Enable remote server pairing', + note: 'Known security gap: the pass code in the access code is sent to whichever peer answers on the public WebTorrent tracker, so someone watching that tracker could capture it. A mutual challenge-response, where both sides prove they know the code without ever sending it, is the planned fix.', + type: SettingsFieldType.CHECKBOX + }, { defaultValue: true, help: 'Choose how conversation titles are generated. The first non-empty line uses a fast deterministic rule; the LLM option uses a model-generated title from the first message exchange.', @@ -879,6 +888,7 @@ function toSettingsSection(section: SettingsSectionEntry): SettingsSection { label: s.label, max: s.max, min: s.min, + note: s.note, options: s.options as SettingsFieldConfig['options'], placeholder: s.placeholder, radioOptions: s.radioOptions, diff --git a/ui/apps/chat/src/lib/constants/storage.constants.ts b/ui/apps/chat/src/lib/constants/storage.constants.ts index fa8d6a09dfb9..31cd6d0b9f04 100644 --- a/ui/apps/chat/src/lib/constants/storage.constants.ts +++ b/ui/apps/chat/src/lib/constants/storage.constants.ts @@ -16,6 +16,13 @@ export const DB_APP_NAME_DEPRECATED = 'LlamacppWebui'; export const ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.alwaysAllowedTools`; +/** Id of the server the selector and new requests target, restored on page load. */ +/* Kept under the name it was first stored under; only the constant was renamed. */ +export const ACTIVE_SERVER_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.activeBackend`; + +/** Base URLs of the preset servers already added once, so one the user removed stays removed. */ +export const PRESET_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.presetServers`; + export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`; export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`; @@ -32,10 +39,16 @@ export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteMod /** Open state the user set for a model list section or one of its families, by id. */ export const MODEL_GROUP_OPEN_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.modelGroupOpen`; -/** Recently used model ids, most recent first, backend-qualified. */ +/** Per-model load and inference overrides, keyed by server-qualified model id. */ +export const MODEL_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.modelOverrides`; + +/** Model the user picked last, kept across reloads. Stores `{ id, model }`. */ +export const SELECTED_MODEL_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.selectedModel`; + +/** Recently used model ids, most recent first, server-qualified. */ export const RECENT_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.recentModels`; -/** Model ids kept out of the selector, backend-qualified. */ +/** Model ids kept out of the selector, server-qualified. */ export const HIDDEN_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.hiddenModels`; export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`; /** Sidebar conversation sections the user folded. */ @@ -47,11 +60,15 @@ export const SIDEBAR_EXPANDED_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.sidebarExp export const SIDEBAR_WIDTH_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.sidebarWidth`; export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`; export const DISMISSED_RECOMMENDED_MCP_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.dismissedRecommendedMcpServers`; +/* Likewise: the value is the key already written, the name is the one the list now has. */ +export const DISMISSED_RECOMMENDED_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.dismissedRecommendedBackends`; /** Key prefix for per-conversation resumable stream state, conversationId is appended */ export const STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX = `${STORAGE_APP_NAME}.streamResume.`; // Deprecated old key names (kept for backward compat while users migrate) +/** @deprecated Remote servers are settings-backed now; the demo build kept them here */ +export const LEGACY_TUNNEL_SERVERS_LOCALSTORAGE_KEY = 'llama_servers'; /** @deprecated Use {@link ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY} instead */ export const DEPRECATED_ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME_DEPRECATED}.alwaysAllowedTools`; /** @deprecated Use {@link CONFIG_LOCALSTORAGE_KEY} instead */ diff --git a/ui/apps/chat/src/lib/constants/stream.constants.ts b/ui/apps/chat/src/lib/constants/stream.constants.ts index 64f67243c266..b590996c045c 100644 --- a/ui/apps/chat/src/lib/constants/stream.constants.ts +++ b/ui/apps/chat/src/lib/constants/stream.constants.ts @@ -2,6 +2,10 @@ // while the tab was hidden. covers brief background pauses without thrashing live streams export const STREAM_VISIBILITY_KICK_MS = 3000; +// minimum gap between synthesized live timing updates for servers that do not +// stream their own, keeps the per-chunk state updates cheap +export const STREAM_LIVE_TIMINGS_INTERVAL_MS = 500; + // separator joining a conversation id and its per-model stream identity // suffix (conv::model) used by the server side replay buffer export const CONVERSATION_ID_SEPARATOR = '::'; diff --git a/ui/apps/chat/src/lib/constants/ui.constants.ts b/ui/apps/chat/src/lib/constants/ui.constants.ts index 90f06c68d8e0..fb9fe638183d 100644 --- a/ui/apps/chat/src/lib/constants/ui.constants.ts +++ b/ui/apps/chat/src/lib/constants/ui.constants.ts @@ -5,8 +5,13 @@ import { SHORTCUT_SEARCH, SHORTCUT_SETTINGS } from './keyboard-shortcuts.constants'; -import { MODELS_ROUTE_ID, PROJECTS_ROUTE_ID, SETTINGS_ROUTE_ID } from './routes.constants'; -import { Search, Settings, SquarePen } from '@lucide/svelte'; +import { + MODELS_ROUTE_ID, + PROJECTS_ROUTE_ID, + SERVERS_ROUTE_ID, + SETTINGS_ROUTE_ID +} from './routes.constants'; +import { Search, Server, Settings, SquarePen } from '@lucide/svelte'; import { SidebarAction, ToolSource } from '#lib/enums/index.ts'; import type { DesktopIconStripItem } from '#lib/types/index.ts'; @@ -96,8 +101,8 @@ export const STATS_UNITS = { export const DEFAULT_MOBILE_BREAKPOINT = 768; -/** Models listed per remote provider before the "+ X more" line; search covers the rest. */ -export const REMOTE_PROVIDER_MODEL_LIMIT = 12; +/** Models listed per remote server before the "+ X more" line; search covers the rest. */ +export const REMOTE_SERVER_MODEL_LIMIT = 12; /** Model rows mounted before the list scrolls; a local catalog can hold hundreds. */ export const MODEL_ROW_WINDOW = 50; @@ -182,6 +187,12 @@ export const SIDEBAR_FOOTER_ITEMS: DesktopIconStripItem[] = [ keys: SHORTCUT_MODELS, tooltip: 'Models' }, + { + action: SidebarAction.SERVERS, + activeRouteId: SERVERS_ROUTE_ID, + icon: Server, + tooltip: 'Servers' + }, { action: SidebarAction.SETTINGS, activeRouteId: SETTINGS_ROUTE_ID, diff --git a/ui/apps/chat/src/lib/constants/webrtc.constants.ts b/ui/apps/chat/src/lib/constants/webrtc.constants.ts new file mode 100644 index 000000000000..c5ccf643af4f --- /dev/null +++ b/ui/apps/chat/src/lib/constants/webrtc.constants.ts @@ -0,0 +1,66 @@ +/** + * Remote server transport constants. + * + * A remote server is addressed by a synthetic origin whose host carries its + * id (`https://.remote.llamacpp.invalid`). The suffix is reserved + * by RFC 6761, so a request that ever escapes the tunnel transport dies in the + * browser instead of reaching the server that happens to serve this UI. + */ + +/** Reserved host suffix of a remote server's synthetic origin. */ +export const TUNNEL_HOST_SUFFIX = 'remote.llamacpp.invalid'; + +/** + * Public WebTorrent trackers used as the signaling rendezvous. The offer is + * announced on all of them at once and the first answer wins. + */ +export const TRACKER_URLS = [ + 'wss://tracker.openwebtorrent.com', + 'wss://tracker.webtorrent.dev' +] as const; + +/** STUN servers used to discover the public address for NAT traversal. */ +export const STUN_URLS = ['stun:stun.l.google.com:19302', 'stun:stun1.l.google.com:19302'] as const; + +/** Deadlines for each step of establishing a tunnel. */ +export const WEBRTC_TIMEOUTS = { + /** Deadline for the host to answer and the access code to be accepted */ + CONNECT_MS: 30_000, + /** Deadline for gathering ICE candidates before the offer goes out */ + ICE_GATHER_MS: 10_000, + /** + * Wait before reconnecting the channels a backgrounded tab lost. The network + * is not always back the moment the tab is. + */ + RECOVER_MS: 2_000, + /** Deadline for the WebSocket handshake with a tracker */ + TRACKER_CONNECT_MS: 10_000 +} as const; + +/** + * Lengths of the codes shared with the user. The share code is the room code + * followed by the pass code. + */ +export const CODE_LENGTHS = { + /** WebTorrent trackers require an info_hash of exactly this length */ + INFO_HASH: 20, + /** Secret authenticating the client on the data channel */ + PASS: 32, + /** Rendezvous key, padded to INFO_HASH */ + ROOM: 8, + /** Room and pass code concatenated, what the user pastes */ + SHARE: 40 +} as const; + +/** Lengths of the random identifiers exchanged over the wire. */ +export const ID_LENGTHS = { + /** Correlates an offer with its answer */ + OFFER: 20, + /** Identifies this peer to the trackers */ + PEER: 20, + /** Correlates a tunneled request with its response */ + REQUEST: 16 +} as const; + +/** Characters that are unambiguous to read aloud or type. */ +export const CODE_CHARS = 'ABCDEFGHJKMNPQRSTUVWXYZabcdefghjkmnpqrstuvwxyz23456789'; diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts index 3a9e67de7bcc..be3d4270a015 100644 --- a/ui/apps/chat/src/lib/enums/index.ts +++ b/ui/apps/chat/src/lib/enums/index.ts @@ -16,6 +16,8 @@ export { ToolCallType } from './agentic.enums'; +export { ServerKind, ServerCardState, TunnelStatus } from './servers.enums'; + export { ChatMessageStatsView, ChatMessageStatisticsMode, @@ -84,6 +86,7 @@ export { ModelGroupKind, ModelRowDownloadState, ModelsTableGroupKind, + ModelsTableServerKind, ModelsTableSortKey } from './model.enums'; diff --git a/ui/apps/chat/src/lib/enums/model.enums.ts b/ui/apps/chat/src/lib/enums/model.enums.ts index e3a4d7b4f4f8..faceb6e5fe72 100644 --- a/ui/apps/chat/src/lib/enums/model.enums.ts +++ b/ui/apps/chat/src/lib/enums/model.enums.ts @@ -40,9 +40,10 @@ export enum ModelDownloadConfirmAction { DELETE = 'delete' } -/** What a table row group folds: the quants of one repo, or its variants. */ +/** What a table row group folds: the servers of one repo, its quants, or its variants. */ export enum ModelGroupKind { QUANTS = 'quants', + SERVERS = 'servers', VARIANTS = 'variants' } @@ -67,3 +68,9 @@ export enum ModelsTableSortKey { CONTEXT = 'context', NAME = 'name' } + +/** Kinds a server block carries on top of the manager's own sections. */ +export enum ModelsTableServerKind { + COMPAT = 'compat', + SERVER = 'server' +} diff --git a/ui/apps/chat/src/lib/enums/servers.enums.ts b/ui/apps/chat/src/lib/enums/servers.enums.ts new file mode 100644 index 000000000000..d77a09cff9d7 --- /dev/null +++ b/ui/apps/chat/src/lib/enums/servers.enums.ts @@ -0,0 +1,52 @@ +/** + * Server transport enums. + * + * Every entry is a llama.cpp server the UI can talk to; only the way it is + * reached changes: its own base URL, or a WebRTC tunnel addressed by the access + * code its `llama-server --connect` prints. + */ + +/** How a server is reached. */ +export enum ServerKind { + /** The llama-server serving this UI; it has no base URL of its own. */ + LOCAL = 'local', + /** A remote llama.cpp server, reached through a WebRTC tunnel. */ + REMOTE = 'remote', + /** A llama.cpp server answering on its own base URL. */ + URL = 'url' +} + +/** Tunnel state of a remote server. */ +export enum TunnelStatus { + /** Data channel open and authenticated. */ + CONNECTED = 'connected', + /** Handshake in progress; requests for the server are held back. */ + CONNECTING = 'connecting', + /** Handshake failed or the channel dropped; requests are refused. */ + ERROR = 'error', + /** No tunnel for this server, none attempted. */ + IDLE = 'idle' +} + +/** + * State a server card reports: its tunnel and the server behind it composed. + * A remote server has two things that can fail, and the tunnel wins, since a + * closed tunnel makes the listing meaningless. A remote server nobody + * connected to yet is idle, which is a choice the user made, not a failure. + */ +export enum ServerCardState { + /** The channel dropped, or the handshake failed. */ + CHANNEL_DOWN = 'channel_down', + /** The server is being asked for its models. */ + CHECKING = 'checking', + /** Handshake in progress. */ + CONNECTING = 'connecting', + /** Remote server with no channel, none attempted. */ + IDLE = 'idle', + /** Nothing to report. */ + NONE = 'none', + /** Listing answered. */ + ONLINE = 'online', + /** Channel up, or no channel to fail, and the server did not answer. */ + UNREACHABLE = 'unreachable' +} diff --git a/ui/apps/chat/src/lib/enums/ui.enums.ts b/ui/apps/chat/src/lib/enums/ui.enums.ts index 4ef8428df603..02ce4b740725 100644 --- a/ui/apps/chat/src/lib/enums/ui.enums.ts +++ b/ui/apps/chat/src/lib/enums/ui.enums.ts @@ -22,6 +22,7 @@ export enum SidebarAction { NEW_CHAT = 'new-chat', PROJECTS = 'projects', SEARCH = 'search', + SERVERS = 'servers', SETTINGS = 'settings' } diff --git a/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts b/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts index 48b330ee6e36..903cb3a38b88 100644 --- a/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts +++ b/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts @@ -28,6 +28,7 @@ import { DEFAULT_IMAGE_MIME_TYPE, DEFAULT_MCP_CONFIG, HEADERS, + LOCAL_BACKEND_ID, MCP_RECONNECT, NEWLINE } from '#lib/constants/index.ts'; @@ -534,7 +535,8 @@ export class MCPService { if (useProxy) { requestInit.headers = { - ...getAuthHeaders(), + // the proxy is the local llama-server, so it takes the local key + ...getAuthHeaders(LOCAL_BACKEND_ID), ...(requestInit.headers as Record) }; } diff --git a/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts b/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts new file mode 100644 index 000000000000..a1b8e75a9b9a --- /dev/null +++ b/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts @@ -0,0 +1,46 @@ +/** + * Backend availability for connection error states. + * + * The server error banner and the offline affordances should only appear when + * nothing can serve requests. The local server failing is not enough: with an + * external server configured, the UI stays usable. + */ + +import { + remoteAccessStore, + serversModelsStore, + serversStore, + serverStore +} from '#lib/stores/index.ts'; +import { isLocalServer, isRemoteServer } from '#lib/utils/servers.ts'; + +export interface BackendAvailability { + readonly hasAvailableBackend: boolean; + readonly isOffline: boolean; +} + +export function useBackendAvailability(): BackendAvailability { + const hasAvailableBackend = $derived.by(() => { + // no connection error at all: nothing to report + if (!serverStore.error) return true; + + // the local server failed, but another server may still be usable; a + // remote one is only usable while its tunnel is open + return serversStore.enabled.some( + (server) => + !isLocalServer(server) && + serversModelsStore.get(server.id).error === null && + (!isRemoteServer(server) || remoteAccessStore.isConnected(server.id)) + ); + }); + + return { + get hasAvailableBackend() { + return hasAvailableBackend; + }, + + get isOffline() { + return !hasAvailableBackend; + } + }; +} diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts index d9f81b6adef3..410e1c14e09d 100644 --- a/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts @@ -8,7 +8,7 @@ * fetching model props from the server on demand if they are not cached yet. */ -import { conversationsStore, modelsStore, serverStore } from '#lib/stores/index.ts'; +import { conversationsStore, modelsStore, serversStore, serverStore } from '#lib/stores/index.ts'; import type { DatabaseMessage } from '#lib/types/database.js'; import { getConversationModel } from '#lib/utils/index.ts'; @@ -24,10 +24,15 @@ export function useChatFormModel() { // points the selection at the model that produced it $effect(() => { if (conversationModel && conversationModel !== lastSyncedConversationModel) { - if (modelsStore.models.some((m) => m.model === conversationModel)) { + const option = modelsStore.models.find((m) => m.model === conversationModel); + + // only sync models served by the active server; a model from another + // server must not yank the active tab (and trigger a full server + // switch) just because the conversation used it + if (option && option.serverId === serversStore.active.id) { modelsStore.selectedModelName = conversationModel; modelsStore.selectModelByName(conversationModel); - } else { + } else if (!option) { modelsStore.selectedModelName = null; modelsStore.clearSelection(); } diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts index a06b88b8b0b1..724aae5adc89 100644 --- a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts @@ -24,11 +24,11 @@ import { voiceTranscriptsStore } from '#lib/stores/index.ts'; import type { ChatUploadedFile } from '#lib/types/index.ts'; +import { apiUrl } from '#lib/utils/api-base.ts'; import { getAuthHeaders } from '#lib/utils/api-headers.ts'; import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; -import { withBase } from '#lib/utils/base-path.ts'; import { DictationSpan } from '#lib/utils/dictation-span.ts'; -import { uuid } from '#lib/utils/index.ts'; +import { getServer, isRemoteServer, uuid } from '#lib/utils/index.ts'; import { resolveVoiceMode, voiceModeLabel } from '#lib/utils/voice-mode.ts'; import { resolveMicrophone, @@ -97,18 +97,15 @@ function readAsDataUrl(blob: Blob): Promise { }); } -/** The microphone's server: its routes, its auth, and whether a websocket reaches it. */ +/** The microphone's server: the active one, its auth, and whether a websocket reaches it. */ function serverContext(): { context: VoiceServerContext; realtime: boolean } { const headers = getAuthHeaders(); - // a browser websocket can not send the API key + // a browser websocket can send neither the API key nor a tunnel's transport const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization'); return { - context: { - getHeaders: () => getAuthHeaders(), - resolveUrl: (path) => withBase(path.replace(/^\./, '')) - }, - realtime: !hasAuth + context: { getHeaders: () => getAuthHeaders(), resolveUrl: (path) => apiUrl(path) }, + realtime: !isRemoteServer(getServer()) && !hasAuth }; } @@ -131,7 +128,7 @@ export function useChatFormVoice(options: UseChatFormVoiceOptions) { if (!isRouter) return null; return ( - modelsStore.models.find( + modelsStore.activeModels.find( (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model) )?.model ?? null ); @@ -151,7 +148,7 @@ export function useChatFormVoice(options: UseChatFormVoiceOptions) { const anyAudioModel = $derived.by(() => { void modelsStore.props.cacheVersion; - return isRouter && modelsStore.models.some((option) => hasAudio(option.model)); + return isRouter && modelsStore.activeModels.some((option) => hasAudio(option.model)); }); let hintSeen = $state(readHintSeen()); diff --git a/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts b/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts index 66bd9058269b..9e6da6e29437 100644 --- a/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts @@ -15,13 +15,16 @@ import { chatStore, contextStatsStore, conversationsStore, - modelsStore + modelsStore, + serverStore } from '#lib/stores/index.ts'; import type { DatabaseMessage } from '#lib/types/index.ts'; import { page } from '$app/state'; export interface UseContextGaugeReturn { readonly activeModelId: string | null; + /** Whether the active server can load the model it is serving. */ + readonly canLoadActiveModel: boolean; readonly isActiveModelLoaded: boolean; readonly isActiveModelLoading: boolean; readonly contextTotal: number | null; @@ -100,6 +103,9 @@ export function useContextGauge(): UseContextGaugeReturn { contextStatsStore.averageTokensPerSecond !== null || transientDetails.length > 0 ); + // loading is a llama.cpp router feature, and the gauge tracks the active model, + // so the active server decides whether a load is possible at all + const canLoadActiveModel = $derived(serverStore.isRouterMode); async function loadModel() { const modelId = contextStatsStore.activeModelId; @@ -120,6 +126,9 @@ export function useContextGauge(): UseContextGaugeReturn { get averageTokensPerSecond() { return contextStatsStore.averageTokensPerSecond; }, + get canLoadActiveModel() { + return canLoadActiveModel; + }, get colorLevel() { return colorLevel; }, diff --git a/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts b/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts index 1f835051e2cd..e85909d9216e 100644 --- a/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts @@ -1,21 +1,39 @@ -import { CHAT_INPUT_FOCUS_SELECTOR } from '#lib/constants/index.ts'; -import { deviceStore, modelsStore, serverStore, uiStore } from '#lib/stores/index.ts'; +import { + CHAT_INPUT_FOCUS_SELECTOR, + LOCAL_BACKEND_ID, + REMOTE_SERVER_MODEL_LIMIT +} from '#lib/constants/index.ts'; +import { + deviceStore, + modelsStore, + serversModelsStore, + serversStore, + serverStore, + uiStore +} from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; import { filterModelOptions, + type GroupedModelOptions, groupFavoriteOptions, groupModelOptions, + groupServerOptions, type ModelItem } from '#lib/utils/index.ts'; import { nameOf } from '#lib/utils/model-names.ts'; +import { rawModelId } from '#lib/utils/model-option-id.ts'; import { onMount } from 'svelte'; import { SvelteSet } from 'svelte/reactivity'; +/** Groups of the favorites tab, which lists favorites only. */ +const EMPTY_GROUPS: GroupedModelOptions = { available: [], servers: [] }; + export interface UseModelsSelectorOptions { currentModel: () => string | null; useGlobalSelection?: () => boolean; + /** The caller's model outranks the active server's selection, e.g. a message's own. */ onModelChange?: () => - | ((modelId: string, modelName: string) => Promise | boolean | void) + | ((modelId: string, modelName: string, serverId?: string) => Promise | boolean | void) | undefined; onOpenChange?: (open: boolean) => void; } @@ -35,10 +53,14 @@ export interface UseModelsSelectorReturn { readonly loadedItems: ModelItem[]; readonly filteredOptions: ModelOption[]; readonly isEmpty: boolean; + readonly isServerView: boolean; readonly groupedFilteredOptions: ReturnType; readonly isLoadingModel: boolean; readonly searchTerm: string; + closeServer(): void; + openServer(serverId: string): void; setSearchTerm(value: string): void; + showServerModels(serverId: string): Promise; handleSelect(modelId: string): Promise; handleOpenChange(open: boolean): void; isFavorite(model: string): boolean; @@ -53,20 +75,36 @@ export interface UseModelsSelectorReturn { * selection handling, and model loading. */ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSelectorReturn { - let isLoadingModel = $state(false); - let searchTerm = $state(''); - - const options = $derived( + /** + * Current view: the favorites of every server, the local server's models, or + * the remote servers'. Favorites are the default while there is at least one. + */ + /** Remote server drilled into from its section; null while browsing. */ + let serverViewId = $state(null); + + const isServerView = $derived(serverViewId !== null); + const isLocalOption = (option: ModelOption) => option.serverId === LOCAL_BACKEND_ID; + // every enabled server's models are one list: favorites, then the local + // server, then one section per remote server + const allOptions = $derived( modelsStore.models.filter((option) => { const modelProps = modelsStore.props.getModelProps(option.model); return modelProps?.ui !== false; }) ); + const options = $derived( + serverViewId ? allOptions.filter((option) => option.serverId === serverViewId) : allOptions + ); const loading = $derived(modelsStore.loading); const updating = $derived(modelsStore.updating); const activeId = $derived(modelsStore.selectedModelId); - // a lone llama.cpp server without a router has nothing to choose from + // Router mode and external servers both expose a selectable model list; only + // a lone llama.cpp server without a router has nothing to choose from. + const isMultiModel = $derived( + serverStore.isRouterMode || + serversStore.enabled.some((server) => server.id !== LOCAL_BACKEND_ID) + ); const isRouter = $derived(serverStore.isRouterMode); const serverModel = $derived(modelsStore.singleModelName); const currentModel = $derived(opts.currentModel()); @@ -74,31 +112,57 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele const isHighlightedCurrentModelActive = $derived.by(() => { if (!isRouter || !currentModel) return false; - const currentOption = options.find((option) => option.model === currentModel); + const currentOption = allOptions.find((option) => option.model === currentModel); return currentOption ? currentOption.id === activeId : false; }); const isCurrentModelInCache = $derived.by(() => { if (!isRouter || !currentModel) return true; - return options.some((option) => option.model === currentModel); + return allOptions.some((option) => option.model === currentModel); }); - // the search, the rows and the sections all read the visible set; only the current - // model resolves against `options`, since it can be hidden and still selected + + let isLoadingModel = $state(false); + let searchTerm = $state(''); + + // the search, the rows and the sections all read the visible set, while the + // favorites and loaded blocks span every server and read the full one + const allVisible = $derived(allOptions.filter((option) => !modelsStore.isHidden(option.id))); const visibleOptions = $derived(options.filter((option) => !modelsStore.isHidden(option.id))); - // one filter pass feeds the favorites, the loaded rows and the sections alike + // the sections read the view's own set; the favorites and the loaded rows span + // every server and read the full visible one const filteredOptions = $derived(filterModelOptions(visibleOptions, searchTerm)); - const loadedItems = $derived( - filteredOptions - .filter((option) => modelsStore.isModelLoaded(option.model)) - .map((option) => ({ option })) + const filteredAllOptions = $derived(filterModelOptions(allVisible, searchTerm)); + const remoteServers = $derived( + serversStore.enabled + .filter((server) => server.id !== LOCAL_BACKEND_ID) + .map((server) => { + const state = serversModelsStore.get(server.id); + + return { + catalog: state.models.length, + error: state.error, + loading: state.loading, + name: server.name, + serverId: server.id + }; + }) ); + // loaded models lead the list, whichever server serves them + const isLoadedHere = (option: ModelOption) => modelsStore.isModelLoaded(option.model); + const loadedItems = $derived.by(() => { + if (isServerView) return []; + + return filteredAllOptions + .filter((option) => isLoadedHere(option)) + .map((option) => ({ option })); + }); const loadedIds = $derived(new SvelteSet(loadedItems.map((item) => item.option.id))); // loaded models lead the list: their own sections list them once, so a // loaded favorite shows there and not twice const favoriteItems = $derived( groupFavoriteOptions( - filteredOptions.filter((option) => !loadedIds.has(option.id)), + filteredAllOptions.filter((option) => !loadedIds.has(option.id)), modelsStore.favoriteModelIds ) ); @@ -108,7 +172,26 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele (option) => !modelsStore.favoriteModelIds.has(option.model) && !loadedIds.has(option.id) ) ); - const groupedFilteredOptions = $derived(groupModelOptions(sectionOptions)); + const serverSections = $derived( + groupServerOptions( + sectionOptions, + remoteServers, + // a drill-in or a search reaches every model, the sections stay short + serverViewId || searchTerm ? Infinity : REMOTE_SERVER_MODEL_LIMIT, + modelsStore.recentModelIds + ) + ); + const groupedFilteredOptions = $derived.by(() => { + if (isServerView) { + const sections = serverSections.filter((section) => section.serverId === serverViewId); + + return { ...EMPTY_GROUPS, servers: sections }; + } + + const local = groupModelOptions(sectionOptions.filter(isLocalOption)); + + return { ...local, servers: serverSections }; + }); const isEmpty = $derived( filteredOptions.length === 0 && favoriteItems.length === 0 && loadedItems.length === 0 ); @@ -123,15 +206,18 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele function handleOpenChange(open: boolean) { if (loading || updating) return; - // a single-model desktop server has no list: the trigger opens the manager - // instead. a phone has no manager, so its drawer opens with the one model - if (!isRouter && !deviceStore.isMobile) { + // a lone llama.cpp server without a router has no list: the trigger opens + // the manager instead. router mode and external servers both expose a + // selectable list, so their role does not decide here, and a phone has no + // manager at all: its drawer opens either way + if (!isMultiModel && !deviceStore.isMobile) { if (open) uiStore.openModelsManager(); return; } searchTerm = ''; + serverViewId = null; if (open && isRouter) { modelsStore.props.fetchModalitiesForLoadedModels(); @@ -140,15 +226,37 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele opts.onOpenChange?.(open); } + /** + * Switch the rendered view. Views are display only: the server that serves + * requests follows the selected model, not the view. + */ + /** Drill into one remote server's full model list. */ + function openServer(serverId: string) { + serverViewId = serverId; + searchTerm = ''; + } + + function closeServer() { + serverViewId = null; + searchTerm = ''; + } + + /** Refresh a server's models, e.g. right after it was added. */ + async function showServerModels(serverId: string): Promise { + await serversModelsStore.ensureLoaded(serverId); + } + async function handleSelect(modelId: string) { - const option = options.find((opt) => opt.id === modelId); + // favorites live above the tabs and may belong to another server, so the + // lookup spans every enabled server + const option = allOptions.find((opt) => opt.id === modelId); if (!option) return; let shouldCloseMenu = true; if (onModelChange) { - const result = await onModelChange(option.id, option.model); + const result = await onModelChange(rawModelId(option.id), option.model, option.serverId); if (result === false) { shouldCloseMenu = false; @@ -169,9 +277,11 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele // only the built-in server loads on request, and only in router mode; a model // that is already loading is only selected, not loaded a second time + const canLoadHere = option.serverId === LOCAL_BACKEND_ID && isRouter; + if ( !onModelChange && - isRouter && + canLoadHere && !modelsStore.isModelLoaded(option.model) && !modelsStore.isModelLoading(option.model) ) { @@ -210,11 +320,11 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele }; } - return options.find((option) => option.model === currentModel); + return allOptions.find((option) => option.model === currentModel); } if (activeId) { - return options.find((option) => option.id === activeId); + return allOptions.find((option) => option.id === activeId); } return undefined; @@ -225,6 +335,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele return activeId; }, + closeServer, + get emptyMessage() { return emptyMessage; }, @@ -267,13 +379,17 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele }, get isMultiModel() { - return isRouter; + return isMultiModel; }, get isRouter() { return isRouter; }, + get isServerView() { + return isServerView; + }, + get loadedItems() { return loadedItems; }, @@ -282,6 +398,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele return loading; }, + openServer, + get options() { return options; }, @@ -298,6 +416,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele searchTerm = value; }, + showServerModels, + get updating() { return updating; } diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts index a99eaa39640b..166507492bc3 100644 --- a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts @@ -17,9 +17,10 @@ import { serverStore, settingsStore } from '#lib/stores/index.ts'; +import { apiUrl } from '#lib/utils/api-base.ts'; import { getAuthHeaders } from '#lib/utils/api-headers.ts'; import { describeMicrophoneError } from '#lib/utils/audio-recording.ts'; -import { withBase } from '#lib/utils/base-path.ts'; +import { getServer, isRemoteServer } from '#lib/utils/index.ts'; import { resolveVoiceMode } from '#lib/utils/voice-mode.ts'; import { PauseDetector, @@ -73,11 +74,8 @@ function serverContext(): { context: VoiceServerContext; realtime: boolean } { const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization'); return { - context: { - getHeaders: () => getAuthHeaders(), - resolveUrl: (path) => withBase(path.replace(/^\./, '')) - }, - realtime: !hasAuth + context: { getHeaders: () => getAuthHeaders(), resolveUrl: (path) => apiUrl(path) }, + realtime: !isRemoteServer(getServer()) && !hasAuth }; } @@ -98,7 +96,7 @@ export function useVoiceConversation() { void modelsStore.props.cacheVersion; const loadedAudioModel = isRouter - ? (modelsStore.models.find( + ? (modelsStore.activeModels.find( (option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model) )?.model ?? null) : null; @@ -115,7 +113,7 @@ export function useVoiceConversation() { }); // the chat model reads its own replies when it can speak, else another model that can const speechModel = $derived.by(() => { - const options = modelsStore.models; + const options = modelsStore.activeModels; const chat = options.find((option) => option.model === chatModel); if (chat?.speaks) return chat.model; diff --git a/ui/apps/chat/src/lib/services/chat-wire.ts b/ui/apps/chat/src/lib/services/chat-wire.ts new file mode 100644 index 000000000000..f3e48da72e97 --- /dev/null +++ b/ui/apps/chat/src/lib/services/chat-wire.ts @@ -0,0 +1,119 @@ +/** + * llama.cpp chat wire format: turns the streamed chunks of a chat completion + * into canonical events, and reads back the model a payload reports. + * + * llama-server answers with `timings` and `prompt_progress` of its own, so the + * client keeps no timing fallback and asks for no usage chunk. + */ + +import type { + ApiChatCompletionStreamChunk, + ApiChatCompletionToolCallDelta, + ApiChatCompletionUsage +} from '#lib/types/api.js'; +import type { ChatMessagePromptProgress, ChatMessageTimings } from '#lib/types/chat.js'; + +/** One canonical delta decoded from a server's stream payload. */ +export type ChatStreamEvent = + | { type: 'done' } + | { type: 'error'; message: string } + | { type: 'finish_reason'; reason: string } + | { type: 'id'; id: string } + | { type: 'model'; model: string } + | { type: 'prompt_progress'; progress: ChatMessagePromptProgress } + | { type: 'text'; text: string } + | { type: 'thinking'; text: string } + | { type: 'timings'; timings: ChatMessageTimings; promptProgress?: ChatMessagePromptProgress } + | { type: 'tool_calls'; deltas: ApiChatCompletionToolCallDelta[] } + | { type: 'usage'; usage: ApiChatCompletionUsage }; + +/** Decodes one stream's payloads. Create one per request. */ +export interface ChatStreamReader { + /** Canonical deltas for one parsed SSE payload. */ + readChunk(payload: unknown): ChatStreamEvent[]; +} + +/** + * Model name a payload reports. Streaming chunks carry it on the delta, final + * responses on the message, and some gateways on metadata or the choice itself. + */ +export function extractModelName(data: unknown): string | undefined { + const asRecord = (value: unknown): Record | undefined => + typeof value === 'object' && value !== null ? (value as Record) : undefined; + const getTrimmedString = (value: unknown): string | undefined => + typeof value === 'string' && value.trim() ? value.trim() : undefined; + const root = asRecord(data); + + if (!root) return undefined; + + const rootModel = getTrimmedString(root.model); + + if (rootModel) return rootModel; + + const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; + + if (!firstChoice) return undefined; + + const metadataModel = getTrimmedString(asRecord(firstChoice.metadata)?.model); + + if (metadataModel) return metadataModel; + + const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); + + if (deltaModel) return deltaModel; + + const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); + + if (messageModel) return messageModel; + + return getTrimmedString(firstChoice.model); +} + +function readChunk(payload: unknown): ChatStreamEvent[] { + if (!payload || typeof payload !== 'object') return []; + + const chunk = payload as ApiChatCompletionStreamChunk; + const events: ChatStreamEvent[] = []; + const model = extractModelName(chunk); + + if (chunk.id) events.push({ id: chunk.id, type: 'id' }); + + if (model) events.push({ model, type: 'model' }); + + if (chunk.prompt_progress) { + events.push({ progress: chunk.prompt_progress, type: 'prompt_progress' }); + } + + if (chunk.timings) { + events.push({ + promptProgress: chunk.prompt_progress, + timings: chunk.timings, + type: 'timings' + }); + } + + if (chunk.usage) events.push({ type: 'usage', usage: chunk.usage }); + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (choice?.finish_reason) events.push({ reason: choice.finish_reason, type: 'finish_reason' }); + + if (!delta) return events; + + if (delta.content) events.push({ text: delta.content, type: 'text' }); + + // endpoints disagree on the reasoning field name; take the first one set + const reasoning = delta.reasoning_content ?? delta.reasoning ?? delta.reasoning_text; + + if (reasoning) events.push({ text: reasoning, type: 'thinking' }); + + if (delta.tool_calls) events.push({ deltas: delta.tool_calls, type: 'tool_calls' }); + + return events; +} + +/** New streaming reader for one request. */ +export function createChatStreamReader(): ChatStreamReader { + return { readChunk }; +} diff --git a/ui/apps/chat/src/lib/services/chat.service.ts b/ui/apps/chat/src/lib/services/chat.service.ts index 38c7eba144b1..98f3a789cfbb 100644 --- a/ui/apps/chat/src/lib/services/chat.service.ts +++ b/ui/apps/chat/src/lib/services/chat.service.ts @@ -31,6 +31,7 @@ import { ReasoningFormat, StreamConnectionState } from '#lib/enums/index.ts'; +import { createChatStreamReader, extractModelName } from '#lib/services/chat-wire.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; import type { @@ -45,6 +46,7 @@ import type { } from '#lib/types/index.ts'; import type { OpenAIToolDefinition } from '#lib/types/mcp.js'; import { isAbortError } from '#lib/utils/abort.ts'; +import { apiChatUrl, apiUrl } from '#lib/utils/api-base.ts'; import { ApiError } from '#lib/utils/api-fetch.ts'; import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts'; import { formatAttachmentText } from '#lib/utils/formatters.ts'; @@ -97,7 +99,7 @@ export class ChatService { static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { try { const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; - const res = await fetch(url, { signal }); + const res = await fetch(apiUrl(url), { signal }); if (!res.ok) return true; @@ -518,7 +520,10 @@ export class ChatService { let idEmitted = false; let toolCallIndexOffset = 0; let hasOpenToolCallBatch = false; + let firstTokenAt: number | null = null; + let usage: ApiChatCompletionUsage | undefined; + const streamReader = createChatStreamReader(); const finalizeOpenToolCallBatch = () => { if (!hasOpenToolCallBatch) { return; @@ -558,6 +563,9 @@ export class ChatService { onToolCallChunk?.(serializedToolCalls); } }; + const markToken = () => { + firstTokenAt ??= Date.now(); + }; const onVisibilityChange = () => { if (typeof document === 'undefined') return; @@ -659,59 +667,94 @@ export class ChatService { continue; } + let parsed: unknown; + try { - const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); - const choice = parsed.choices?.[0]; + parsed = JSON.parse(data); + } catch (parseError) { + console.error('Error parsing JSON chunk:', parseError); - if (choice?.finish_reason) finishReason = choice.finish_reason; + continue; + } - const content = choice?.delta?.content; - const reasoningContent = choice?.delta?.reasoning_content; - const toolCalls = choice?.delta?.tool_calls; - const timings = parsed.timings; - const promptProgress = parsed.prompt_progress; - const chunkModel = ChatService.extractModelName(parsed); + for (const event of streamReader.readChunk(parsed)) { + switch (event.type) { + case 'done': + streamFinished = true; - if (chunkModel && !modelEmitted) { - modelEmitted = true; - onModel?.(chunkModel); - } + break; - if (parsed.id && !idEmitted) { - idEmitted = true; - onCompletionId?.(parsed.id); - } + case 'error': + throw new Error(event.message); - if (promptProgress) { - ChatService.notifyTimings(undefined, promptProgress, onTimings); - } + case 'finish_reason': + finishReason = event.reason; - if (timings) { - ChatService.notifyTimings(timings, promptProgress, onTimings); - lastTimings = timings; - } + break; - if (content) { - finalizeOpenToolCallBatch(); - aggregatedContent += content; + case 'id': + if (!idEmitted) { + idEmitted = true; + onCompletionId?.(event.id); + } - if (!abortSignal?.aborted) { - onChunk?.(content); - } - } + break; - if (reasoningContent) { - finalizeOpenToolCallBatch(); - fullReasoningContent += reasoningContent; + case 'model': + if (!modelEmitted) { + modelEmitted = true; + onModel?.(event.model); + } - if (!abortSignal?.aborted) { - onReasoningChunk?.(reasoningContent); - } - } + break; + + case 'prompt_progress': + ChatService.notifyTimings(undefined, event.progress, onTimings); + + break; + + case 'text': + finalizeOpenToolCallBatch(); + aggregatedContent += event.text; + + if (!abortSignal?.aborted) { + onChunk?.(event.text); + } + + markToken(); + + break; + + case 'thinking': + finalizeOpenToolCallBatch(); + fullReasoningContent += event.text; + + if (!abortSignal?.aborted) { + onReasoningChunk?.(event.text); + } + + markToken(); + + break; + + case 'timings': + ChatService.notifyTimings(event.timings, event.promptProgress, onTimings); + lastTimings = event.timings; + + break; + + case 'tool_calls': + processToolCallDelta(event.deltas); + + break; + + case 'usage': + // servers may split usage across chunks (some report input + // tokens on message_start and output tokens on message_delta) + usage = { ...usage, ...event.usage }; - processToolCallDelta(toolCalls); - } catch (e) { - console.error('Error parsing JSON chunk:', e); + break; + } } } } @@ -822,7 +865,7 @@ export class ChatService { * conv::model identity when a model was bound at POST time. */ static async lookupStreamSessions(conversationIds: string[]): Promise { - const resp = await fetch(API_STREAM.LOOKUP, { + const resp = await fetch(apiUrl(API_STREAM.LOOKUP), { body: JSON.stringify({ conversation_ids: conversationIds }), headers: getJsonHeaders(), method: 'POST' @@ -919,7 +962,7 @@ export class ChatService { } try { - const response = await fetch(API_CHAT.COMPLETIONS, { + const response = await fetch(apiChatUrl(), { body: JSON.stringify(requestBody), headers: getJsonHeaders(), method: 'POST', @@ -1297,7 +1340,7 @@ export class ChatService { // tag streaming requests with the conversation id, this single header is the opt in for the // server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit - // model the ::model suffix keeps the per model session distinct + // model the ::model suffix keeps the per model session distinct. if (stream && conversationId) { headers[HEADERS.X_CONVERSATION_ID_HEADER] = streamIdentity(conversationId, options.model); // persist the pending stream before the fetch: a reload during the model load or @@ -1305,7 +1348,7 @@ export class ChatService { ChatService.saveStreamState(conversationId, 0, options.model ?? null); } - const response = await fetch(API_CHAT.COMPLETIONS, { + const response = await fetch(apiChatUrl(), { body: JSON.stringify(requestBody), headers, method: 'POST', @@ -1417,7 +1460,7 @@ export class ChatService { if (model) body.model = model; try { - const res = await fetch(API_CHAT.CONTROL, { + const res = await fetch(apiUrl(API_CHAT.CONTROL), { body: JSON.stringify(body), headers: getJsonHeaders(), method: 'POST' @@ -1448,62 +1491,7 @@ export class ChatService { const query = `${STREAM_QUERY_PARAMS.CONV_ID}=${encodeURIComponent(streamId)}`; const offset = from === undefined ? '' : `&${STREAM_QUERY_PARAMS.FROM}=${from}`; - return `${API_STREAM.BASE}?${query}${offset}`; - } - - /** - * Extracts model name from Chat Completions API response data. - * Handles various response formats including streaming chunks and final responses. - * - * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name - * in the response. We override it with the actual model name from serverStore. - * - * @param data - Raw response data from the Chat Completions API - * @returns Model name string if found, undefined otherwise - * @private - */ - private static extractModelName(data: unknown): string | undefined { - const asRecord = (value: unknown): Record | undefined => { - return typeof value === 'object' && value !== null - ? (value as Record) - : undefined; - }; - const getTrimmedString = (value: unknown): string | undefined => { - return typeof value === 'string' && value.trim() ? value.trim() : undefined; - }; - const root = asRecord(data); - - if (!root) return undefined; - - // 1) root (some implementations provide `model` at the top level) - const rootModel = getTrimmedString(root.model); - - if (rootModel) { - return rootModel; - } - - // 2) streaming choice (delta) or final response (message) - const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; - - if (!firstChoice) { - return undefined; - } - - // priority: delta.model (first chunk) else message.model (final response) - const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); - - if (deltaModel) { - return deltaModel; - } - - const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); - - if (messageModel) { - return messageModel; - } - - // avoid guessing from non-standard locations (metadata, etc.) - return undefined; + return apiUrl(`${API_STREAM.BASE}?${query}${offset}`); } /** @@ -1539,7 +1527,7 @@ export class ChatService { } const data: ApiChatCompletionResponse = JSON.parse(responseText); - const responseModel = ChatService.extractModelName(data); + const responseModel = extractModelName(data); if (responseModel) { onModel?.(responseModel); diff --git a/ui/apps/chat/src/lib/services/huggingface.service.ts b/ui/apps/chat/src/lib/services/huggingface.service.ts index dd5e67820085..6eca16603ee9 100644 --- a/ui/apps/chat/src/lib/services/huggingface.service.ts +++ b/ui/apps/chat/src/lib/services/huggingface.service.ts @@ -404,7 +404,7 @@ export class HuggingFaceService { * exist, so callers should provide a fallback. */ static getAvatarUrl(author: string): string { - // OpenRouter-style model ids prefix the provider with a tilde + // OpenRouter-style model ids prefix the server with a tilde // (`~openai/gpt-...`); the avatars endpoint only resolves bare names return `${HF_AVATARS_URL}${PATH_SEPARATOR}${author.replace(/^~/, '')}`; } @@ -534,7 +534,7 @@ export class HuggingFaceService { if (!response.ok) throw new Error(`Failed to fetch model details: ${response.status}`); - // the hub answers with a list of provider entries for one repo; the gguf and + // the hub answers with a list of server entries for one repo; the gguf and // card data sit on the entry that carries them const payload = (await response.json()) as HfModelDetailInfo | HfModelDetailInfo[]; const entries = Array.isArray(payload) ? payload : [payload]; diff --git a/ui/apps/chat/src/lib/services/index.ts b/ui/apps/chat/src/lib/services/index.ts index 04a63ad898de..c6d7a25c3433 100644 --- a/ui/apps/chat/src/lib/services/index.ts +++ b/ui/apps/chat/src/lib/services/index.ts @@ -19,6 +19,14 @@ * */ +/** + * **ServersService** - Server connectivity checks + * + * Probes an external server's models endpoint to validate its URL and + * credentials before it is saved. Stateless. + */ +export { ServersService } from './servers.service'; + /** * **ChatService** - Chat Completions API communication layer * diff --git a/ui/apps/chat/src/lib/services/migration.service.ts b/ui/apps/chat/src/lib/services/migration.service.ts index 68c798db3187..c3c6132c1811 100644 --- a/ui/apps/chat/src/lib/services/migration.service.ts +++ b/ui/apps/chat/src/lib/services/migration.service.ts @@ -18,6 +18,7 @@ import { IDXDB_VERSION, LEGACY_AGENTIC_REGEX, LEGACY_REASONING_TAGS, + LEGACY_TUNNEL_SERVERS_LOCALSTORAGE_KEY, NEW_TO_DEPRECATED_MAP, SEARCH_CHATS_TOOL_KEY, SETTINGS_KEYS, @@ -27,6 +28,12 @@ import { import { BooleanString, MessageRole } from '#lib/enums/index.ts'; import type { McpServerOverride } from '#lib/types/database.js'; import { withSearchChatsDisabled } from '#lib/utils/search-chats.ts'; +import { + createRemoteServer, + normalizeShareCode, + parseServersSettings +} from '#lib/utils/servers.ts'; +import { uuid } from '#lib/utils/uuid.ts'; import Dexie from 'dexie'; // Types @@ -867,6 +874,64 @@ const searchChatsDefaultDisabledMigration: Migration = { } } }; +const TUNNEL_SERVERS_LEGACY_KEY = LEGACY_TUNNEL_SERVERS_LOCALSTORAGE_KEY; +/* A migration runs once per browser, so its id is written in stone: the name moved, the id did not. */ +const REMOTE_SERVERS_MIGRATION_ID = 'remote-providers-v1'; +/** + * Fold the `llama_servers` registry the first remote-access build kept in its own + * localStorage key into the settings-backed server list, as remote servers. + * + * The tunnel build stored exactly one shape - `{ type: 'tunnel', code }` - and a + * code it cannot validate is worthless, so unusable entries are dropped. Codes + * are credentials, so the legacy key is left in place: a user who downgrades + * still has what they pasted. + */ +const remoteServersMigration: Migration = { + description: 'Move tunnel servers onto the server list as remote servers', + id: REMOTE_SERVERS_MIGRATION_ID, + + async run(): Promise { + const raw = localStorage.getItem(TUNNEL_SERVERS_LEGACY_KEY); + + if (!raw) return; + + let servers: unknown; + + try { + servers = JSON.parse(raw); + } catch { + return; + } + + if (!Array.isArray(servers)) return; + + const migrated = servers.flatMap((server) => { + const entry = server as { code?: unknown; type?: unknown }; + + if (entry?.type !== 'tunnel' || typeof entry.code !== 'string') return []; + + const code = normalizeShareCode(entry.code); + + return code ? [createRemoteServer(code, uuid(), 'Remote server')] : []; + }); + + if (migrated.length === 0) return; + + const configRaw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY); + const config = configRaw ? JSON.parse(configRaw) : {}; + const kept = parseServersSettings(config[SETTINGS_KEYS.SERVERS]); + const codes = new Set(kept.map((server) => server.code)); + const added = migrated.filter((server) => !codes.has(server.code)); + + if (added.length === 0) return; + + config[SETTINGS_KEYS.SERVERS] = JSON.stringify([...kept, ...added]); + localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(config)); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) + console.log(`[Migration] Remote servers: moved ${added.length} tunnel servers`); + } +}; const migrations: Migration[] = [ localStorageMigration, idxdbMigration, @@ -879,7 +944,8 @@ const migrations: Migration[] = [ renderKeysMigration, modelOrgNameKeyMigration, mcpServerOverridesToToolPolicyMigration, - searchChatsDefaultDisabledMigration + searchChatsDefaultDisabledMigration, + remoteServersMigration ]; export const MigrationService = { diff --git a/ui/apps/chat/src/lib/services/models.service.ts b/ui/apps/chat/src/lib/services/models.service.ts index 87abe1a94245..9bda05679b7c 100644 --- a/ui/apps/chat/src/lib/services/models.service.ts +++ b/ui/apps/chat/src/lib/services/models.service.ts @@ -6,23 +6,49 @@ * modelsStore and its status manager. */ -import { API_MODELS, MODEL_ID, type ModelSidecar } from '#lib/constants/index.ts'; +import { + API_MODELS, + LOCAL_BACKEND_ID, + MODEL_ID, + type ModelSidecar, + SIDECAR_TOKENS +} from '#lib/constants/index.ts'; import { ServerModelStatus } from '#lib/enums/index.ts'; -import type { ModelSidecarBadge, ParsedModelId } from '#lib/types/models.js'; +import type { ModelSidecarBadge, ModelSidecarFile, ParsedModelId } from '#lib/types/models.js'; import { getAuthHeaders } from '#lib/utils/api-headers.ts'; -import { withBase } from '#lib/utils/base-path.ts'; import { apiDelete, apiFetch, + apiModelsUrl, apiPost, + apiUrl, extractSseDataPayload, isAuxSidecar, normalizeModelName, repoOf, sidecarFromFileToken, + sidecarFromTag, splitSseRecords } from '#lib/utils/index.ts'; +/** Sidecar token a file name carries, for the forms parsing an id as a model misses. */ +function sidecarTokenInFilename(modelId: string): ModelSidecar | null { + // the token can sit after a colon (`org/model:mtp`), a dash or an underscore + const name = modelId.toLowerCase(); + const match = SIDECAR_TOKENS.find((token) => + new RegExp(`(^|[-_:])${token}([-_.:]|$)`).test(name) + ); + + return (match as ModelSidecar | undefined) ?? null; +} + +/** Parameter label a parsed id reports, e.g. `35B-A3B`. */ +function paramsLabel(parsed: ParsedModelId): string | null { + if (!parsed.params) return null; + + return `${parsed.params}${parsed.activatedParams ? `-${parsed.activatedParams}` : ''}`; +} + export class ModelsService { private static readonly SSE_RECONNECT_MS = 1000; @@ -122,6 +148,10 @@ export class ModelsService { return model.status.value === ServerModelStatus.LOADED; } + static isModelLoading(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADING; + } + /** * * @@ -130,8 +160,24 @@ export class ModelsService { * */ - static isModelLoading(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADING; + /** + * True when a router entry id marks a downloaded sidecar file, e.g. + * `org/model:Q4_0-mtp` or `org/model:mmproj`, not a loadable model. + */ + static isSidecarEntry(modelId: string): boolean { + const idx = modelId.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR); + + if (idx !== MODEL_ID.NOT_FOUND && sidecarFromTag(modelId.slice(idx + 1))) return true; + + // the router also lists projector and draft files by filename, e.g. + // `org/model-mmproj-F16.gguf` or `mmproj-model.gguf` + const name = modelId.split('/').pop() ?? modelId; + + return ( + MODEL_ID.SIDECAR_INFIX_REGEX.test(name) || + MODEL_ID.SIDECAR_PREFIX_REGEX.test(name) || + MODEL_ID.SIDECAR_SUFFIX_REGEX.test(name) + ); } /** @@ -141,7 +187,7 @@ export class ModelsService { * @returns List of available models with basic metadata */ static async list(): Promise { - return apiFetch(API_MODELS.LIST); + return apiFetch(apiModelsUrl()); } /** @@ -151,16 +197,21 @@ export class ModelsService { * * @param modelId - Model identifier to load * @param extraArgs - Optional additional arguments to pass to the model instance + * @param serverId - Server serving the model; the active one when omitted * @returns Load response from the server */ - static async load(modelId: string, extraArgs?: string[]): Promise { + static async load( + modelId: string, + extraArgs?: string[], + serverId?: string + ): Promise { const payload: { model: string; extra_args?: string[] } = { model: modelId }; if (extraArgs && extraArgs.length > 0) { payload.extra_args = extraArgs; } - return apiPost(API_MODELS.LOAD, payload); + return apiPost(API_MODELS.LOAD, payload, { serverId }); } /** @@ -334,16 +385,57 @@ export class ModelsService { return result; } + /** + * Sidecar files a listing carries as their own entries, keyed by the repo + * they belong to. The router lists a downloaded sidecar as a model of its + * own, so this keeps the pairing that the model list itself is filtered to + * drop. + */ + static sidecarFilesByRepo(response: ApiModelsListResponse): Record { + const byRepo: Record = {}; + + for (const entry of response.data ?? []) { + const parsed = ModelsService.parseModelId(entry.id); + const sidecar = parsed.sidecar ?? sidecarTokenInFilename(entry.id); + + if (!sidecar || isAuxSidecar(sidecar)) continue; + + // the repo is the id with its quant tag and sidecar token taken off; naming + // parts of the id are not touched, a model name may carry `-4b` for instance + const model = entry.id + .split(MODEL_ID.QUANTIZATION_SEPARATOR)[0] + .replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, '') + .replace(new RegExp(`[-_ ]?${sidecar}([-_ ]?draft)?$`, 'i'), ''); + + if (!model) continue; + + const files = (byRepo[model] ??= []); + + if (files.some((file) => file.kind === sidecar)) continue; + + files.push({ + id: entry.id, + kind: sidecar, + model, + params: paramsLabel(parsed), + quant: parsed.quantization + }); + } + + return byRepo; + } + /** * Unload a model (ROUTER mode only). * Sends POST request to `/models/unload`. Note: the endpoint returns success * before unloading completes — use polling to await actual unload status. * * @param modelId - Model identifier to unload + * @param serverId - Server serving the model; the active one when omitted * @returns Unload response from the server */ - static async unload(modelId: string): Promise { - return apiPost(API_MODELS.UNLOAD, { model: modelId }); + static async unload(modelId: string, serverId?: string): Promise { + return apiPost(API_MODELS.UNLOAD, { model: modelId }, { serverId }); } /** @@ -354,14 +446,18 @@ export class ModelsService { */ static async watchModelEvents( signal: AbortSignal, - onEvent: (event: ApiModelsSseEvent) => void + onEvent: (event: ApiModelsSseEvent) => void, + serverId?: string ): Promise { + // a llama.cpp server serves its feed at its own origin, so the request is + // pinned to the server it is about + const id = serverId ?? LOCAL_BACKEND_ID; const decoder = new TextDecoder(); while (!signal.aborted) { try { - const response = await fetch(withBase(API_MODELS.SSE), { - headers: getAuthHeaders(), + const response = await fetch(apiUrl(API_MODELS.SSE, id), { + headers: getAuthHeaders(id), signal }); diff --git a/ui/apps/chat/src/lib/services/props.service.ts b/ui/apps/chat/src/lib/services/props.service.ts index 4a2f4e884799..ae2443f5c180 100644 --- a/ui/apps/chat/src/lib/services/props.service.ts +++ b/ui/apps/chat/src/lib/services/props.service.ts @@ -15,17 +15,21 @@ export class PropsService { * In ROUTER mode, returns server-wide settings without model-specific modalities. * * @param autoload - If false, prevents automatic model loading (default: false) + * @param serverId - Server to ask; defaults to the active one. * @returns Server properties including default generation settings and capabilities * @throws {Error} If the request fails or returns invalid data */ - static async fetch(autoload = false): Promise { + static async fetch(autoload = false, serverId?: string): Promise { const params: Record = {}; if (!autoload) { params.autoload = 'false'; } - return apiFetchWithParams('./props', params, { authOnly: true }); + return apiFetchWithParams('./props', params, { + authOnly: true, + serverId + }); } /** diff --git a/ui/apps/chat/src/lib/services/servers.service.ts b/ui/apps/chat/src/lib/services/servers.service.ts new file mode 100644 index 000000000000..6bb655f25290 --- /dev/null +++ b/ui/apps/chat/src/lib/services/servers.service.ts @@ -0,0 +1,217 @@ +/** + * ServersService - Stateless server connectivity checks and model listing + * + * Probes a server's models endpoint to validate its URL and credentials, and + * normalizes the response into the UI model shape. No reactive state; + * consumed by the servers settings UI and the per-server model cache. + */ + +import { API_MODELS, LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; +import { ModelsService } from '#lib/services/models.service.ts'; +import type { ApiModelsListResponse, ModelOption, Server } from '#lib/types/index.ts'; +import { isAbortError } from '#lib/utils/abort.ts'; +import { apiUrl } from '#lib/utils/api-base.ts'; +import { getAuthHeadersForServer } from '#lib/utils/api-headers.ts'; +import { isLlamaCppProps } from '#lib/utils/server-props.ts'; +import { + isLocalServer, + isReachableServer, + isRemoteServer, + readModelContextLength, + serverModelsUrl, + serverOrigin +} from '#lib/utils/servers.ts'; + +/** What a check reports about an endpoint that is not a llama.cpp server. */ +const NOT_LLAMA_CPP = 'This endpoint is not a llama.cpp server: /props answered something else'; + +/** Models returned by a server, plus the failure detail when the call fails. */ +export interface ServerModelsResult { + error?: string; + models: ModelOption[]; + ok: boolean; + status: number | null; + /** Untouched list payload of the local server, kept so the router rows and their load statuses can be rebuilt without asking again. */ + raw?: ApiModelsListResponse; +} + +/** What probing a server's endpoint said about it. */ +export interface ServerProbe { + /** The endpoint refused the request for want of a key, so it wants one. */ + authRequired: boolean; + /** + * Whether the endpoint answered /props like a llama.cpp server. Null while it + * did not answer at all: a key-protected or hidden /props decides nothing. + */ + llamaCpp: boolean | null; +} + +/** Outcome of a server connectivity check. */ +export interface ServerTestResult { + error?: string; + modelCount?: number; + ok: boolean; + status: number | null; +} + +export class ServersService { + /** + * List the models a server exposes on its models endpoint. + * + * @param server - Server to query. Does not need to be registered yet. + * @param signal - Optional abort signal for a cancelled request. + */ + static async listModels(server: Server, signal?: AbortSignal): Promise { + if (!isReachableServer(server)) { + return { + error: isRemoteServer(server) ? 'Access code is required' : 'Server URL is required', + models: [], + ok: false, + status: null + }; + } + + // the local server has no origin of its own; its models endpoint is base + // relative, and a remote one is answered over its tunnel + const url = isLocalServer(server) + ? apiUrl(API_MODELS.LIST, LOCAL_BACKEND_ID) + : serverModelsUrl(server); + + try { + const response = await fetch(url, { + headers: getAuthHeadersForServer(server), + signal + }); + + if (!response.ok) { + return { + error: await describeFailure(response), + models: [], + ok: false, + status: response.status + }; + } + + const body = (await response.json()) as { data?: unknown }; + const entries = Array.isArray(body?.data) ? body.data : []; + const models = entries.flatMap((entry) => normalizeServerModel(entry)); + // the local rows carry load status, external ones carry the context size + const raw = body as ApiModelsListResponse; + + return { models, ok: true, raw, status: response.status }; + } catch (error) { + if (isAbortError(error)) { + return { models: [], ok: false, status: null }; + } + + return { + error: error instanceof Error ? error.message : String(error), + models: [], + ok: false, + status: null + }; + } + } + + /** + * Ask a server whether it wants an API key, and whether it is a llama.cpp + * server at all. A llama-server guards /props with the key it was started + * with, so a 401 there means the form needs one; an endpoint that answers + * something else there is not one this UI can drive. + */ + static async probe(server: Server): Promise { + // a remote server has no URL to ask about: its code is tested by opening + // the tunnel, which is what the form's connection test does + if (isRemoteServer(server)) return { authRequired: false, llamaCpp: null }; + + const base = serverOrigin(server).replace(/\/+$/, ''); + + if (!base) return { authRequired: false, llamaCpp: null }; + + try { + const response = await fetch(`${base}/props`, { + headers: getAuthHeadersForServer(server), + signal: AbortSignal.timeout(5000) + }); + + if (response.status === 401) return { authRequired: true, llamaCpp: null }; + + if (!response.ok) return { authRequired: false, llamaCpp: null }; + + return { authRequired: false, llamaCpp: isLlamaCppProps(await response.json()) }; + } catch { + return { authRequired: false, llamaCpp: null }; + } + } + + /** + * Check that a server answers on its models endpoint, and that it is a + * llama.cpp server. + * + * @param server - Server to probe. Does not need to be registered yet. + * @param signal - Optional abort signal for a cancelled test. + */ + static async test(server: Server, signal?: AbortSignal): Promise { + const result = await ServersService.listModels(server, signal); + + if (result.ok) { + const probe = await ServersService.probe(server); + + // only a definite answer counts: an endpoint whose /props cannot be read + // is reported by the models listing above, not accused here + if (probe.llamaCpp === false) { + return { error: NOT_LLAMA_CPP, ok: false, status: result.status }; + } + } + + return { + error: result.error, + modelCount: result.models.length, + ok: result.ok, + status: result.status + }; + } +} + +/** Build a human-readable message from a non-OK response. */ +async function describeFailure(response: Response): Promise { + const status = `${response.status} ${response.statusText}`.trim(); + + try { + const body = (await response.json()) as { error?: { message?: string }; message?: string }; + const message = body?.error?.message ?? body?.message; + + if (message) return `${status}: ${message}`; + } catch { + // non-JSON error body, fall back to the status line + } + + return status; +} + +/** + * Normalize one entry of an OpenAI-compatible `/v1/models` response. External + * servers only guarantee an id, so that doubles as the display name. + */ +function normalizeServerModel(entry: unknown): ModelOption[] { + if (!entry || typeof entry !== 'object') return []; + + const raw = entry as Record; + const id = typeof raw.id === 'string' ? raw.id.trim() : ''; + + if (!id) return []; + + // a llama-compat server lists its projector and draft sidecars as models too + if (ModelsService.isSidecarEntry(id)) return []; + + return [ + { + capabilities: [], + contextLength: readModelContextLength(raw), + id, + model: id, + name: id, + status: raw.status as ApiModelDataEntry['status'] + } + ]; +} diff --git a/ui/apps/chat/src/lib/services/tools.service.ts b/ui/apps/chat/src/lib/services/tools.service.ts index dd401f79d34d..5d5a2a0561ec 100644 --- a/ui/apps/chat/src/lib/services/tools.service.ts +++ b/ui/apps/chat/src/lib/services/tools.service.ts @@ -5,12 +5,17 @@ * No reactive state; consumed by toolsStore. */ -import { API_TOOLS, HEADERS, HOME_TILDE, SEARCH } from '#lib/constants/index.ts'; +import { API_TOOLS, HEADERS, HOME_TILDE, LOCAL_BACKEND_ID, SEARCH } from '#lib/constants/index.ts'; import { BuiltInTool, GlobSearchType, ToolResponseField } from '#lib/enums/index.ts'; import type { GlobEntry, ServerToolInfo, ToolExecutionResult } from '#lib/types/index.ts'; import { getJsonHeaders } from '#lib/utils/api-headers.ts'; -import { withBase } from '#lib/utils/base-path.ts'; -import { apiFetch, buildCaseInsensitiveGlob, joinPath, lastPathSegment } from '#lib/utils/index.ts'; +import { + apiFetch, + apiUrl, + buildCaseInsensitiveGlob, + joinPath, + lastPathSegment +} from '#lib/utils/index.ts'; import { parseSseJsonStream, type SseJsonEvent } from '#lib/utils/sse.ts'; export class ToolsService { @@ -31,6 +36,7 @@ export class ToolsService { body: JSON.stringify({ params, tool: toolName }), headers: cwd ? { [HEADERS.X_TOOL_CWD_HEADER]: cwd } : undefined, method: 'POST', + serverId: LOCAL_BACKEND_ID, signal }); @@ -70,6 +76,7 @@ export class ToolsService { body: JSON.stringify({ params, tool: toolName }), headers: Object.keys(headers).length > 0 ? headers : undefined, method: 'POST', + serverId: LOCAL_BACKEND_ID, signal }); } @@ -80,7 +87,8 @@ export class ToolsService { * @returns Array of tool definitions in OpenAI-compatible format */ static async list(): Promise { - return apiFetch(API_TOOLS.LIST); + // pinned to the local server: server tools do not exist on other servers + return apiFetch(API_TOOLS.LIST, { serverId: LOCAL_BACKEND_ID }); } /** @@ -133,11 +141,13 @@ export class ToolsService { signal?: AbortSignal, cwd?: string ): AsyncGenerator { - const headers = getJsonHeaders(); + // pinned to the local server like the tool list and the non-streaming + // call: the tools offered to the model are the local server's + const headers = getJsonHeaders(LOCAL_BACKEND_ID); if (cwd) headers[HEADERS.X_TOOL_CWD_HEADER] = cwd; - const response = await fetch(withBase(API_TOOLS.EXECUTE), { + const response = await fetch(apiUrl(API_TOOLS.EXECUTE, LOCAL_BACKEND_ID), { body: JSON.stringify({ params, stream: true, tool: toolName }), headers, method: 'POST', diff --git a/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts b/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts index de6fc2f06954..6a5524b9a006 100644 --- a/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts +++ b/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts @@ -4,7 +4,7 @@ * Single owner of the "is this conversation doing something" state: * - `local` - this browser is piping a stream (send, server-stream attach, * or resume-wait while the owning model loads) - * - `remote` - the backend reports a running session, no local pipe yet + * - `remote` - the server reports a running session, no local pipe yet * (global snapshot on mount / visibilitychange) * * The union of both drives the sidebar spinners (`loadingConvs`); `local` @@ -21,7 +21,7 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity'; export class ChatActivityStore { /** Convs this browser is piping a stream for (send, attach, resume-wait). */ private local = new SvelteSet(); - /** Convs the backend reports as having a running session (snapshot sync). */ + /** Convs the server reports as having a running session (snapshot sync). */ private remote = new SvelteSet(); /** Convs with any activity, the union the sidebar spinners render. */ loadingConvs = $derived.by(() => { @@ -40,7 +40,7 @@ export class ChatActivityStore { private compacting = new SvelteMap(); /** - * Apply a backend snapshot of running sessions (mount / visibilitychange). + * Apply a server snapshot of running sessions (mount / visibilitychange). * Diffed so unchanged entries do not re-trigger reactivity. */ applyRemoteSnapshot(running: Iterable): void { diff --git a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts index 73c8d21d64e0..349025242a81 100644 --- a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts @@ -956,6 +956,15 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { modelOverride?: string | null, firstUserMessageContent?: string ): Promise { + // a conversation keeps the model that generated it, which can belong to + // another server; make that server active so the request is not sent to + // a server that does not serve the model + const requestedModel = modelOverride ?? getConversationModel(allMessages); + + if (requestedModel) { + await modelsStore.ensureModelServer(requestedModel); + } + // the ::model suffix in the stream identity is only for router mode, where it routes to the // owning child. in single-model mode the identity stays the bare conv id so that attach, stop // and reattach all agree, regardless of fresh send vs regenerate passing a resolved model @@ -1442,10 +1451,9 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { assistantContent: string, convId: string ): Promise { - const effectiveModel = - serverStore.isRouterMode && modelsStore.selectedModelName - ? modelsStore.selectedModelName - : undefined; + const effectiveModel = serverStore.isRouterMode + ? (modelsStore.activeModelId ?? undefined) + : undefined; const configValue = settingsStore.config; const titlePromptTemplate = typeof configValue.titleGenerationPrompt === 'string' && diff --git a/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts b/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts index dfa8e5a5d24c..49b6c7a89ffa 100644 --- a/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts +++ b/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts @@ -171,7 +171,7 @@ export class ChatStreamManager { } /** - * Resync the activity ledger's remote set from the backend. Called by the layout at mount and + * Resync the activity ledger's remote set from the server. Called by the layout at mount and * on visibilitychange, no polling. A snapshot semantic: stale entries for sessions that * finalized while the browser was elsewhere are dropped naturally. */ diff --git a/ui/apps/chat/src/lib/stores/index.ts b/ui/apps/chat/src/lib/stores/index.ts index 5b011b0fcf2d..7cf7f0a5ce51 100644 --- a/ui/apps/chat/src/lib/stores/index.ts +++ b/ui/apps/chat/src/lib/stores/index.ts @@ -38,6 +38,13 @@ export { conversationsStore } from './conversations/index.svelte'; // GROUPS (named sidebar sections for chats) export { groupsStore } from './groups.svelte'; +// SERVERS +export { serversStore } from './servers.svelte'; +export { serversModelsStore } from './serversModels.svelte'; + +// REMOTE ACCESS (WebRTC tunnels to remote servers) +export { remoteAccessStore } from './remote-access.svelte'; + // MODELS export { modelsStore } from './models/index.svelte'; diff --git a/ui/apps/chat/src/lib/stores/init.ts b/ui/apps/chat/src/lib/stores/init.ts index 532abfeb352a..c98b95f880df 100644 --- a/ui/apps/chat/src/lib/stores/init.ts +++ b/ui/apps/chat/src/lib/stores/init.ts @@ -1,8 +1,13 @@ // direct imports, not via the barrel, to avoid circular deps import { conversationsStore } from './conversations/index.svelte'; import { groupsStore } from './groups.svelte'; +import { serverStore } from './llama-server.svelte'; +import { modelsStore } from './models/index.svelte'; import { permissionsStore } from './permissions.svelte'; import { projectsStore } from './projects.svelte'; +import { remoteAccessStore } from './remote-access.svelte'; +import { serversStore } from './servers.svelte'; +import { serversModelsStore } from './serversModels.svelte'; import { settingsStore } from './settings/index.svelte'; import { tabsStore } from './tabs.svelte'; import { toolsStore } from './tools.svelte'; @@ -11,8 +16,12 @@ import { BUILT_IN_EXTENSIONS } from '#lib/extensions/builtin.ts'; import { extensions } from '#lib/extensions/registry.svelte.ts'; import { MigrationService } from '#lib/services/migration.service.ts'; import { validateApiKey } from '#lib/utils/api-key-validation.ts'; +import { isLocalServer, isRemoteServer } from '#lib/utils/servers.ts'; import { browser } from '$app/env'; +let activeTunnel: Promise | null = null; +let hydration: Promise | null = null; +let extensionsStarted: Promise | null = null; let startup: Promise | null = null; /** Ask the browser not to evict conversations and settings under storage pressure, best effort. */ @@ -26,22 +35,126 @@ async function requestPersistentStorage(): Promise { } } +/** + * Open the tunnel of the server the app opens with, when it is a remote one. + * + * Deliberately not awaited: the app paints with the server card reading + * Connecting, and its models arrive when the handshake lands. Servers that are + * merely enabled are left for the moment one of their models is picked. + * + * Runs once per session: startup asks twice - once before the local server is + * known to exist, once after - and the second ask is the same tunnel. + */ +function connectActiveServer(): void { + const active = serversStore.active; + + if (activeTunnel || !isRemoteServer(active)) return; + + activeTunnel = remoteAccessStore + .connect(active.id) + // the closed tunnel is exactly why the startup prefetch skipped this + // server, so both lists are asked for once it is open + .then(async () => { + await serversModelsStore.refresh(active.id); + await modelsStore.switchServer(); + }) + .catch(() => { + // the server card carries the failure and offers a retry + }); +} + +/** + * Read the stored state the first request needs: migrations, the settings that + * carry the API key, and the server that request goes to. + * + * Separate from {@link initStores} so a `load` can await it. The requests + * `initStores` starts belong after a load, where plain `window.fetch` is the + * intended one and SvelteKit raises no warning. + */ +export function hydrateStores(): Promise { + if (!browser) return Promise.resolve(); + + hydration ??= (async () => { + await MigrationService.runAllMigrations(); + + settingsStore.initialize(); + + // the tunnel transport goes in before anything can ask for a model list: a + // request aimed at a server's tunnel origin must never reach the server + // that happens to serve this UI + remoteAccessStore.installInterceptor(); + + serversStore.initialize(); + + connectActiveServer(); + })(); + + return hydration; +} + +/** + * Register the built-in extensions and activate the ones that are switched + * on, once. The startup does it; the load of an extension's page does too, + * as it runs before the startup and needs the page to be there. + */ +export function startExtensions(): Promise { + if (!browser) return Promise.resolve(); + + extensionsStarted ??= (async () => { + // extensions start once the settings they read are in place + await hydrateStores(); + + for (const manifest of BUILT_IN_EXTENSIONS) extensions.register(manifest); + + await extensions.activateEnabled(); + })(); + + return extensionsStarted; +} + export function initStores(): Promise { if (!browser) return Promise.resolve(); startup ??= (async () => { - await MigrationService.runAllMigrations(); + await hydrateStores(); void requestPersistentStorage(); - settingsStore.initialize(); + // the servers the page was published with, asked for alongside the rest of + // the startup rather than ahead of it, and loaded as they join the list + const presets = serversStore + .addPresetServers() + .then((ids) => Promise.all(ids.map((id) => serversModelsStore.ensureLoaded(id)))); + + // prefetch every server's model list in the background; failures are + // per-server and never block startup + void serversModelsStore.loadAll().then(() => modelsStore.warmHubDetails()); + permissionsStore.initialize(); toolsStore.initialize(); - // extensions start once the settings they read are in place - for (const manifest of BUILT_IN_EXTENSIONS) extensions.register(manifest); + // the local server state backs the installation facts and decides whether + // /tools exists at all, so probe it first and only then list the tools; + // otherwise they stay empty until the tools menu is opened + void Promise.all([serverStore.prefetchLocalState(), presets]).then(async () => { + // a build served without a local server has no local server to fall + // back on, so the active server is only settled once that is known + connectActiveServer(); - await extensions.activateEnabled(); + void toolsStore.fetchServerTools(); + + // the page asked the local server for its models before it was known to + // be missing; the server the app falls back on, such as one the page was + // published with, answers instead (a remote one does once its tunnel opens) + const active = serversStore.active; + + if (serverStore.localServerMissing && !isLocalServer(active) && !isRemoteServer(active)) { + await modelsStore.switchServer(); + await modelsStore.ensureFirstModelSelected(); + } + }); + + await startExtensions(); void versionStore.initialize(); void groupsStore.initialize(); @@ -57,10 +170,11 @@ export function initStores(): Promise { return startup; } -/** Load shared by the app's pages: start the stores, then check the API key. */ +/** Load shared by the app's pages: read the stored state, then check the API key. */ export async function loadAppRoute(fetch: typeof globalThis.fetch): Promise { // loads run before the root layout script, so the stored API key reaches - // the probe only once the settings store has read localStorage - await initStores(); + // the probe only once the settings store has read localStorage; the rest of + // the startup belongs to the layout, after this load + await hydrateStores(); await validateApiKey(fetch); } diff --git a/ui/apps/chat/src/lib/stores/llama-server.svelte.ts b/ui/apps/chat/src/lib/stores/llama-server.svelte.ts index 84968528f359..f9820d33b413 100644 --- a/ui/apps/chat/src/lib/stores/llama-server.svelte.ts +++ b/ui/apps/chat/src/lib/stores/llama-server.svelte.ts @@ -6,8 +6,10 @@ * PropsService for the /props fetch. */ +import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; import { ServerRole } from '#lib/enums/index.ts'; import { PropsService } from '#lib/services/props.service.ts'; +import { getServer } from '#lib/utils/api-base.ts'; import { ApiError } from '#lib/utils/index.ts'; const LOADING_RETRY_INTERVAL_MS = 1000; @@ -18,7 +20,16 @@ class ServerStore { props = $state(null); role = $state(null); status = $state(null); + private fetchBackendId: string | undefined; private fetchPromise: Promise | null = null; + /** + * Whether a local llama.cpp server answered. Null until the first probe + * resolves: the bundled build is served by one, the hosted PWA is not. + */ + private localAvailable: boolean | null = null; + /** Local server state kept alive while an external server is active. */ + private localState: { props: ApiLlamaCppServerProps | null; role: ServerRole | null } | null = + null; private retryTimer: ReturnType | null = null; get contextSize(): number | null { @@ -28,7 +39,12 @@ class ServerStore { } get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null { - return this.props?.default_generation_settings?.params || null; + return this.localProps?.default_generation_settings?.params || null; + } + + /** A local llama.cpp server answered, either now or earlier this session. */ + get hasLocalServer(): boolean { + return this.localAvailable === true; } get isModelMode(): boolean { @@ -39,10 +55,52 @@ class ServerStore { return this.role === ServerRole.ROUTER; } + /** + * Local server state. The local server is the base of the installation, so + * its props and role come from the mirror while another server is active + * instead of following the selected model. + */ + private get localFacts(): { props: ApiLlamaCppServerProps | null; role: ServerRole | null } { + // no resolvable server (early startup, non-browser call): props belong to + // the origin serving this UI, which is the local server + const serverId = getServer()?.id ?? LOCAL_BACKEND_ID; + + if (serverId === LOCAL_BACKEND_ID) return { props: this.props, role: this.role }; + + return this.localState ?? { props: null, role: null }; + } + + /** The local server runs as a router, independent of the active server. */ + get localIsRouter(): boolean { + return this.localFacts.role === ServerRole.ROUTER; + } + + /** Props of the local server, from the active server or the cache. */ + get localProps(): ApiLlamaCppServerProps | null { + return this.localFacts.props; + } + + /** The local server is known to be unreachable: local-only UI stays out. */ + get localServerMissing(): boolean { + return this.localAvailable === false; + } + get uiSettings(): Record | undefined { return this.props?.ui_settings ?? this.props?.webui_settings; } + /** + * Keep the local server state before switching to an external server, so + * switching back restores it instead of asking the server again. + */ + cacheLocalState(): void { + // props only exist while a llama.cpp server is active; an external to + // external switch must not overwrite the kept local state with blanks + if (!this.props) return; + + this.localState = { props: this.props, role: this.role }; + } + clear(): void { this.clearRetryTimer(); this.props = null; @@ -51,6 +109,7 @@ class ServerStore { this.loading = false; this.role = null; this.fetchPromise = null; + this.fetchBackendId = undefined; } /** @@ -59,7 +118,12 @@ class ServerStore { * splash and the chat screen every retry tick. */ async fetch({ background = false }: { background?: boolean } = {}): Promise { - if (this.fetchPromise) return this.fetchPromise; + // props and role describe one server. a fetch started for another server + // must not be reused, and its response must not commit once the active + // server has changed while it was in flight + const serverId = getServer()?.id; + + if (this.fetchPromise && this.fetchBackendId === serverId) return this.fetchPromise; this.clearRetryTimer(); @@ -73,19 +137,33 @@ class ServerStore { this.error = null; } - const fetchPromise = (async () => { + const promise = (async () => { try { const props = await PropsService.fetch(); + // the active server changed while this request was in flight + if (getServer()?.id !== serverId) return; + this.props = props; this.error = null; this.status = null; this.detectRole(props); + + if (serverId === LOCAL_BACKEND_ID) { + this.localAvailable = true; + // mirror the local facts as they arrive: a later clear() must not + // be able to wipe what the local server told us + this.localState = { props, role: this.role }; + } } catch (error: unknown) { + if (getServer()?.id !== serverId) return; + this.error = error instanceof Error ? error.message : String(error); this.status = error instanceof ApiError ? error.status : null; console.error('Error fetching server properties:', error); + if (serverId === LOCAL_BACKEND_ID) this.localAvailable = false; + if (this.status === 503) { this.scheduleRetry(); } @@ -93,13 +171,56 @@ class ServerStore { if (!background) { this.loading = false; } - - this.fetchPromise = null; } })(); - this.fetchPromise = fetchPromise; - await fetchPromise; + this.fetchPromise = promise; + this.fetchBackendId = serverId; + + // a server switch clears the in-flight handle; only the fetch that is + // still the current one may release it + void promise + .catch(() => {}) + .finally(() => { + if (this.fetchPromise === promise) { + this.fetchPromise = null; + this.fetchBackendId = undefined; + } + }); + + await promise; + } + + /** + * Load the local server state in the background at startup. The local tab + * then opens from memory instead of asking for props on the first click. + */ + async prefetchLocalState(): Promise { + if (this.localState) return; + + try { + const props = await PropsService.fetch(false, LOCAL_BACKEND_ID); + + this.localState = { + props, + role: props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL + }; + this.localAvailable = true; + } catch { + // no local server in this deployment: mark it missing so the local + // server drops out of the enabled list + this.localAvailable = false; + } + } + + /** Restore the state kept by {@link cacheLocalState}; no request is made. */ + restoreLocalState(): void { + if (!this.localState) return; + + this.props = this.localState.props; + this.role = this.localState.role; + this.error = null; + this.status = null; } private clearRetryTimer(): void { diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts index 7928defaad30..8d765ba61ef7 100644 --- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts @@ -10,24 +10,30 @@ import { FAVORITE_MODELS_LOCALSTORAGE_KEY, HIDDEN_MODELS_LOCALSTORAGE_KEY, + LOCAL_BACKEND_ID, MODEL_GROUP_OPEN_LOCALSTORAGE_KEY, MODEL_REMOVE_BLOCKED_FALLBACK, MODEL_REMOVE_BLOCKED_REASONS, RECENT_MODEL_LIMIT, - RECENT_MODELS_LOCALSTORAGE_KEY + RECENT_MODELS_LOCALSTORAGE_KEY, + SELECTED_MODEL_LOCALSTORAGE_KEY } from '#lib/constants/index.ts'; import { ReasoningEffort, ServerModelSource, ServerModelStatus } from '#lib/enums/index.ts'; import { HuggingFaceService } from '#lib/services/huggingface.service.ts'; import { ModelsService } from '#lib/services/models.service.ts'; -// direct imports between stores, not via the barrel, to avoid circular deps import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts'; import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import { type ModelPropsHost, ModelPropsManager } from '#lib/stores/models/props.svelte.ts'; import { type ModelStatusHost, ModelStatusManager } from '#lib/stores/models/status.svelte.ts'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serversStore } from '#lib/stores/servers.svelte.ts'; +import { serversModelsStore } from '#lib/stores/serversModels.svelte.ts'; import type { ApiModelDataEntry } from '#lib/types/index.ts'; import type { ParsedModelId } from '#lib/types/models.js'; import { getConversationModel } from '#lib/utils/conversation-utils.ts'; import { repoOf } from '#lib/utils/model-names.ts'; +import { qualifyModelId, rawModelId, serverIdFromModelId } from '#lib/utils/model-option-id.ts'; +import { readModelContextLength } from '#lib/utils/servers.ts'; import { browser } from '$app/env'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { toast } from 'svelte-sonner'; @@ -41,14 +47,39 @@ const MAX_GROUP_OPEN_ENTRIES = 200; */ const HUB_WARM_LIMIT = 12; +/** Selection kept from the last session, so a reload does not drop the picked model. */ +function loadStoredSelection(): { id: string; model: string | null } | null { + if (!browser) return null; + + try { + const raw = localStorage.getItem(SELECTED_MODEL_LOCALSTORAGE_KEY); + + if (!raw) return null; + + const parsed = JSON.parse(raw) as { id?: unknown; model?: unknown }; + + if (typeof parsed?.id !== 'string' || !parsed.id) return null; + + return { id: parsed.id, model: typeof parsed.model === 'string' ? parsed.model : null }; + } catch { + return null; + } +} + +const storedSelection = loadStoredSelection(); + /** Models kept out of the selector. */ function loadHiddenModels(): Set { if (!browser) return new SvelteSet(); try { const raw = localStorage.getItem(HIDDEN_MODELS_LOCALSTORAGE_KEY); + const ids = raw ? (JSON.parse(raw) as string[]) : []; - return raw ? new SvelteSet(JSON.parse(raw) as string[]) : new SvelteSet(); + // ids saved before models were qualified by server all belonged to the local one + return new SvelteSet( + ids.map((id) => (serverIdFromModelId(id) ? id : qualifyModelId(LOCAL_BACKEND_ID, id))) + ); } catch { return new SvelteSet(); } @@ -72,7 +103,7 @@ function loadGroupOpenState(): SvelteMap { } } -/** Recently used backend-qualified ids, most recent first. */ +/** Recently used server-qualified ids, most recent first. */ function loadRecentModels(): string[] { if (!browser) return []; @@ -92,16 +123,59 @@ function loadRecentModels(): string[] { } class ModelsStore implements ModelPropsHost, ModelStatusHost { + activeModels = $state([]); error = $state(null); favoriteModelIds = $state>(this.loadFavoritesFromStorage()); groupOpenState = $state>(loadGroupOpenState()); hiddenModelIds = $state>(loadHiddenModels()); loading = $state(false); - models = $state([]); + /** + * Every selectable model across enabled servers. The active server's + * models come from {@link activeModels}; the rest come from the background + * prefetch cache. Ids are server-qualified so the same model name on two + * servers stays distinct. + */ + /** + * Computed once per state change. Rows reach this through per-model props + * lookups, so a getter that rebuilt the list on every read made opening the + * selector quadratic in the size of the catalog. + */ + models = $derived.by((): ModelOption[] => { + const activeServerId = serversStore.active.id; + const merged: ModelOption[] = []; + const seen = new SvelteSet(); + const push = (option: ModelOption, serverId: string) => { + const id = qualifyModelId(serverId, rawModelId(option.id)); + + // a server can be listed twice while a switch is in flight: the rows + // of the previous server are still in activeModels + if (seen.has(id)) return; + + seen.add(id); + merged.push({ ...option, id, serverId }); + }; + + for (const option of this.activeModels) { + // keep the server an option was built for: rows from the previous + // server must not be relabelled while a switch is in flight + push(option, option.serverId ?? activeServerId); + } + + for (const server of serversStore.enabled) { + if (server.id === activeServerId) continue; + + for (const option of serversModelsStore.get(server.id).models) { + push(option, option.serverId ?? server.id); + } + } + + return merged; + }); recentModelIds = $state(loadRecentModels()); routerModels = $state([]); - selectedModelId = $state(null); - selectedModelName = $state(null); + selectedModelId = $state(storedSelection?.id ?? null); + + selectedModelName = $state(storedSelection?.model ?? null); updating = $state(false); @@ -115,6 +189,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { // Without this, ?model= URL handler races an in-progress fetch and sees an empty list. private inflightFetch: Promise | null = null; + /** A restored selection loses to the active conversation's model, a fresh pick does not. */ + private selectionFromStorage = storedSelection !== null; + /** * Model the active conversation view resolves to. Router mode: the user's * selection first, then the conversation's own model. Otherwise the single @@ -122,14 +199,20 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { */ get activeModelId(): string | null { if (!serverStore.isRouterMode) { + // external servers expose a selectable list; prefer the user's pick + const selected = this.selectedModelId + ? this.models.find((m) => m.id === this.selectedModelId) + : undefined; + + if (selected) return selected.model; + return this.models.length > 0 ? this.models[0].model : this.singleModelName; } - if (this.selectedModelId) { - const selected = this.models.find((m) => m.id === this.selectedModelId); + const picked = this.selectedModelId && !this.selectionFromStorage ? this.selectedModelId : null; + const selected = picked ? this.models.find((m) => m.id === picked) : undefined; - if (selected) return selected.model; - } + if (selected) return selected.model; const conversationModel = getConversationModel(conversationsStore.activeMessages); @@ -139,7 +222,11 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { if (model) return model.model; } - return null; + const restored = this.selectedModelId + ? this.models.find((m) => m.id === this.selectedModelId) + : undefined; + + return restored?.model ?? null; } get loadedModelIds(): string[] { @@ -192,6 +279,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { clearSelection(): void { this.selectedModelId = null; this.selectedModelName = null; + this.persistSelection(); } /** Family keys folded away under one section, for a list that restores them. */ @@ -265,7 +353,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return; } - // favorites are shared across backends, so a stored id may belong to another one + // favorites are shared across servers, so a stored id may belong to another one const favorite = this.favoriteModelIds.values().next()?.value; const favoriteOption = favorite ? availableModels.find((m) => m.id === favorite || m.model === favorite) @@ -281,6 +369,26 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { await this.selectModelById(availableModels[0].id); } + /** + * Make the server serving `modelName` active when it is not already. + * A conversation keeps the model that generated it, which can belong to a + * server other than the active one. + */ + async ensureModelServer(modelName: string): Promise { + const options = this.models.filter((model) => model.model === modelName && model.serverId); + + // the same model name can be served by several servers: stay on the active one + // when it has it, rather than switching to whichever server is listed first + if ( + options.length === 0 || + options.some((model) => model.serverId === serversStore.active.id) + ) { + return; + } + + await this.selectModelById(options[0].id); + } + /** * Fetch list of models from server and detect server role. * Also fetches modalities for MODEL mode (single model). @@ -288,7 +396,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { async fetch(force = false): Promise { if (this.inflightFetch) return this.inflightFetch; - if (this.models.length > 0 && !force) return; + if (this.activeModels.length > 0 && !force) return; this.inflightFetch = this.runFetch(); try { @@ -315,7 +423,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { this._status.syncPausedDownloads(); // keep the selector options in sync: a downloaded / deleted model shows // up here too, not only in the router model rows - this.models = this.buildModelOptions(response); + this.activeModels = this.buildModelOptions(response); this.warmHubDetails(); await this.props.fetchModalitiesForLoadedModels(); @@ -375,10 +483,26 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return null; } + /** + * Load state a model's own server reports. Only the local server has a status + * feed, so an external server answers from its own model listing. + */ getModelStatus(modelId: string): ServerModelStatus | null { + // the local feed reports the models its router serves: a same-named entry + // on a remote server must not shadow it const model = this.routerModels.find((m) => m.id === modelId); - return (model?.status?.value as ServerModelStatus) ?? null; + if (model) return (model.status?.value as ServerModelStatus) ?? null; + + // a model the local router does not serve belongs to a server, and the + // first enabled server listing it owns the state + for (const server of serversStore.enabled) { + const option = serversModelsStore.get(server.id).models.find((m) => m.model === modelId); + + if (option) return (option.status?.value as ServerModelStatus) ?? null; + } + + return null; } /** @@ -386,8 +510,13 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { * hidden by the user, so an automatic pick never lands on one it doesn't show. */ getVisibleModels(): ModelOption[] { - return this.models.filter( - (option) => this.props.getModelProps(option.model)?.ui !== false && !this.isHidden(option.id) + const activeServerId = serversStore.active.id; + + // the active server's rows carry raw ids, hidden ones are saved server-qualified + return this.activeModels.filter( + (option) => + this.props.getModelProps(option.model)?.ui !== false && + !this.isHidden(qualifyModelId(option.serverId ?? activeServerId, rawModelId(option.id))) ); } @@ -450,13 +579,27 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { async selectModelById(modelId: string, options?: { recordRecent?: boolean }): Promise { if (!modelId || this.updating) return; - if (this.selectedModelId === modelId) { - if (options?.recordRecent) this.recordRecentModel(modelId); + const serverId = serverIdFromModelId(modelId) ?? serversStore.active.id; + const rawId = rawModelId(modelId); + // the selection is stored server-qualified, matching the aggregated + // model list, no matter which form the caller passed + const qualifiedId = qualifyModelId(serverId, rawId); + + // a model from another server makes that server active first; a remote + // one is only reachable once its tunnel is open, so the switch asks for it + if (serverId !== serversStore.active.id) { + serversStore.setActive(serverId); + await serversModelsStore.connectAndLoad(serverId); + await this.switchServer(); + } + + if (this.selectedModelId === qualifiedId) { + if (options?.recordRecent) this.recordRecentModel(qualifiedId); return; } - const option = this.models.find((model) => model.id === modelId); + const option = this.activeModels.find((model) => model.id === rawId); if (!option) throw new Error('Selected model is not available'); @@ -464,10 +607,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { this.error = null; try { - this.selectedModelId = option.id; + this.selectedModelId = qualifiedId; this.selectedModelName = option.model; + this.selectionFromStorage = false; + this.persistSelection(); - if (options?.recordRecent) this.recordRecentModel(modelId); + if (options?.recordRecent) this.recordRecentModel(qualifiedId); // reasoning levels belong to the model that answered: another model in the chat starts at Default const conversationModel = getConversationModel(conversationsStore.activeMessages); @@ -582,6 +727,53 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { } } + /** + * Activate a server for the selector tabs. Everything comes from memory: + * the model list and router rows are prefetched at startup and the local + * server state is kept while an external server is active. The selection + * is left alone, switching tabs must not pick a model. + */ + async switchServer(): Promise { + this.error = null; + + const server = serversStore.active; + + // local props describe the server the UI is served from; keep them while + // an external server is active instead of dropping and refetching + if (server.id === LOCAL_BACKEND_ID) { + serverStore.restoreLocalState(); + } else { + serverStore.cacheLocalState(); + serverStore.clear(); + } + + const cached = serversModelsStore.get(server.id); + + if (!cached.loaded) { + // nothing prefetched for this server (startup prefetch failed): load it once + await this.fetch(true); + + return; + } + + if (server.id === LOCAL_BACKEND_ID && !serverStore.props) { + // first visit to the local tab in this session + await serverStore.fetch({ background: true }); + } + + this.activeModels = cached.models; + this.loading = false; + + // the local router rows carry the load statuses; the startup prefetch + // already returned them, so a tab switch rebuilds the list from memory + if (server.id === LOCAL_BACKEND_ID && this.routerModels.length === 0 && cached.raw) { + this.routerModels = cached.raw.data; + this.activeModels = this.buildModelOptions(cached.raw); + } + + this.warmHubDetails(); + } + toDisplayName(id: string): string { const segments = id.split(/\\|\//); const candidate = segments.pop(); @@ -621,10 +813,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { const repos: string[] = []; - for (const option of this.models) { - const repo = repoOf(option.model); + for (const server of serversStore.enabled) { + for (const option of serversModelsStore.get(server.id).models) { + const repo = repoOf(option.model); - if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo); + if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo); + } } // the Hub rate limits aggressively and an unauthenticated burst of requests @@ -674,13 +868,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value) ), - // 0 is the server's way of leaving the trained context unknown; a - // model-mode listing reports it as meta.n_ctx_train instead - contextLength: - item.context_length || - (typeof item.meta?.n_ctx_train === 'number' && item.meta.n_ctx_train > 0 - ? item.meta.n_ctx_train - : undefined), + // external servers report the context in their listing, so the + // gauge keeps working when the list is rebuilt on reload + contextLength: readModelContextLength(item), description: details?.description, details: details?.details, draftSidecars: this._status.getDraftSidecars(modelId), @@ -690,6 +880,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { model: modelId, name: this.toDisplayName(displayNameSource), parsedId: ModelsService.parseModelId(modelId), + // stamp the server here so the option keeps its origin even + // after another server becomes active + serverId: serversStore.active.id, speaks: item.architecture?.output_modalities?.includes('audio') ?? false, tags: item.tags ?? [] }; @@ -715,6 +908,23 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { } } + private persistSelection(): void { + if (!browser) return; + + try { + if (!this.selectedModelId) { + localStorage.removeItem(SELECTED_MODEL_LOCALSTORAGE_KEY); + } else { + localStorage.setItem( + SELECTED_MODEL_LOCALSTORAGE_KEY, + JSON.stringify({ id: this.selectedModelId, model: this.selectedModelName }) + ); + } + } catch { + console.warn('[ModelsStore] Failed to persist the model selection'); + } + } + /** Move a model to the front of the recently used list. */ private recordRecentModel(qualifiedId: string): void { this.recentModelIds = [ @@ -746,7 +956,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { const response = await ModelsService.list(); this.routerModels = response.data; - this.models = this.buildModelOptions(response); + this.activeModels = this.buildModelOptions(response); await this.props.fetchModalitiesForLoadedModels(); @@ -756,10 +966,10 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { this.selectModelById(visible[0].id); } } else { - this.models = await this.fetchModelModeInternal(); + this.activeModels = await this.fetchModelModeInternal(); } } catch (error) { - this.models = []; + this.activeModels = []; this.error = error instanceof Error ? error.message : 'Failed to load models'; throw error; diff --git a/ui/apps/chat/src/lib/stores/models/props.svelte.ts b/ui/apps/chat/src/lib/stores/models/props.svelte.ts index d7c89c438624..61f8c6a8256d 100644 --- a/ui/apps/chat/src/lib/stores/models/props.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/props.svelte.ts @@ -18,15 +18,20 @@ import { } from '#lib/constants/index.ts'; import { FileTypeCategory, ModelModality, ReasoningEffort } from '#lib/enums/index.ts'; import { PropsService } from '#lib/services/props.service.ts'; -// direct imports between stores, not via the barrel, to avoid circular deps import { serverStore } from '#lib/stores/llama-server.svelte.ts'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serversStore } from '#lib/stores/servers.svelte.ts'; import type { ReasoningEffortLevel } from '#lib/types/index.ts'; // deep imports, not the '#lib/utils/index.ts' barrel: it re-exports modules that reach back // into the stores, and going through it here would read a half-built module import { TTLCache } from '#lib/utils/cache-ttl.ts'; import { detectThinkingSupport } from '#lib/utils/chat-template-thinking-detector.ts'; +import { rawModelId } from '#lib/utils/model-option-id.ts'; import { SvelteMap } from 'svelte/reactivity'; +/** Grace period before /props is retried for a model after a failed fetch. */ +const PROPS_FETCH_RETRY_COOLDOWN_MS = 30_000; + /** * The slice of modelsStore the manager reads. Kept narrow on purpose so it * cannot reach around the host's full surface; modelsStore implements this @@ -34,7 +39,9 @@ import { SvelteMap } from 'svelte/reactivity'; */ export interface ModelPropsHost { /** Model rows the manager mirrors fetched modalities onto. */ - models: ModelOption[]; + activeModels: ModelOption[]; + /** Every enabled server's models, used to resolve a model's owner. */ + readonly models: ModelOption[]; readonly selectedModelName: string | null; readonly loadedModelIds: string[]; isModelLoaded(modelId: string): boolean; @@ -51,6 +58,8 @@ export class ModelPropsManager { maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES, ttlMs: MODEL_PROPS_CACHE.TTL_MS }); + /** Last failed /props fetch per model; requests stay on cooldown after it. */ + private failedPropsAt = new SvelteMap(); /** Props requests in flight, a second caller awaits the same request. */ private fetching = new SvelteMap>(); @@ -130,7 +139,7 @@ export class ModelPropsManager { try { const results = await Promise.all(propsPromises); - this.host.models = this.host.models.map((model) => { + this.host.activeModels = this.host.activeModels.map((model) => { const modelIndex = loadedModelIds.indexOf(model.model); if (modelIndex === -1) return model; @@ -159,14 +168,33 @@ export class ModelPropsManager { * @returns Props data or null if fetch failed or model not loaded */ async fetchModelProps(modelId: string): Promise { + // props not loaded yet (mid server switch): the server role and model + // load state are not trustworthy, so do not guess + if (!serverStore.props) return null; + const cached = this.cache.get(modelId); if (cached) return cached; + // never ask a server about a model another server serves; a name that + // exists on two servers resolves to the active one's entry first + const option = this.host.models.find( + (m) => m.model === modelId || m.id === modelId || rawModelId(m.id) === modelId + ); + + if (option?.serverId && option.serverId !== serversStore.active.id) return null; + if (serverStore.isRouterMode && !this.host.isModelLoaded(modelId)) { return null; } + // a failing model would otherwise be refetched on every reactive update + const failedAt = this.failedPropsAt.get(modelId); + + if (failedAt !== undefined && Date.now() - failedAt < PROPS_FETCH_RETRY_COOLDOWN_MS) { + return null; + } + const inFlight = this.fetching.get(modelId); if (inFlight) return inFlight; @@ -175,10 +203,12 @@ export class ModelPropsManager { .then((props) => { this.cache.set(modelId, props); this.cacheVersion++; + this.failedPropsAt.delete(modelId); return props; }) .catch((error) => { + this.failedPropsAt.set(modelId, Date.now()); console.warn(`Failed to fetch props for model ${modelId}:`, error); return null; @@ -194,7 +224,12 @@ export class ModelPropsManager { const props = this.getModelProps(modelId); const nCtx = props?.default_generation_settings?.n_ctx; - return typeof nCtx === 'number' ? nCtx : null; + if (typeof nCtx === 'number') return nCtx; + + // external servers report the context in their model listing, when they report one + const model = this.host.activeModels.find((m) => m.model === modelId || m.id === modelId); + + return model?.contextLength ?? null; } getModelModalities(modelId: string): ModelModalities | null { @@ -202,7 +237,7 @@ export class ModelPropsManager { return this.buildModalities(serverStore.props.modalities); } - const model = this.host.models.find((m) => m.model === modelId || m.id === modelId); + const model = this.host.activeModels.find((m) => m.model === modelId || m.id === modelId); if (model?.modalities) { return model.modalities; @@ -304,7 +339,7 @@ export class ModelPropsManager { if (!props?.modalities) return; - this.host.models = this.host.models.map((model) => + this.host.activeModels = this.host.activeModels.map((model) => model.model === modelId ? { ...model, modalities: this.buildModalities(props.modalities!) } : model diff --git a/ui/apps/chat/src/lib/stores/models/status.svelte.ts b/ui/apps/chat/src/lib/stores/models/status.svelte.ts index ade2b701a881..46ceaa6d3921 100644 --- a/ui/apps/chat/src/lib/stores/models/status.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/status.svelte.ts @@ -8,6 +8,7 @@ import { CLI_FLAGS, HF_UD_QUANT_PREFIX_REGEX, + LOCAL_BACKEND_ID, MODEL_ID, type ModelSidecar, STORAGE_APP_NAME @@ -20,9 +21,11 @@ import { } from '#lib/enums/index.ts'; import { HuggingFaceService } from '#lib/services/huggingface.service.ts'; import { ModelsService } from '#lib/services/models.service.ts'; -// direct imports between stores, not via the barrel, to avoid circular deps import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import type { ModelPropsManager } from '#lib/stores/models/props.svelte.ts'; +import { serversStore } from '#lib/stores/servers.svelte.ts'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serversModelsStore } from '#lib/stores/serversModels.svelte.ts'; // explicit type imports: the app.d.ts globals resolve to `any`, so import the real types import type { ApiModelsSseDownloadFailedData, @@ -32,6 +35,7 @@ import type { } from '#lib/types/index.ts'; import type { ModelSidecarBadge } from '#lib/types/models.js'; import { repoOf } from '#lib/utils/model-names.ts'; +import { serverIdFromModelId } from '#lib/utils/model-option-id.ts'; import { isAuxSidecar, isDraftSidecar, sidecarFromFileToken } from '#lib/utils/sidecars.ts'; import { browser } from '$app/env'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -42,13 +46,20 @@ import { toast } from 'svelte-sonner'; * cannot reach around the host's full surface; modelsStore implements this * structurally. */ +/** How long a remote operation is watched before its last answer is kept. */ +const REMOTE_STATUS_TIMEOUT_MS = 10 * 60 * 1000; +const REMOTE_STATUS_POLL_MS = 2000; + export interface ModelStatusHost { error: string | null; + /** Load state a model's own server reports, local or external. */ + getModelStatus(modelId: string): ServerModelStatus | null; readonly props: ModelPropsManager; /** Router model rows the status feed updates. */ routerModels: ApiModelDataEntry[]; fetchRouterModels(): Promise; isModelLoaded(modelId: string): boolean; + switchServer(): Promise; toDisplayName(id: string): string; } @@ -196,6 +207,9 @@ export class ModelStatusManager { return result; }); + /** Feed reader of the active server when it is not the built-in one. */ + private serverAbort: AbortController | null = null; + private serverReaderId: string | null = null; // /models/sse feed state, the single source of truth for status and load progress private statusAbort: AbortController | null = null; private statusReaderActive = false; @@ -213,6 +227,8 @@ export class ModelStatusManager { * removes the model along with the partial files of its download. */ async cancelDownload(repoWithTag: string): Promise { + await this.ensureLocalTarget(); + if (!serverStore.isRouterMode) { toast.error('Model downloads are only available in router mode'); @@ -265,12 +281,16 @@ export class ModelStatusManager { * settled status, so no waiter is registered here. */ async cancelLoad(modelId: string): Promise { - if (!serverStore.isRouterMode) return; + const serverId = this.serverIdFor(modelId); + + // the local server takes unload requests in router mode only; a remote + // server cancels on its own + if (serverId === LOCAL_BACKEND_ID && !serverStore.localIsRouter) return; this.subscribe(); try { - await ModelsService.unload(modelId); + await ModelsService.unload(modelId, serverId); toast.info(`Load cancelled: ${this.host.toDisplayName(modelId)}`); } catch (error) { toast.error(`Failed to cancel load: ${this.host.toDisplayName(modelId)}`); @@ -306,6 +326,28 @@ export class ModelStatusManager { return; } + // a second click must not post again: the server would answer that the model + // already exists, so a download already on its way is left alone + if (this.pendingDownloads.has(model) || this.isDownloadInProgress(model)) return; + + // the marker and its last reported progress survive until the request is + // accepted: dropping them first would drop the paused state, which reads as + // a lost download if the server rejects the resume + const snapshot = this.pausedDownloads.get(model) ?? null; + const wasPaused = this.pausedDownloads.has(model); + + // the download reads as in flight from here, so the chip shows it before the + // server answers; the feed refines it with the real counts. Both marks land + // ahead of the first await, so a second click in the same tick still coalesces + this.pendingDownloads.add(model); + this.downloadProgress.set(model, snapshot ?? emptyDownloadProgress()); + + // the local server owns the download endpoint, so the target moves only once + // the row already reads as downloading + await this.ensureLocalTarget(); + + // read once the target has moved: the listing describes the server taking the + // download, not the server the row was clicked from const registered = this.host.routerModels.find( (m) => downloadIdKey(m.id) === downloadIdKey(model) ); @@ -317,25 +359,9 @@ export class ModelStatusManager { ...extraSidecars ].filter((kind, i, all) => SERVER_SIDECARS.includes(kind) && all.indexOf(kind) === i); - // a second click must not post again: the server would answer that the model - // already exists, so a download already on its way is left alone - if (this.pendingDownloads.has(model) || this.isDownloadInProgress(model)) return; - - this.pendingDownloads.add(model); - // the feed must be live so the resulting models_reload event refreshes the list this.subscribe(); - // the marker and its last reported progress survive until the request is - // accepted: dropping them first would drop the paused state, which reads as - // a lost download if the server rejects the resume - const snapshot = this.pausedDownloads.get(model) ?? null; - const wasPaused = this.pausedDownloads.has(model); - - // the download reads as in flight from here, so the chip shows it before the - // server answers; the feed refines it with the real counts - this.downloadProgress.set(model, snapshot ?? emptyDownloadProgress()); - try { const res = await ModelsService.downloadModel(model, sidecars); @@ -495,7 +521,10 @@ export class ModelStatusManager { return this.loadingStates.get(modelId) ?? false; } - async load(modelId: string): Promise { + async load(modelId: string, extraArgs?: string[]): Promise { + const serverId = this.serverIdFor(modelId); + const isLocal = serverId === LOCAL_BACKEND_ID; + if (this.host.isModelLoaded(modelId)) return; if (this.loadingStates.get(modelId)) return; @@ -506,13 +535,21 @@ export class ModelStatusManager { // the feed drives completion, so it must be live before the request this.subscribe(); - const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED); + const reachedLoaded = isLocal + ? this.waitForStatus(modelId, ServerModelStatus.LOADED) + : Promise.resolve(); reachedLoaded.catch(() => {}); try { - await ModelsService.load(modelId); - await reachedLoaded; + await ModelsService.load(modelId, extraArgs, serverId); + + if (isLocal) { + await reachedLoaded; + } else { + await this.waitForRemoteStatus(serverId, modelId, ServerModelStatus.LOADED); + } + toast.success(`Model loaded: ${this.host.toDisplayName(modelId)}`); } catch (error) { this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed')); @@ -532,6 +569,8 @@ export class ModelStatusManager { * before the request: an unconfirmed pause would leave the chip spinning. */ async pauseDownload(repoWithTag: string): Promise { + await this.ensureLocalTarget(); + if (!serverStore.isRouterMode) { toast.error('Model downloads are only available in router mode'); @@ -571,13 +610,37 @@ export class ModelStatusManager { subscribe(): void { if (this.statusReaderActive) return; - if (!serverStore.isRouterMode) return; + // the feed reports the local server's models, so the local server's role + // decides, not the active server's + if (!serverStore.localIsRouter) return; this.statusReaderActive = true; this.statusAbort = new AbortController(); void this.runStatusReader(this.statusAbort.signal); } + /** + * Read the status feed of the active server, when it is not the built-in one. + * + * A remote router reports its load progress over its own feed, which the + * tunnel carries like any other request; the operations that wait on a load + * keep their polling as the fallback. Pass null to close it. + */ + subscribeServer(serverId: string | null): void { + if (this.serverReaderId === serverId) return; + + this.serverAbort?.abort(); + this.serverAbort = null; + this.serverReaderId = serverId; + + if (!serverId) return; + + const abort = new AbortController(); + + this.serverAbort = abort; + void this.runStatusReader(abort.signal, serverId); + } + /** * A model the server lists as incomplete is a download that never finished: it * runs in no tab and carries no progress. Offer it as paused with the sidecars @@ -609,6 +672,9 @@ export class ModelStatusManager { } async unload(modelId: string): Promise { + const serverId = this.serverIdFor(modelId); + const isLocal = serverId === LOCAL_BACKEND_ID; + if (!this.host.isModelLoaded(modelId)) return; if (this.loadingStates.get(modelId)) return; @@ -618,13 +684,21 @@ export class ModelStatusManager { this.subscribe(); - const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED); + const reachedUnloaded = isLocal + ? this.waitForStatus(modelId, ServerModelStatus.UNLOADED) + : Promise.resolve(); reachedUnloaded.catch(() => {}); try { - await ModelsService.unload(modelId); - await reachedUnloaded; + await ModelsService.unload(modelId, serverId); + + if (isLocal) { + await reachedUnloaded; + } else { + await this.waitForRemoteStatus(serverId, modelId, ServerModelStatus.UNLOADED); + } + toast.info(`Model unloaded: ${this.host.toDisplayName(modelId)}`); } catch (error) { this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed')); @@ -641,6 +715,7 @@ export class ModelStatusManager { this.statusReaderActive = false; this.statusAbort?.abort(); this.statusAbort = null; + this.subscribeServer(null); this.loadProgress.clear(); this.downloadProgress.clear(); this.failedDownloads.clear(); @@ -812,6 +887,13 @@ export class ModelStatusManager { return this.pausedDownloads.delete(repoWithTag); } + private async ensureLocalTarget(): Promise { + if (serversStore.active.id === LOCAL_BACKEND_ID) return; + + serversStore.setActive(LOCAL_BACKEND_ID); + await serversModelsStore.ensureLoaded(LOCAL_BACKEND_ID); + await this.host.switchServer(); + } /** True when the option is the model itself, or a sidecar its download pulls. */ private pullsSidecar(repoWithTag: string): boolean { const { model, sidecar } = downloadTarget(repoWithTag); @@ -849,8 +931,27 @@ export class ModelStatusManager { void this.host.fetchRouterModels(); } - private async runStatusReader(signal: AbortSignal): Promise { - await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event)); + private async runStatusReader(signal: AbortSignal, serverId?: string): Promise { + await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event), serverId); + } + + /** + * Server a model belongs to: the one its qualified id names, else whichever + * enabled server lists it. Loading an external llama-server's model must not + * become a request to the server this UI is served from. + */ + private serverIdFor(modelId: string): string { + const qualified = serverIdFromModelId(modelId); + + if (qualified) return qualified; + + for (const server of serversStore.enabled) { + const state = serversModelsStore.get(server.id); + + if (state.models.some((option) => option.model === modelId)) return server.id; + } + + return serversStore.active.id; } private setPausedDownload(repoWithTag: string, progress: ModelDownloadProgress | null): void { @@ -883,6 +984,42 @@ export class ModelStatusManager { for (const waiter of waiters) if (waiter.target === status) waiter.resolve(); } + /** + * Wait for an external server to report a status. Its changes never reach the + * local feed, so its own listing is asked again until the status lands or the + * deadline passes, which leaves the last answer in place. + */ + private async waitForRemoteStatus( + serverId: string, + modelId: string, + target: ServerModelStatus + ): Promise { + const deadline = Date.now() + REMOTE_STATUS_TIMEOUT_MS; + + while (Date.now() < deadline) { + await serversModelsStore.refresh(serverId); + + const status = this.host.getModelStatus(modelId); + + if (status === ServerModelStatus.FAILED) { + throw new Error( + target === ServerModelStatus.LOADED + ? 'the server failed to load it' + : 'the server failed to unload it' + ); + } + + const reached = + target === ServerModelStatus.LOADED + ? status === ServerModelStatus.LOADED || status === ServerModelStatus.SLEEPING + : status === ServerModelStatus.UNLOADED || status === null; + + if (reached) return; + + await new Promise((resolve) => setTimeout(resolve, REMOTE_STATUS_POLL_MS)); + } + } + // a load and the requests that wait on it can each hold a waiter for the same model private waitForStatus(modelId: string, target: ServerModelStatus): Promise { return new Promise((resolve, reject) => { diff --git a/ui/apps/chat/src/lib/stores/remote-access.svelte.ts b/ui/apps/chat/src/lib/stores/remote-access.svelte.ts new file mode 100644 index 000000000000..16977c1f34e0 --- /dev/null +++ b/ui/apps/chat/src/lib/stores/remote-access.svelte.ts @@ -0,0 +1,353 @@ +/** + * remoteAccessStore - tunnels to remote llama.cpp servers. + * + * A remote server is a `llama-server --connect` host reached over a WebRTC + * data channel. Its access code is stored on the server itself, in the + * server list the settings carry, so this store owns no persistence: it owns + * the live tunnels and the transport that routes a request to the right one. + * + * Routing works on the address, not on the origin. Every request addressed at + * a server's synthetic tunnel origin (`tunnelOrigin`) is answered by that + * server's data channel, and every other request is left to the browser. A + * request addressed at a tunnel origin that has no live tunnel is refused here + * and never reaches the server serving this UI: the origin cannot resolve, so a + * missed call site fails loudly instead of talking to the wrong server. + * + * Tunnels are opened lazily. Only the server the app opens with is connected + * during startup; the rest connect when they are first needed - a model of + * theirs is picked, or the user hits Connect on their card. + * + * The whole transport is experimental and sits behind a setting that is off by + * default: while it is off no tunnel opens, and the ones already open close. + */ + +import { CODE_LENGTHS, SETTINGS_KEYS, WEBRTC_TIMEOUTS } from '#lib/constants/index.ts'; +import { TunnelStatus } from '#lib/enums/index.ts'; +// direct import between stores, not via the barrel, to avoid circular deps +import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import type { Server } from '#lib/types/index.ts'; +import { getServers } from '#lib/utils/api-base.ts'; +import { codeOf, isRemoteServer, parseTunnelOrigin, tunnelHostLabel } from '#lib/utils/servers.ts'; +import { ClientTunnel } from '#lib/utils/webrtc-tunnel.ts'; +import { browser } from '$app/env'; + +/** Servers reached over a tunnel, in list order. */ +function remoteServers(): Server[] { + return getServers().filter(isRemoteServer); +} + +class RemoteAccessStore { + /** Last tunnel failure by server id. */ + errorMessage = $state>({}); + /** Tunnel state by server id; servers with no state never attempted a tunnel. */ + status = $state>({}); + + /** In-flight handshakes by server id, so a second caller joins the first. */ + private attempts = new Map>(); + /** + * Handshake count by server id, bumped when an attempt is dropped, so an + * answer that lands after the user gave up is not stored. + */ + private generation = new Map(); + private originalFetch: typeof window.fetch | null = null; + /** Pending visibility recovery, so one burst of tab switches opens one handshake. */ + private recoverTimer: ReturnType | null = null; + /** Live tunnels by server id. */ + private tunnels = new Map(); + /** Requests held back while their server's handshake runs. */ + private waiters = new Map void; resolve: () => void }>>(); + + /** Whether the experimental remote pairing is turned on; every entry point hides without it. */ + get isEnabled(): boolean { + return Boolean(settingsStore.getConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING)); + } + + /** + * Open the tunnel of a remote server, or join the handshake already running. + * + * Resolves once the data channel is open and the pass code was accepted. + */ + async connect(serverId: string): Promise { + if (this.isConnected(serverId)) return; + + const running = this.attempts.get(serverId); + + if (running) return running; + + const generation = this.generationOf(serverId); + const attempt = this.open(serverId, generation).finally(() => { + if (this.attempts.get(serverId) === attempt) this.attempts.delete(serverId); + }); + + this.attempts.set(serverId, attempt); + + return attempt; + } + + /** Close a server's tunnel and forget its state; the server itself stays. */ + disconnect(serverId: string): void { + // a handshake still running must land on nothing: the channel is not wanted + this.dropAttempt(serverId); + this.tunnels.get(serverId)?.disconnect(); + this.tunnels.delete(serverId); + this.errorMessage[serverId] = ''; + this.rejectWaiters(serverId, new Error('Remote access closed')); + + // a server that was never connected has no state to clear + if (serverId in this.status) delete this.status[serverId]; + } + + /** Last tunnel failure of a server. */ + errorOf(serverId: string): string { + return this.errorMessage[serverId] ?? ''; + } + + /** + * Replace `window.fetch` with the tunnel transport. + * + * Called from `hydrateStores`, before any request goes out. A server is + * recognised by its host label rather than by the page origin, so several + * tunnels coexist and requests to real URLs are untouched. + */ + installInterceptor(): void { + if (!browser || this.originalFetch) return; + + this.originalFetch = window.fetch.bind(window); + const original = this.originalFetch; + + window.fetch = (input: RequestInfo | URL, init?: RequestInit) => { + try { + const href = + input instanceof Request ? input.url : input instanceof URL ? input.href : String(input); + const label = parseTunnelOrigin(href); + + if (label !== null) return this.request(label, input, init); + } catch { + // unparseable input: it is not addressed at a tunnel + } + + return original(input, init); + }; + } + + isConnected(serverId: string): boolean { + return this.statusOf(serverId) === TunnelStatus.CONNECTED; + } + + isConnecting(serverId: string): boolean { + return this.statusOf(serverId) === TunnelStatus.CONNECTING; + } + + /** + * Reconnect the channels a backgrounded tab lost. + * + * iOS Safari and Android PWAs tear a peer connection down when the tab goes + * to the background, which leaves the card on a failure the user can only + * clear with the retry icon. Only a channel that was already wanted is + * recovered: an idle server is one nobody asked to connect. The recovery + * stays silent, so a failure just lands back on the card. + */ + recoverChannels(): void { + if (!browser) return; + + // a tab switch is not the network being back: let the connection settle, + // and let a burst of switches coalesce into one attempt per server + if (this.recoverTimer) clearTimeout(this.recoverTimer); + + this.recoverTimer = setTimeout(() => { + this.recoverTimer = null; + + if (navigator.onLine === false) return; + + for (const server of remoteServers()) { + if (!server.enabled || this.statusOf(server.id) !== TunnelStatus.ERROR) continue; + + void this.connect(server.id).catch(() => { + // the card keeps the failure and offers the retry control + }); + } + }, WEBRTC_TIMEOUTS.RECOVER_MS); + } + + /** + * Answer a request addressed at a tunnel origin. + * + * @param label Host label of the origin the request was addressed at. + */ + async request(label: string, input: RequestInfo | URL, init?: RequestInit): Promise { + const server = this.findByHostLabel(label); + + // A tunnel origin that no server owns is never served: the request was + // built for a server that is gone, or aimed at the suffix on purpose. + if (!server) { + throw new Error(`No remote server is registered for the tunnel host "${label}"`); + } + + const tunnel = this.tunnels.get(server.id); + + if (tunnel?.isConnected) return tunnel.fetch(input, init); + + if (this.status[server.id] === TunnelStatus.CONNECTING) { + await new Promise((resolve, reject) => { + const queued = this.waiters.get(server.id) ?? []; + + queued.push({ reject, resolve }); + this.waiters.set(server.id, queued); + }); + + // the handshake resolved; if the channel died before this ran, the + // server is simply not connected and says so + const opened = this.tunnels.get(server.id); + + if (!opened?.isConnected) { + throw new Error(`"${server.name}" disconnected while the request was waiting`); + } + + return opened.fetch(input, init); + } + + throw new Error( + `"${server.name}" is not connected. Reconnect it in the servers dialog, ` + + 'or pick a model of another server.' + ); + } + + /** Tunnel state of a server. */ + statusOf(serverId: string): TunnelStatus { + return this.status[serverId] ?? TunnelStatus.IDLE; + } + + /** + * Drop the tunnels of servers that left the list or were disabled, so a + * disabled server stops holding a data channel open. Turning remote pairing + * off disables them all. + */ + syncServers(): void { + const kept = new Set( + this.isEnabled + ? remoteServers() + .filter((server) => server.enabled) + .map((b) => b.id) + : [] + ); + + for (const serverId of [...this.tunnels.keys()]) { + if (!kept.has(serverId)) this.disconnect(serverId); + } + + for (const serverId of Object.keys(this.status)) { + if (!kept.has(serverId)) { + delete this.status[serverId]; + delete this.errorMessage[serverId]; + } + } + } + + uninstallInterceptor(): void { + if (!this.originalFetch) return; + + window.fetch = this.originalFetch; + this.originalFetch = null; + } + + /** + * Give up on a server's handshake: a later answer must land on nothing, and a + * caller asking again must start over instead of joining the dropped attempt. + */ + private dropAttempt(serverId: string): void { + this.generation.set(serverId, this.generationOf(serverId) + 1); + this.attempts.delete(serverId); + } + + /** Server whose tunnel origin carries this host label. */ + private findByHostLabel(label: string): Server | undefined { + return remoteServers().find((server) => tunnelHostLabel(server.id) === label); + } + + private generationOf(serverId: string): number { + return this.generation.get(serverId) ?? 0; + } + + /** Handshake a server's tunnel is not already running. */ + private async open(serverId: string, generation: number): Promise { + const server = getServers().find((candidate) => candidate.id === serverId); + + if (!this.isEnabled) { + throw new Error( + 'Remote server pairing is turned off. Turn it on in Settings, under General.' + ); + } + + if (!isRemoteServer(server) || !codeOf(server)) { + throw new Error('This server has no usable access code'); + } + + this.status[serverId] = TunnelStatus.CONNECTING; + this.errorMessage[serverId] = ''; + + // a dropped attempt must report nothing: the channel belongs to whoever asked + // for it last, and this one may land after that + const stillWanted = () => generation === this.generationOf(serverId); + const code = codeOf(server); + const tunnel = new ClientTunnel( + code.slice(0, CODE_LENGTHS.ROOM), + code.slice(CODE_LENGTHS.ROOM), + { + onConnected: () => { + if (stillWanted()) this.status[serverId] = TunnelStatus.CONNECTED; + }, + onDisconnected: () => { + // the channel died: the server stays registered, its requests fail + // loudly until it is reconnected + if (!stillWanted()) return; + + this.status[serverId] = TunnelStatus.ERROR; + this.errorMessage[serverId] = 'Disconnected from the host'; + this.tunnels.delete(serverId); + this.rejectWaiters(serverId, new Error('Remote access disconnected')); + } + } + ); + + try { + await tunnel.connect(); + + // dropped while the handshake ran: the user is done with this channel, so + // the answer must not bring it back + if (!stillWanted()) { + tunnel.disconnect(); + + return; + } + + this.tunnels.set(serverId, tunnel); + this.status[serverId] = TunnelStatus.CONNECTED; + this.releaseWaiters(serverId); + } catch (error: unknown) { + tunnel.disconnect(); + + // dropped while the handshake ran: the state the user asked for stands, + // and the failure belongs to an attempt nobody waits on any more + if (!stillWanted()) throw error; + + this.tunnels.delete(serverId); + this.status[serverId] = TunnelStatus.ERROR; + this.errorMessage[serverId] = error instanceof Error ? error.message : String(error); + this.rejectWaiters(serverId, error instanceof Error ? error : new Error(String(error))); + + throw error; + } + } + + private rejectWaiters(serverId: string, error: Error): void { + for (const waiter of this.waiters.get(serverId) ?? []) waiter.reject(error); + this.waiters.delete(serverId); + } + + private releaseWaiters(serverId: string): void { + for (const waiter of this.waiters.get(serverId) ?? []) waiter.resolve(); + this.waiters.delete(serverId); + } +} + +export const remoteAccessStore = new RemoteAccessStore(); diff --git a/ui/apps/chat/src/lib/stores/servers.svelte.ts b/ui/apps/chat/src/lib/stores/servers.svelte.ts new file mode 100644 index 000000000000..9e68b90b16dd --- /dev/null +++ b/ui/apps/chat/src/lib/stores/servers.svelte.ts @@ -0,0 +1,220 @@ +/** + * serversStore - llama.cpp servers the UI can talk to. + * + * The built-in server is the llama-server serving this UI. The others are + * user-configured: a server on a base URL, or a remote one reached over a + * WebRTC tunnel by the access code its `llama-server --connect` prints. The + * store registers the resolved list with the api-base registry, which services + * use to build request URLs, and tells the remote access store when a tunnel is + * no longer wanted. A page published with servers (`llama-chat --server`) + * lists them in `llama-chat.json`, and the store adds each of them once. + */ + +import { + ACTIVE_SERVER_LOCALSTORAGE_KEY, + LOCAL_BACKEND_ID, + PRESET_SERVERS_FILE, + PRESET_SERVERS_LOCALSTORAGE_KEY, + PRESET_SERVERS_META, + SETTINGS_KEYS +} from '#lib/constants/index.ts'; +import { serverStore } from '#lib/stores/llama-server.svelte.ts'; +import { remoteAccessStore } from '#lib/stores/remote-access.svelte.ts'; +import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import type { Server } from '#lib/types/index.ts'; +import { setServersResolver } from '#lib/utils/api-base.ts'; +import { + createLocalServer, + createRemoteServer, + createUrlServer, + isRemoteServer, + parsePresetServers, + parseServersSettings, + presetServersToAdd +} from '#lib/utils/servers.ts'; +import { uuid } from '#lib/utils/uuid.ts'; +import { browser } from '$app/env'; + +function loadActiveServerId(): string { + if (!browser) return LOCAL_BACKEND_ID; + + try { + return localStorage.getItem(ACTIVE_SERVER_LOCALSTORAGE_KEY) ?? LOCAL_BACKEND_ID; + } catch { + return LOCAL_BACKEND_ID; + } +} + +function persistActiveServerId(serverId: string): void { + if (!browser) return; + + try { + localStorage.setItem(ACTIVE_SERVER_LOCALSTORAGE_KEY, serverId); + } catch { + /* ignore */ + } +} + +function loadSeenPresets(): string[] { + try { + const seen: unknown = JSON.parse(localStorage.getItem(PRESET_SERVERS_LOCALSTORAGE_KEY) ?? '[]'); + + return Array.isArray(seen) ? seen.filter((url) => typeof url === 'string') : []; + } catch { + return []; + } +} + +function persistSeenPresets(urls: string[]): void { + try { + localStorage.setItem(PRESET_SERVERS_LOCALSTORAGE_KEY, JSON.stringify(urls)); + } catch { + /* ignore */ + } +} + +class ServersStore { + activeId = $state(loadActiveServerId()); + + get active(): Server { + const active = this.enabled.find((server) => server.id === this.activeId); + + return active ?? this.enabled[0] ?? this.local; + } + + get enabled(): Server[] { + return this.list.filter((server) => server.enabled && !this.isMissingLocal(server)); + } + + /** Every remote server, connected or not; their code is the credential. */ + get external(): Server[] { + return parseServersSettings(settingsStore.config[SETTINGS_KEYS.SERVERS]); + } + + get list(): Server[] { + return [this.local, ...this.external]; + } + + get local(): Server { + return createLocalServer( + settingsStore.config.apiKey?.toString().trim() || undefined, + settingsStore.config[SETTINGS_KEYS.LOCAL_BACKEND_ENABLED] !== false + ); + } + + /** Servers a tunnel can be opened to, enabled or not. */ + get remote(): Server[] { + return this.external.filter(isRemoteServer); + } + + /** + * Add the servers the page was published with, listed in `llama-chat.json` + * next to index.html, each once: one the user removed stays removed. Only a + * page whose index.html carries the marker for the file asks for it, so one + * served by llama-server or the dev server adds nothing. With + * no local server to fall back on, the app picks the first enabled one. + * + * @returns the ids of the servers it added + */ + async addPresetServers(): Promise { + if (!browser || !document.querySelector(`meta[name="${PRESET_SERVERS_META}"]`)) return []; + + let raw: unknown; + + try { + const response = await fetch(`./${PRESET_SERVERS_FILE}`, { cache: 'no-cache' }); + + if (!response.ok) return []; + + raw = await response.json(); + } catch { + // not there, or not JSON: an SPA fallback that answers with index.html + return []; + } + + const urls = parsePresetServers(raw); + + if (urls.length === 0) return []; + + const seen = loadSeenPresets(); + const added = presetServersToAdd(urls, this.external, seen).map((url) => + createUrlServer(url, uuid()) + ); + + if (added.length > 0) this.saveExternal([...this.external, ...added]); + + persistSeenPresets([...new Set([...seen, ...urls])]); + + return added.map((server) => server.id); + } + + /** + * Pair with a `llama-server --connect` host by its access code. + * + * The tunnel is not opened here: the caller decides when the handshake is + * worth running (the servers dialog tests it, startup only tests the + * server the app opens with). + */ + addRemoteServer(code: string, name?: string, id?: string, apiKey?: string): Server { + const server = createRemoteServer(code, id ?? uuid(), name, apiKey); + + this.saveExternal([...this.external, server]); + + return server; + } + + /** Point at a llama.cpp server on its own base URL. */ + addServer(server: Server): void { + this.saveExternal([...this.external, server]); + } + + initialize(): void { + if (!browser) return; + + setServersResolver(() => ({ activeId: this.active.id, servers: this.list })); + } + + removeServer(serverId: string): void { + this.saveExternal(this.external.filter((server) => server.id !== serverId)); + + // the code is the credential: dropping the server drops the tunnel too, + // and the store forgets its state on the next sync + remoteAccessStore.disconnect(serverId); + remoteAccessStore.syncServers(); + + if (this.activeId === serverId) { + this.setActive(LOCAL_BACKEND_ID); + } + } + + setActive(serverId: string): void { + const id = this.list.some((server) => server.id === serverId) ? serverId : LOCAL_BACKEND_ID; + + this.activeId = id; + persistActiveServerId(id); + } + + setLocalEnabled(enabled: boolean): void { + settingsStore.updateConfig(SETTINGS_KEYS.LOCAL_BACKEND_ENABLED, enabled); + } + + updateServer(serverId: string, updates: Partial): void { + this.saveExternal( + this.external.map((server) => (server.id === serverId ? { ...server, ...updates } : server)) + ); + + // a disabled or re-coded server must not keep its data channel + remoteAccessStore.syncServers(); + } + + /** The built-in server counts only when a local server answered this session. */ + private isMissingLocal(server: Server): boolean { + return server.id === LOCAL_BACKEND_ID && serverStore.localServerMissing; + } + + private saveExternal(servers: Server[]): void { + settingsStore.updateConfig(SETTINGS_KEYS.SERVERS, JSON.stringify(servers)); + } +} + +export const serversStore = new ServersStore(); diff --git a/ui/apps/chat/src/lib/stores/serversModels.svelte.ts b/ui/apps/chat/src/lib/stores/serversModels.svelte.ts new file mode 100644 index 000000000000..a10a68bb8aff --- /dev/null +++ b/ui/apps/chat/src/lib/stores/serversModels.svelte.ts @@ -0,0 +1,150 @@ +/** + * serversModelsStore - Per-server model catalog cache. + * + * Prefetches every enabled server's model list, so switching servers is + * instant and the switcher can show load state. The active server's list + * still lives in modelsStore, which owns selection and chat wiring; this + * cache is the prefetch layer the switches start from. + * + * Remote servers are listed through their tunnel, so the prefetch skips one + * until its tunnel is open: a handshake per server is what `connectAndLoad` + * is for, never the startup sweep. + */ + +import { ModelsService } from '#lib/services/models.service.ts'; +import { ServersService } from '#lib/services/servers.service.ts'; +import { remoteAccessStore } from '#lib/stores/remote-access.svelte.ts'; +import { serversStore } from '#lib/stores/servers.svelte.ts'; +import type { Server } from '#lib/types/index.ts'; +import type { ApiModelsListResponse } from '#lib/types/index.ts'; +import type { ModelSidecarFile } from '#lib/types/models.js'; +import type { ModelOption } from '#lib/types/models.js'; +import { isRemoteServer } from '#lib/utils/servers.ts'; + +export interface ServerModelsState { + /** Draft sidecars the listing carries, keyed by the repo they belong to. */ + drafts?: Record; + error: string | null; + loaded: boolean; + loading: boolean; + models: ModelOption[]; + /** Untouched list payload, kept for the local server so its router rows survive a tab switch. */ + raw?: ApiModelsListResponse; +} + +const EMPTY_STATE: ServerModelsState = { + error: null, + loaded: false, + loading: false, + models: [] +}; + +class ServersModelsStore { + private states = $state>({}); + + clear(serverId: string): void { + delete this.states[serverId]; + } + + /** + * Open a remote server's tunnel and list it. + * + * The card's Connect action and the server switch both want this; startup + * prefetch does not, since a handshake per server on first paint is not what + * a list of servers asked for. + */ + async connectAndLoad(serverId: string): Promise { + await this.ensureLoaded(serverId, { connect: true }); + } + + /** + * Load a server's models once. + * + * @param connect - Open the tunnel of a remote server when it has none. Off + * by default: the startup prefetch must not handshake with every server. + */ + async ensureLoaded(serverId: string, options?: { connect?: boolean }): Promise { + const server = serversStore.enabled.find((candidate) => candidate.id === serverId); + + if (!server) return; + + const state = this.states[serverId]; + + if (state?.loaded || state?.loading) return; + + if (isRemoteServer(server) && !remoteAccessStore.isConnected(serverId)) { + if (!options?.connect) return; + + this.states[serverId] = { error: null, loaded: false, loading: true, models: [] }; + + try { + await remoteAccessStore.connect(serverId); + } catch (error: unknown) { + this.states[serverId] = { + error: error instanceof Error ? error.message : String(error), + loaded: false, + loading: false, + models: [] + }; + + return; + } + } + + this.states[serverId] = { error: null, loaded: false, loading: true, models: [] }; + await this.fetch(server); + } + + get(serverId: string): ServerModelsState { + return this.states[serverId] ?? EMPTY_STATE; + } + + /** Prefetch every enabled server's model list. */ + async loadAll(): Promise { + const enabled = serversStore.enabled; + const ids = new Set(enabled.map((server) => server.id)); + + for (const id of Object.keys(this.states)) { + if (!ids.has(id)) { + delete this.states[id]; + } + } + + await Promise.all(enabled.map((server) => this.ensureLoaded(server.id))); + } + + /** + * Ask a server for its list again, keeping what is already known. Used while + * a remote load settles, since its status never reaches the local feed. + */ + async refresh(serverId: string): Promise { + const server = serversStore.enabled.find((candidate) => candidate.id === serverId); + + // a remote server without a tunnel has nothing to answer; keep what is + // already known rather than reporting a transport failure as a server one + if (!server || (isRemoteServer(server) && !remoteAccessStore.isConnected(serverId))) return; + + await this.fetch(server); + } + + /** Forget a server's list and ask for it again, tunnel included. */ + async reload(serverId: string): Promise { + this.clear(serverId); + await this.connectAndLoad(serverId); + } + + private async fetch(server: Server): Promise { + const result = await ServersService.listModels(server); + + this.states[server.id] = { + drafts: result.raw ? ModelsService.sidecarFilesByRepo(result.raw) : undefined, + error: result.error ?? null, + loaded: result.ok, + loading: false, + models: result.models, + raw: result.raw + }; + } +} + +export const serversModelsStore = new ServersModelsStore(); diff --git a/ui/apps/chat/src/lib/stores/settings/index.svelte.ts b/ui/apps/chat/src/lib/stores/settings/index.svelte.ts index 4cdc095f8971..ed9cce3b780a 100644 --- a/ui/apps/chat/src/lib/stores/settings/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/settings/index.svelte.ts @@ -8,7 +8,7 @@ */ import { SETTING_CONFIG_DEFAULT, SETTINGS_KEYS } from '#lib/constants/index.ts'; -import { ColorMode } from '#lib/enums/index.ts'; +import { ColorMode, ServerKind } from '#lib/enums/index.ts'; import { ParameterSyncService } from '#lib/services/parameter-sync.service.ts'; import { SettingsService } from '#lib/services/settings.service.ts'; import { deviceStore } from '#lib/stores/device.svelte.ts'; @@ -90,6 +90,28 @@ class SettingsStore { ? { ...this.config } : Object.fromEntries(Object.entries(this.config).filter(([key]) => key !== 'apiKey')); + // Servers each carry a credential: an API key, or the access code that + // opens a tunnel to someone's machine. Keys go unless the user opts in; a + // remote server without its code is meaningless, so it goes whole. + if (SETTINGS_KEYS.SERVERS in configToExport && !includeSensitiveData) { + try { + const servers = JSON.parse(configToExport[SETTINGS_KEYS.SERVERS] as string) as Array< + Record + >; + const safeServers = servers + .filter((server) => server.kind !== ServerKind.REMOTE) + .map((server) => { + delete server.apiKey; + + return server; + }); + + configToExport[SETTINGS_KEYS.SERVERS] = JSON.stringify(safeServers); + } catch { + delete (configToExport as Record)[SETTINGS_KEYS.SERVERS]; + } + } + // Handle MCP servers: exclude custom headers unless user opts in if ('mcpServers' in configToExport && !includeSensitiveData) { try { diff --git a/ui/apps/chat/src/lib/stores/tools.svelte.ts b/ui/apps/chat/src/lib/stores/tools.svelte.ts index 0e926cd1673e..3c0f6ebd065a 100644 --- a/ui/apps/chat/src/lib/stores/tools.svelte.ts +++ b/ui/apps/chat/src/lib/stores/tools.svelte.ts @@ -26,9 +26,11 @@ import { type ToolProvider } from '#lib/extensions/index.ts'; import { ToolsService } from '#lib/services/tools.service.ts'; -// direct imports between stores, not via the barrel, to avoid circular deps +import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; import { projectsStore } from '#lib/stores/projects.svelte.ts'; +// direct imports between stores, not via the barrel, to avoid circular deps +import { serversStore } from '#lib/stores/servers.svelte.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; import type { OpenAIToolDefinition, @@ -251,6 +253,16 @@ class ToolsStore { } async fetchServerTools(): Promise { + // the /tools endpoint only exists on the local llama.cpp server, so the + // list follows the built-in server rather than the selected model + if (!serverStore.hasLocalServer || !serversStore.local.enabled) { + this._serverToolInfos = []; + this._serverTools = []; + this.cwdAwareTools = new SvelteSet(); + + return; + } + if (this._loading) return; this._loading = true; diff --git a/ui/apps/chat/src/lib/stores/ui.svelte.ts b/ui/apps/chat/src/lib/stores/ui.svelte.ts index ac33944a0e44..fb7c3d3e6a9d 100644 --- a/ui/apps/chat/src/lib/stores/ui.svelte.ts +++ b/ui/apps/chat/src/lib/stores/ui.svelte.ts @@ -121,6 +121,8 @@ class UiStore { projectDialog = $state(null); /** Open state of the project picker, a phone's stand-in for the project submenus. */ projectPicker = $state(null); + /** Open state of the servers dialog, the door to pairing on a phone. */ + serversOpen = $state(false); /** Section ids in the order the user dragged them into, kept across reloads. */ sidebarSectionOrder = $state(loadSidebarSectionOrder()); /** Expanded desktop sidebar width in px; null keeps the stylesheet default. */ @@ -263,6 +265,17 @@ class UiStore { this.projectPicker = options; } + /** Open the servers, where a server is paired or pointed at: a page on a phone, a dialog elsewhere. */ + openServers(): void { + if (deviceStore.isMobile) { + void goto(ROUTES.SERVERS); + + return; + } + + this.serversOpen = true; + } + /** * Puts sections in the order the user gave them. One never moved, such as a * new group or project, keeps its default place: right before the next section in @@ -298,6 +311,7 @@ class UiStore { this.modelsReturnRoute = page.url.hash || ROUTES.START; } + /** Ask the composer to take focus, e.g. after a dialog closes onto the chat. */ requestComposerFocus(): void { this.composerFocusRequested = true; diff --git a/ui/apps/chat/src/lib/types/api.d.ts b/ui/apps/chat/src/lib/types/api.d.ts index 371913fc4c26..c47aed3bd79b 100644 --- a/ui/apps/chat/src/lib/types/api.d.ts +++ b/ui/apps/chat/src/lib/types/api.d.ts @@ -307,6 +307,11 @@ export interface ApiLlamaCppServerProps { webui_settings?: Record; ui_settings?: Record; cors_proxy_enabled?: boolean; + /** + * Access code a host started with `--connect` offers, so another device can + * pair with it. Absent on a server that does not run a host. + */ + connect_code?: string; } export interface ApiChatCompletionRequest { @@ -382,7 +387,9 @@ export interface ApiChatCompletionStreamChunk { metadata?: { model?: string }; delta: { content?: string; + reasoning?: string; reasoning_content?: string; + reasoning_text?: string; model?: string; tool_calls?: ApiChatCompletionToolCallDelta[]; }; @@ -396,6 +403,17 @@ export interface ApiChatCompletionStreamChunk { cache_n?: number; }; prompt_progress?: ChatMessagePromptProgress; + /** Token counts, sent by OpenAI-compatible servers on the final chunk. */ + usage?: ApiChatCompletionUsage; +} + +export interface ApiChatCompletionUsage { + cached_tokens?: number; + completion_tokens?: number; + prompt_cache_hit_tokens?: number; + prompt_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number }; + total_tokens?: number; } export interface ApiChatCompletionResponse { diff --git a/ui/apps/chat/src/lib/types/index.ts b/ui/apps/chat/src/lib/types/index.ts index ada84091cc94..bddea98d76da 100644 --- a/ui/apps/chat/src/lib/types/index.ts +++ b/ui/apps/chat/src/lib/types/index.ts @@ -26,6 +26,7 @@ export type { ApiChatCompletionToolCallDelta, ApiChatCompletionToolCall, ApiChatCompletionStreamChunk, + ApiChatCompletionUsage, ApiChatCompletionResponse, ApiSlotData, ApiProcessingState, @@ -36,6 +37,9 @@ export type { ApiStreamSession } from './api'; +// Server types +export type { Server } from './servers'; + // HuggingFace types export type { HfCatalogBuild, diff --git a/ui/apps/chat/src/lib/types/models.d.ts b/ui/apps/chat/src/lib/types/models.d.ts index 9d4a7c88882b..49ba8e4e27e8 100644 --- a/ui/apps/chat/src/lib/types/models.d.ts +++ b/ui/apps/chat/src/lib/types/models.d.ts @@ -16,9 +16,11 @@ export interface ModelOption { id: string; name: string; model: string; + /** Server that serves this model; set on the aggregated option list. */ + serverId?: string; description?: string; capabilities: string[]; - /** Context size reported by the provider's model listing, when it reports one. */ + /** Context size reported by the server's model listing, when it reports one. */ contextLength?: number; /** Draft sidecars downloaded alongside the model, e.g. `mtp` or `dflash`. */ draftSidecars?: ModelSidecarBadge[]; @@ -27,6 +29,8 @@ export interface ModelOption { speaks?: boolean; details?: ApiModelDetails['details']; meta?: ApiModelDataEntry['meta']; + /** Load state the server's own listing reports, when it reports one. */ + status?: ApiModelDataEntry['status']; parsedId?: ParsedModelId; aliases?: string[]; tags?: string[]; @@ -58,6 +62,21 @@ export interface ModelDownloadEntry { repoWithTag: string; } +/** + * A draft sidecar file a model listing reports as its own entry. The router lists a + * downloaded sidecar as a model, so this is what pairs it back with its model. + */ +export interface ModelSidecarFile { + id: string; + kind: ModelSidecar; + /** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */ + model: string; + /** Parameter count the sidecar reports, e.g. `35B-A3B`. */ + params: string | null; + /** Quantization of the sidecar file, e.g. `Q4_0`. */ + quant: string | null; +} + export interface ParsedModelId { raw: string; orgName: string | null; @@ -86,15 +105,3 @@ export interface ModalityCapabilities { hasAudio: boolean; hasVideo: boolean; } - -/** Sidecar file a listing reports as its own model entry, paired back to its model. */ -export interface ModelSidecarFile { - id: string; - kind: ModelSidecar; - /** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */ - model: string; - /** Parameter count the sidecar reports, e.g. `35B-A3B`. */ - params: string | null; - /** Quantization of the sidecar file, e.g. `Q4_0`. */ - quant: string | null; -} diff --git a/ui/apps/chat/src/lib/types/servers.d.ts b/ui/apps/chat/src/lib/types/servers.d.ts new file mode 100644 index 000000000000..7eceddcf6c9f --- /dev/null +++ b/ui/apps/chat/src/lib/types/servers.d.ts @@ -0,0 +1,23 @@ +import type { ServerKind } from '#lib/enums/index.ts'; + +/** + * A llama.cpp server the UI talks to: the one serving this app, one the user + * pointed at by URL, or one reached over a WebRTC tunnel by its access code. + * All of them speak the llama.cpp API, so nothing is negotiated at runtime. + */ +export interface Server { + /** Sent as `Authorization: Bearer ` when set. */ + apiKey?: string; + /** Endpoint origin, no trailing slash; empty for the built-in and remote servers. */ + baseUrl: string; + /** Remote servers only: 40-character access code of the `llama-server --connect` host. */ + code?: string; + enabled: boolean; + /** Extra request headers sent verbatim, e.g. an authenticating proxy. */ + headers?: Record; + id: string; + /** How the server is reached; absent in stored data means `url`. */ + kind: ServerKind; + modelCount?: number; + name: string; +} diff --git a/ui/apps/chat/src/lib/types/settings.d.ts b/ui/apps/chat/src/lib/types/settings.d.ts index cca82cd997bd..d559b3f69227 100644 --- a/ui/apps/chat/src/lib/types/settings.d.ts +++ b/ui/apps/chat/src/lib/types/settings.d.ts @@ -29,6 +29,8 @@ export interface SettingsEntry { /** Options rendered for RADIO fields. Each entry maps a `value` (the radio's selected value) to the underlying config `key` whose boolean state mirrors it. */ radioOptions?: Array<{ value: string; label: string; key: string; isExperimental?: boolean }>; isExperimental?: boolean; + /** Caveat shown under the help text, e.g. a known security limitation. */ + note?: string; isPositiveInteger?: boolean; /** When true, the field is rendered as a password input (e.g. API key). */ isPrivate?: boolean; @@ -59,6 +61,8 @@ export interface SettingsFieldConfig { label: string; type: SettingsFieldType; isExperimental?: boolean; + /** Caveat shown under the help text, e.g. a known security limitation. */ + note?: string; isPositiveInteger?: boolean; isPrivate?: boolean; placeholder?: string; diff --git a/ui/apps/chat/src/lib/utils/api-base.ts b/ui/apps/chat/src/lib/utils/api-base.ts new file mode 100644 index 000000000000..0a6265b94513 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/api-base.ts @@ -0,0 +1,110 @@ +/** + * API base resolution for servers. + * + * The UI can talk to more than one server endpoint. Services build request + * URLs through {@link apiUrl} so a request always targets the right server. + * The servers store registers a resolver here; this module never imports the + * store, which keeps URL resolution free of store dependencies. + */ + +import { withBase } from './base-path'; +import { isLocalServer, serverChatUrl, serverModelsUrl, serverOrigin } from './servers'; +import { API_ABSOLUTE_URL_PROTOCOLS, API_CHAT, API_MODELS } from '#lib/constants/index.ts'; +import type { Server } from '#lib/types/index.ts'; + +/** Server list and active selection as exposed to URL resolution. */ +export interface ServersSnapshot { + activeId: string; + servers: Server[]; +} + +type ServersResolver = () => ServersSnapshot; + +let resolveServers: ServersResolver | null = null; + +/** Registered once by the servers store. */ +export function setServersResolver(resolver: ServersResolver | null): void { + resolveServers = resolver; +} + +/** + * Look up a server by id, defaulting to the active one. + */ +export function getServer(serverId?: string): Server | undefined { + const snapshot = resolveServers?.(); + + if (!snapshot) return undefined; + + const id = serverId ?? snapshot.activeId; + + return snapshot.servers.find((server) => server.id === id); +} + +/** Every registered server, in list order. */ +export function getServers(): Server[] { + return resolveServers?.().servers ?? []; +} + +/** + * API root for a server: its own base URL, the synthetic tunnel origin of a + * remote server, or an empty string for the local one. + */ +export function getServerBaseUrl(serverId?: string): string { + const server = getServer(serverId); + + return server ? serverOrigin(server) : ''; +} + +/** + * Request target for a server's chat completions endpoint. Returns a relative + * path for the local server and an absolute URL for every other one — its own + * base URL, or the tunnel origin of a remote server — so callers can hand the + * result straight to `fetch` (or to `apiFetch`, which resolves relative paths + * against the base). + */ +export function apiChatUrl(serverId?: string): string { + const server = getServer(serverId); + + if (server && !isLocalServer(server)) { + return serverChatUrl(server); + } + + return apiUrl(API_CHAT.COMPLETIONS, serverId); +} + +/** + * Request target for a server's models listing. Returns a plain path for the + * local server (so `apiFetch` applies the base) and an absolute URL otherwise. + */ +export function apiModelsUrl(serverId?: string): string { + const server = getServer(serverId); + + if (server && !isLocalServer(server)) { + return serverModelsUrl(server); + } + + return API_MODELS.LIST; +} + +/** + * Absolute URL for an API path on a server. + * + * Absolute URLs pass through untouched. Paths on the local server keep the + * existing base-path-relative form, so serving under a subpath still works. + * Paths on external servers resolve against the server's API root. + */ +export function apiUrl(path: string, serverId?: string): string { + if (API_ABSOLUTE_URL_PROTOCOLS.some((protocol) => path.startsWith(protocol))) { + return path; + } + + const baseUrl = getServerBaseUrl(serverId); + + if (!baseUrl) { + return withBase(path); + } + + const root = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`; + + return new URL(path.replace(/^\.?\//, ''), root).toString(); +} diff --git a/ui/apps/chat/src/lib/utils/api-fetch.ts b/ui/apps/chat/src/lib/utils/api-fetch.ts index 85534558a80f..0f90d910e16a 100644 --- a/ui/apps/chat/src/lib/utils/api-fetch.ts +++ b/ui/apps/chat/src/lib/utils/api-fetch.ts @@ -1,10 +1,6 @@ +import { apiUrl } from './api-base'; import { getAuthHeaders, getJsonHeaders } from './api-headers'; -import { withBase } from './base-path'; -import { - API_ABSOLUTE_URL_PROTOCOLS, - ERROR_MESSAGES, - HTTP_CODE_TO_STRING -} from '#lib/constants/index.ts'; +import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '#lib/constants/index.ts'; /** * API Fetch Utilities @@ -36,6 +32,8 @@ export interface ApiFetchOptions extends Omit { * Default: false (uses JSON headers with Content-Type: application/json) */ authOnly?: boolean; + /** Server to target; defaults to the active one. */ + serverId?: string; /** * Additional headers to merge with default headers. */ @@ -63,11 +61,10 @@ export interface ApiFetchOptions extends Omit { * ``` */ export async function apiFetch(path: string, options: ApiFetchOptions = {}): Promise { - const { authOnly = false, headers: customHeaders, ...fetchOptions } = options; - const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); + const { authOnly = false, headers: customHeaders, serverId, ...fetchOptions } = options; + const baseHeaders = authOnly ? getAuthHeaders(serverId) : getJsonHeaders(serverId); const headers = { ...baseHeaders, ...customHeaders }; - // absolute URLs with an allowed protocol pass through untouched; relative paths get the base prefix - const url = API_ABSOLUTE_URL_PROTOCOLS.some((p) => path.startsWith(p)) ? path : withBase(path); + const url = apiUrl(path, serverId); let response; @@ -110,7 +107,7 @@ export async function apiFetchWithParams( params: Record, options: ApiFetchOptions = {} ): Promise { - const url = new URL(basePath, window.location.href); + const url = new URL(apiUrl(basePath, options.serverId), window.location.href); for (const [key, value] of Object.entries(params)) { if (value !== undefined && value !== null) { diff --git a/ui/apps/chat/src/lib/utils/api-headers.ts b/ui/apps/chat/src/lib/utils/api-headers.ts index fa7f1b91748a..d7903f9e7242 100644 --- a/ui/apps/chat/src/lib/utils/api-headers.ts +++ b/ui/apps/chat/src/lib/utils/api-headers.ts @@ -1,26 +1,45 @@ +import { getServer } from './api-base'; import { redactValue } from './redact'; import { CORS_PROXY, HEADERS } from '#lib/constants/index.ts'; import { MimeTypeApplication } from '#lib/enums/index.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import type { Server } from '#lib/types/index.ts'; /** - * Get authorization headers for API requests - * Includes Bearer token if API key is configured + * Get authorization headers for API requests to a server. */ -export function getAuthHeaders(): Record { - const currentConfig = settingsStore.config; - const apiKey = currentConfig.apiKey?.toString().trim(); +export function getAuthHeaders(serverId?: string): Record { + const server = getServer(serverId); + + if (server) return getAuthHeadersForServer(server); + + // no servers resolver yet (early startup, or a non-browser call): keep the + // pre-servers behaviour and authenticate against the serving origin + const apiKey = settingsStore.config.apiKey?.toString().trim(); return apiKey ? { [HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKey}` } : {}; } +/** + * Get authorization headers for a server object, including one that is not + * registered yet (used on the add-server form). + */ +export function getAuthHeadersForServer(server: Server): Record { + const headers: Record = { ...(server.headers ?? {}) }; + const apiKey = server.apiKey?.trim(); + + if (apiKey) headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`; + + return headers; +} + /** * Get standard JSON headers with optional authorization */ -export function getJsonHeaders(): Record { +export function getJsonHeaders(serverId?: string): Record { return { [HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON, - ...getAuthHeaders() + ...getAuthHeaders(serverId) }; } diff --git a/ui/apps/chat/src/lib/utils/api-key-validation.ts b/ui/apps/chat/src/lib/utils/api-key-validation.ts index 2783b32414ee..f6c938557424 100644 --- a/ui/apps/chat/src/lib/utils/api-key-validation.ts +++ b/ui/apps/chat/src/lib/utils/api-key-validation.ts @@ -1,34 +1,36 @@ -import { withBase } from './base-path'; import { error } from '@sveltejs/kit'; -import { HEADERS } from '#lib/constants/index.ts'; +import { HEADERS, LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; import { MimeTypeApplication } from '#lib/enums/index.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import { apiUrl } from '#lib/utils/api-base.ts'; import { browser } from '$app/env'; /** - * Validates API key by making a request to the server props endpoint - * Throws SvelteKit errors for authentication failures or server issues + * Validates the stored API key against the server serving this UI. + * + * Pinned to the built-in server, the way the model status feed is: with a + * remote server active, probing *that* server with this key would raise the + * API key splash over a credential that belongs to a different machine. The + * servers a user added carry their own key, and their failures surface on + * their own card instead. + * + * Throws SvelteKit errors for authentication failures or server issues. */ export async function validateApiKey(fetch: typeof globalThis.fetch): Promise { if (!browser) { return; } - const apiKey = settingsStore.config.apiKey; + const headers: Record = { [HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON }; + // Probe /props even without a stored key: on a server started with --api-key + // the unauthenticated request returns 401 and surfaces the API key splash, + // which is the onboarding path for entering the key. + const apiKey = settingsStore.config.apiKey?.toString().trim(); - try { - const headers: Record = { - [HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON - }; - - // Probe /props even without a stored key: on a server started with - // --api-key the unauthenticated request returns 401 and surfaces the - // API key splash, which is the onboarding path for entering the key. - if (apiKey) { - headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`; - } + if (apiKey) headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`; - const response = await fetch(withBase('/props'), { headers }); + try { + const response = await fetch(apiUrl('/props', LOCAL_BACKEND_ID), { headers }); if (!response.ok) { if (response.status === 401 || response.status === 403) { @@ -40,12 +42,10 @@ export async function validateApiKey(fetch: typeof globalThis.fetch): Promise(); + const rank = new Map(); + + recentIds.forEach((id, index) => rank.set(id, index)); + + const rankOf = (id: string) => rank.get(id) ?? Number.MAX_SAFE_INTEGER; + // recently used models lead their section, the rest keep the server's order + const byRecency = (items: ModelItem[]) => + rank.size === 0 ? items : [...items].sort((a, b) => rankOf(a.option.id) - rankOf(b.option.id)); + + for (const option of options) { + const serverId = option.serverId ?? ''; + + if (!byServer.has(serverId)) byServer.set(serverId, []); + + byServer.get(serverId)!.push({ option }); + } + + return servers.map((server) => { + const items = byRecency(byServer.get(server.serverId) ?? []); + + return { ...server, items: items.slice(0, limit), matched: items.length }; + }); } diff --git a/ui/apps/chat/src/lib/utils/model-option-id.ts b/ui/apps/chat/src/lib/utils/model-option-id.ts new file mode 100644 index 000000000000..05ad3123aa31 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/model-option-id.ts @@ -0,0 +1,28 @@ +/** + * Server-qualified model option ids. + * + * The selector lists models from every enabled server, so a bare model id can + * collide. Option ids are qualified as `::`; selection + * strips the prefix to find the row in the active server's list. + */ + +const MODEL_OPTION_ID_SEPARATOR = '::'; + +/** Prefix a raw model id with the server that serves it. */ +export function qualifyModelId(serverId: string, modelId: string): string { + return `${serverId}${MODEL_OPTION_ID_SEPARATOR}${modelId}`; +} + +/** Server id of a qualified model option id, or null when unqualified. */ +export function serverIdFromModelId(qualifiedId: string): string | null { + const index = qualifiedId.indexOf(MODEL_OPTION_ID_SEPARATOR); + + return index === -1 ? null : qualifiedId.slice(0, index); +} + +/** Strip the server prefix from a qualified model option id. */ +export function rawModelId(qualifiedId: string): string { + const index = qualifiedId.indexOf(MODEL_OPTION_ID_SEPARATOR); + + return index === -1 ? qualifiedId : qualifiedId.slice(index + MODEL_OPTION_ID_SEPARATOR.length); +} diff --git a/ui/apps/chat/src/lib/utils/qr-code.ts b/ui/apps/chat/src/lib/utils/qr-code.ts new file mode 100644 index 000000000000..94d002490bed --- /dev/null +++ b/ui/apps/chat/src/lib/utils/qr-code.ts @@ -0,0 +1,39 @@ +/** + * QR code of an access code, for carrying it from the screen it is shown on to + * the camera of another device. + * + * The payload is the code itself, in the shape the scanner accepts: the scanner + * hands whatever it decodes to `normalizeShareCode`, so a code pasted by hand + * and a code scanned off a screen go through the same door. + */ + +import { normalizeShareCode } from './servers'; +import { renderSVG } from 'uqr'; + +/** Module size of the rendered code, in SVG units. The image is sized by CSS. */ +const PIXEL_SIZE = 4; +/** Quiet zone, in modules on each side; the spec asks for four, a camera needs them. */ +const BORDER = 4; + +/** + * Data URL of an SVG QR code carrying the access code, or null when the value + * is not a usable code: a QR a phone cannot pair with is worse than no QR. + * + * Black on white, pinned: a scanner reads dark modules on a light field, and + * the app's dark theme must not invert that. + */ +export function qrCodeDataUrl(code: string): string | null { + const normalized = normalizeShareCode(code); + + if (!normalized) return null; + + const svg = renderSVG(normalized, { + blackColor: '#000000', + border: BORDER, + ecc: 'M', + pixelSize: PIXEL_SIZE, + whiteColor: '#ffffff' + }); + + return `data:image/svg+xml;utf8,${encodeURIComponent(svg)}`; +} diff --git a/ui/apps/chat/src/lib/utils/qr-scan.ts b/ui/apps/chat/src/lib/utils/qr-scan.ts new file mode 100644 index 000000000000..69fbfc9c5909 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/qr-scan.ts @@ -0,0 +1,58 @@ +/** + * Access code scanning from the camera, without a dependency. + * + * Browsers that ship a native `BarcodeDetector` can read the QR code + * `llama-server --connect` prints; everywhere else the field is filled by + * hand, which is the same thing the user does when they copy the code from a + * terminal. The code alphabet excludes the characters that a camera frame is + * most likely to misread, so a miss is a length or charset failure and the + * field simply stays invalid. + */ + +/** Minimal shape of the native detector; it is not in every lib.dom. */ +interface BarcodeDetectorLike { + detect(source: CanvasImageSource): Promise>; +} + +type BarcodeDetectorConstructor = new (options?: { formats?: string[] }) => BarcodeDetectorLike; + +function detectorConstructor(): BarcodeDetectorConstructor | null { + if (typeof window === 'undefined') return null; + + const candidate = (window as unknown as { BarcodeDetector?: BarcodeDetectorConstructor }) + .BarcodeDetector; + + return typeof candidate === 'function' ? candidate : null; +} + +/** Whether this browser can read a QR code out of a camera frame. */ +export function supportsQrScan(): boolean { + return detectorConstructor() !== null; +} + +/** + * Read a QR code out of one video frame. + * + * @returns The decoded value, or null when the frame carries none. + */ +export async function scanFrameForCode(video: HTMLVideoElement): Promise { + const Detector = detectorConstructor(); + + // HAVE_CURRENT_DATA and up: nothing to read out of an empty frame + if (!Detector || video.readyState < 2) return null; + + try { + const barcodes = await new Detector({ formats: ['qr_code'] }).detect(video); + + for (const barcode of barcodes) { + const value = barcode.rawValue ?? barcode.value; + + if (value) return value; + } + + return null; + } catch { + // a frame the detector cannot read is not an error worth surfacing + return null; + } +} diff --git a/ui/apps/chat/src/lib/utils/server-props.ts b/ui/apps/chat/src/lib/utils/server-props.ts new file mode 100644 index 000000000000..51a6d109a784 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/server-props.ts @@ -0,0 +1,26 @@ +/** + * Whether the endpoint behind a server really is a llama.cpp server. + * + * The UI drives llama.cpp endpoints only: it loads and unloads models, reads + * the router's feed and its chat template. An endpoint that speaks just the + * OpenAI API answers /v1/models happily and then breaks somewhere else, so the + * check is made where the UI can still say no - before the server is saved. + */ + +function isJsonObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Whether a /props payload is a llama.cpp one. + * + * Both a model and a router answer with their generation defaults; a router + * builds its props by hand, so the role and the build string stand in for them. + */ +export function isLlamaCppProps(value: unknown): boolean { + if (!isJsonObject(value)) return false; + + if (isJsonObject(value.default_generation_settings)) return true; + + return typeof value.role === 'string' && typeof value.build_info === 'string'; +} diff --git a/ui/apps/chat/src/lib/utils/server-status.ts b/ui/apps/chat/src/lib/utils/server-status.ts new file mode 100644 index 000000000000..414ef05011d1 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/server-status.ts @@ -0,0 +1,102 @@ +/** + * What a server card reports: the pill word, the sentence under it and the + * state the card's controls hang off. + * + * The words carry the weight here. A failed channel and an unreachable server + * need different repairs, and the transport's own sentence - a refused + * passcode, a tracker nobody answered - is the one the user can act on, so it + * is reported as it came rather than folded into a generic label. + */ + +import { ServerCardState, TunnelStatus } from '#lib/enums/index.ts'; + +/** Everything the card knows about a server when it has to name its state. */ +export interface ServerCardFacts { + enabled: boolean; + /** Models the listing reported. */ + modelCount: number; + /** The listing's failure, empty while it answered or was never asked. */ + modelsError: string; + modelsLoaded: boolean; + modelsLoading: boolean; + /** Whether the server is reached over a tunnel. */ + remote: boolean; + tunnel: TunnelStatus; + /** The tunnel's failure, empty while it never failed. */ + tunnelError: string; +} + +export interface ServerCardStatus { + /** Sentence under the pill; empty when the state needs none. */ + detail: string; + /** Pill word; empty for a state the card does not report. */ + label: string; + state: ServerCardState; +} + +/** State a card reports for a server, with the words that state is shown in. */ +export function serverCardStatusOf(facts: ServerCardFacts): ServerCardStatus { + const { + enabled, + modelCount, + modelsError, + modelsLoaded, + modelsLoading, + remote, + tunnel, + tunnelError + } = facts; + const connecting = remote && tunnel === TunnelStatus.CONNECTING; + const idle = + remote && tunnel === TunnelStatus.IDLE && !modelsLoading && !modelsLoaded && !modelsError; + const checking = enabled && !connecting && !idle && !modelsLoaded && !modelsError && !tunnelError; + const online = + enabled && modelsLoaded && !modelsError && (!remote || tunnel === TunnelStatus.CONNECTED); + const offline = enabled && (tunnel === TunnelStatus.ERROR || (!idle && Boolean(modelsError))); + + if (connecting) { + return { + detail: 'Opening the tunnel to the host.', + label: 'Connecting', + state: ServerCardState.CONNECTING + }; + } + + if (checking) { + return { + detail: remote + ? 'Asking the host for the models it serves.' + : 'Asking the server for the models it serves.', + label: 'Checking', + state: ServerCardState.CHECKING + }; + } + + if (online) { + return { + detail: '', + label: modelCount === 1 ? '1 model' : `${modelCount} models`, + state: ServerCardState.ONLINE + }; + } + + if (offline) { + // the channel failing is the one failure the user can act on: the code may + // have expired, or the host may not be listening + if (remote && tunnel === TunnelStatus.ERROR) { + return { detail: tunnelError, label: 'Channel down', state: ServerCardState.CHANNEL_DOWN }; + } + + return { detail: modelsError, label: 'Unreachable', state: ServerCardState.UNREACHABLE }; + } + + if (idle) { + return { + detail: 'Paste the code the host prints, or press Connect.', + label: 'Not connected', + state: ServerCardState.IDLE + }; + } + + return { detail: '', label: '', state: ServerCardState.NONE }; +} diff --git a/ui/apps/chat/src/lib/utils/servers.ts b/ui/apps/chat/src/lib/utils/servers.ts new file mode 100644 index 000000000000..b5dd6567f154 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/servers.ts @@ -0,0 +1,393 @@ +/** + * Server list parsing, defaults and endpoint URLs. + * + * External servers are persisted in settings as a JSON list. Malformed + * entries are dropped instead of throwing so a corrupted settings value can + * never break URL resolution. + * + * A remote server has no URL of its own: it is reached through a WebRTC + * tunnel and addressed by a synthetic origin that carries its id + * (`tunnelOrigin`). The host suffix is reserved, so the address can never be + * mistaken for a real server, and a request that escapes the tunnel transport + * fails in the browser instead of reaching the server serving this UI. + */ + +import { + CODE_CHARS, + CODE_LENGTHS, + DEFAULT_SERVER_CHAT_PATH, + DEFAULT_SERVER_MODELS_PATH, + FAVICON_SERVICE_URL, + LOCAL_BACKEND_ID, + SERVER_ID_PREFIX, + TUNNEL_HOST_SUFFIX +} from '#lib/constants/index.ts'; +import { ServerKind } from '#lib/enums/index.ts'; +import type { Server } from '#lib/types/index.ts'; + +/** Icon size requested from the favicon service, shown at 16px. */ +const FAVICON_SIZE = 64; +/** Longest host label a server id can be shortened to (DNS label limit). */ +const HOST_LABEL_MAX = 63; + +/** Absolute chat completions URL for a server. */ +export function serverChatUrl(server: Server): string { + return joinServerUrl(serverOrigin(server), DEFAULT_SERVER_CHAT_PATH); +} + +/** Absolute models listing URL for a server. */ +export function serverModelsUrl(server: Server): string { + return joinServerUrl(serverOrigin(server), DEFAULT_SERVER_MODELS_PATH); +} + +/** + * Origin a server's requests are addressed at: its own base URL, or the + * synthetic tunnel origin of a remote server. Empty for the built-in + * server, whose requests stay on the origin serving this UI. + */ +export function serverOrigin(server: Server): string { + if (isRemoteServer(server)) return tunnelOrigin(server.id); + + return server.baseUrl.trim().replace(/\/+$/, ''); +} + +/** + * Favicon of a server's root domain, used when no bundled mark matches. Hosts + * rarely serve a favicon themselves, so the subdomain is dropped + * (`api.z.ai` -> `z.ai`) and the icon is requested through a favicon service. + * Returns null when the URL carries no usable host, which is always the case + * for a remote server: a tunnel serves no favicon. + */ +export function serverFaviconUrl(baseUrl: string): string | null { + if (isTunnelHost(hostOf(baseUrl))) return null; + + try { + const labels = new URL(baseUrl).hostname.split('.'); + const root = labels.length > 2 ? labels.slice(-2).join('.') : labels.join('.'); + + return root ? `${FAVICON_SERVICE_URL}${root}&sz=${FAVICON_SIZE}` : null; + } catch { + return null; + } +} + +/** + * The access code of a remote server, trimmed of the spacing a user tends to + * paste with. Empty when the value is not a usable code. + */ +export function codeOf(server?: Server | null): string { + return normalizeShareCode(server?.code ?? ''); +} + +/** The built-in server pointing at the server that serves this UI. */ +export function createLocalServer(apiKey?: string, enabled = true): Server { + return { + apiKey, + baseUrl: '', + enabled, + id: LOCAL_BACKEND_ID, + kind: ServerKind.LOCAL, + name: 'Local' + }; +} + +/** A remote llama.cpp server, reached through the tunnel of its access code. */ +export function createRemoteServer( + code: string, + id: string, + name?: string, + apiKey?: string +): Server { + return { + apiKey, + baseUrl: '', + code: normalizeShareCode(code), + enabled: true, + id, + kind: ServerKind.REMOTE, + name: name?.trim() || 'Remote server' + }; +} + +/** A llama.cpp server answering on its own base URL. */ +export function createUrlServer( + baseUrl: string, + id: string, + name?: string, + apiKey?: string, + headers?: Record +): Server { + return { + apiKey, + baseUrl: baseUrl.trim().replace(/\/+$/, ''), + enabled: true, + headers: headers && Object.keys(headers).length > 0 ? headers : undefined, + id, + kind: ServerKind.URL, + name: name?.trim() || hostOf(baseUrl) || baseUrl.trim() + }; +} + +/** Whether the built-in server is the one serving this UI. */ +export function isLocalServer(server?: Server | null): boolean { + return !server || server.kind === ServerKind.LOCAL; +} + +/** Whether a server is reached through a WebRTC tunnel. */ +export function isRemoteServer(server?: Server | null): boolean { + return server?.kind === ServerKind.REMOTE; +} + +/** + * Whether a server can be reached at all: the built-in one always, a remote + * one with a valid access code, and a URL one with a URL that parses and does + * not squat on the tunnel host suffix. + */ +export function isReachableServer(server: Server): boolean { + if (isLocalServer(server)) return true; + + if (isRemoteServer(server)) return Boolean(codeOf(server)); + + const baseUrl = server.baseUrl.trim(); + + if (!baseUrl || isTunnelHost(hostOf(baseUrl))) return false; + + return isHttpUrl(baseUrl); +} + +/** Host of a URL, empty when it does not parse. */ +export function hostOf(url: string): string { + try { + return new URL(url).hostname; + } catch { + return ''; + } +} + +/** Whether a value parses as an http(s) URL. */ +export function isHttpUrl(value: string): boolean { + try { + const { protocol } = new URL(value); + + return protocol === 'http:' || protocol === 'https:'; + } catch { + return false; + } +} + +/** + * Strip the spacing from a pasted access code and check it against the alphabet + * llama-server generates its code from. Empty when the value is not a code. + */ +export function normalizeShareCode(code: string): string { + const code_ = code.replace(/\s/g, ''); + + if (code_.length !== CODE_LENGTHS.SHARE) return ''; + + for (const char of code_) { + if (!CODE_CHARS.includes(char)) return ''; + } + + return code_; +} + +/** + * Host label a server's tunnel origin carries. Exported so a host coming back + * from the transport can be matched against a server without un-sanitizing it. + */ +export function tunnelHostLabel(serverId: string): string { + return hostLabel(serverId); +} + +/** + * Host label of a tunnel origin, or null when the request is not addressed at + * one. The label is a server id in its hostname-safe form, which is how + * `tunnelOrigin` wrote it. + */ +export function parseTunnelOrigin(url: RequestInfo | URL): string | null { + const href = url instanceof URL ? url.href : typeof url === 'string' ? url : url.url; + const host = hostOf(href); + + if (!isTunnelHost(host)) return null; + + return host.slice(0, host.length - TUNNEL_HOST_SUFFIX.length - 1); +} + +function joinServerUrl(origin: string, path: string): string { + const suffix = path.startsWith('/') ? path : `/${path}`; + + return `${origin}${suffix}`; +} + +/** Hostname-safe form of a server id; both sides of the comparison use it. */ +function hostLabel(serverId: string): string { + return ( + serverId + .trim() + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, HOST_LABEL_MAX) || 'server' + ); +} + +function isTunnelHost(host: string): boolean { + return host === TUNNEL_HOST_SUFFIX || host.endsWith(`.${TUNNEL_HOST_SUFFIX}`); +} + +function parseServerEntry(entry: unknown, index: number): Server | null { + if (!entry || typeof entry !== 'object') return null; + + const raw = entry as Record; + const id = + typeof raw.id === 'string' && raw.id.trim() + ? raw.id.trim() + : `${SERVER_ID_PREFIX}-${index + 1}`; + const name = typeof raw.name === 'string' && raw.name.trim() ? raw.name.trim() : ''; + const enabled = raw.enabled !== false; + const headers = parseServerHeaders(raw.headers); + const code = typeof raw.code === 'string' ? normalizeShareCode(raw.code) : ''; + const baseUrl = typeof raw.baseUrl === 'string' ? raw.baseUrl.trim().replace(/\/+$/, '') : ''; + // a server's key is its credential: dropping it here would silently turn + // every request into an unauthenticated one + const apiKey = + typeof raw.apiKey === 'string' && raw.apiKey.trim() ? raw.apiKey.trim() : undefined; + + // a remote server is its code; without a usable one it is nothing + if (raw.kind === ServerKind.REMOTE) { + if (!code) return null; + + return { + apiKey, + baseUrl: '', + code, + enabled, + headers, + id, + kind: ServerKind.REMOTE, + name: name || 'Remote server' + }; + } + + // the local server is built in and never persisted + if (!baseUrl || id === LOCAL_BACKEND_ID) return null; + + // a URL server must not squat on the host the tunnel transport routes + if (isTunnelHost(hostOf(baseUrl)) || !isHttpUrl(baseUrl)) return null; + + return { + apiKey, + baseUrl, + enabled, + headers, + id, + kind: ServerKind.URL, + name: name || hostOf(baseUrl) + }; +} + +function parseServerHeaders(raw: unknown): Record | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + + const entries = Object.entries(raw as Record) + .filter(([, value]) => typeof value === 'string' && value.trim() !== '') + .map(([key, value]) => [key.trim(), (value as string).trim()] as const); + + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + +/** + * The server URLs a `llama-chat.json` lists, without a trailing slash; entries + * that are not http(s) URLs, or squat on the tunnel host, are dropped. An entry + * is `{ "url": "..." }` or the URL itself. + */ +export function parsePresetServers(raw: unknown): string[] { + const list = (raw as { servers?: unknown } | null)?.servers; + + if (!Array.isArray(list)) return []; + + const urls = list.flatMap((entry) => { + const value = typeof entry === 'string' ? entry : (entry as { url?: unknown } | null)?.url; + + if (typeof value !== 'string') return []; + + const url = value.trim().replace(/\/+$/, ''); + + return isHttpUrl(url) && !isTunnelHost(hostOf(url)) ? [url] : []; + }); + + return [...new Set(urls)]; +} + +/** + * The preset URLs to add: those no stored server points at already, and that + * were not added before, so a preset server the user removed stays removed. + */ +export function presetServersToAdd(urls: string[], servers: Server[], seen: string[]): string[] { + const stored = new Set(servers.map((server) => server.baseUrl)); + + return urls.filter((url) => !stored.has(url) && !seen.includes(url)); +} + +/** + * Parse the persisted servers JSON into server entries. + */ +export function parseServersSettings(rawServers: unknown): Server[] { + if (!rawServers) return []; + + let parsed: unknown; + + if (typeof rawServers === 'string') { + const trimmed = rawServers.trim(); + + if (!trimmed) return []; + + try { + parsed = JSON.parse(trimmed); + } catch (error) { + console.warn('[servers] Failed to parse servers JSON, ignoring value:', error); + + return []; + } + } else { + parsed = rawServers; + } + + if (!Array.isArray(parsed)) return []; + + return parsed.flatMap((entry, index) => { + const server = parseServerEntry(entry, index); + + return server ? [server] : []; + }); +} + +/** + * Read a model's context size out of one `/v1/models` entry. A server leaves it + * unknown when it has not loaded the model, and a model-mode listing reports the + * trained size under `meta.n_ctx_train` instead. + */ +export function readModelContextLength(value: unknown): number | undefined { + if (!value || typeof value !== 'object') return undefined; + + const entry = value as Record; + + if (typeof entry.context_length === 'number' && entry.context_length > 0) { + return entry.context_length; + } + + const meta = entry.meta; + + if (meta && typeof meta === 'object') { + const trained = (meta as Record).n_ctx_train; + + if (typeof trained === 'number' && trained > 0) return trained; + } + + return undefined; +} + +/** Origin a remote server's requests are addressed at. */ +export function tunnelOrigin(serverId: string): string { + return `https://${hostLabel(serverId)}.${TUNNEL_HOST_SUFFIX}`; +} diff --git a/ui/apps/chat/src/lib/utils/timings.ts b/ui/apps/chat/src/lib/utils/timings.ts new file mode 100644 index 000000000000..6ecce85d02bc --- /dev/null +++ b/ui/apps/chat/src/lib/utils/timings.ts @@ -0,0 +1,66 @@ +/** + * Client side timing fallback for servers that do not report their own. + * + * llama.cpp streams per-token timings; OpenAI-compatible servers + * do not. Token counts come from the usage block of the final chunk (or the + * count of streamed deltas as a fallback), times are measured locally: the wait + * for the first token is attributed to prompt processing, the rest to + * generation. Wall clock, so network and queueing are part of the numbers. + */ + +import type { ApiChatCompletionUsage } from '#lib/types/api.js'; +import type { ChatMessageTimings } from '#lib/types/chat.js'; + +export interface StreamClock { + startedAt: number; + firstTokenAt: number | null; + lastTokenAt: number | null; +} + +/** + * Prompt/output/cache token counts. `promptTokens` excludes the cache read + * tokens, which are returned separately as `cacheTokens`, so the two always + * add up to the prompt size. + */ +export function usageTokenCounts(usage: ApiChatCompletionUsage | undefined): { + cacheTokens: number; + completionTokens: number; + promptTokens: number; +} { + // a total that includes the cache reads, which are reported separately + const cacheTokens = + usage?.prompt_tokens_details?.cached_tokens ?? + usage?.prompt_cache_hit_tokens ?? + usage?.cached_tokens ?? + 0; + const promptTotal = usage?.prompt_tokens ?? 0; + + return { + cacheTokens, + completionTokens: usage?.completion_tokens ?? 0, + promptTokens: Math.max(0, promptTotal - cacheTokens) + }; +} + +export function buildTimingsFromUsage( + usage: ApiChatCompletionUsage | undefined, + clock: StreamClock, + fallbackTokens = 0 +): ChatMessageTimings | null { + const { cacheTokens, completionTokens, promptTokens } = usageTokenCounts(usage); + const predictedN = completionTokens || fallbackTokens; + + if (promptTokens === 0 && predictedN === 0) return null; + + const { firstTokenAt, startedAt } = clock; + const lastTokenAt = clock.lastTokenAt ?? firstTokenAt; + + return { + cache_n: cacheTokens, + // clamp so a one-token reply still reports a positive duration + predicted_ms: firstTokenAt && lastTokenAt ? Math.max(1, lastTokenAt - firstTokenAt) : undefined, + predicted_n: predictedN, + prompt_ms: firstTokenAt ? Math.max(1, firstTokenAt - startedAt) : undefined, + prompt_n: promptTokens + }; +} diff --git a/ui/apps/chat/src/lib/utils/webrtc-tunnel.ts b/ui/apps/chat/src/lib/utils/webrtc-tunnel.ts new file mode 100644 index 000000000000..252848ca77a4 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/webrtc-tunnel.ts @@ -0,0 +1,490 @@ +/** + * WebRTC tunnel for remote llama.cpp access. + * + * Signaling uses the WebTorrent tracker WebSocket protocol (no external deps). + * The room code is used as the info_hash rendezvous key; the pass code + * authenticates the client on the data channel after the WebRTC handshake. + * + * The client sends an offer, authenticates via pass code, then every request + * addressed at the server's tunnel origin is forwarded through the data + * channel to the host running llama-connect next to its local server. + * + * One instance serves exactly one server, and `remoteAccessStore` owns them: + * a server is addressed by its tunnel origin, so the request path the host + * receives is already clean and needs no base-path handling. + */ + +import { + CODE_CHARS, + CODE_LENGTHS, + ID_LENGTHS, + STUN_URLS, + TRACKER_URLS, + WEBRTC_TIMEOUTS +} from '#lib/constants/index.ts'; + +const STUN_CONFIG: RTCConfiguration = { + iceServers: STUN_URLS.map((urls) => ({ urls })) +}; + +function randomStr(len: number): string { + const bytes = new Uint8Array(len); + + crypto.getRandomValues(bytes); + let result = ''; + + for (const b of bytes) result += CODE_CHARS[b % CODE_CHARS.length]; + + return result; +} + +// info_hash must be exactly 20 chars for WebTorrent trackers +function roomToInfoHash(roomCode: string): string { + return roomCode.padEnd(CODE_LENGTHS.INFO_HASH, '0').slice(0, CODE_LENGTHS.INFO_HASH); +} + +// Non-trickle ICE: the offer carries whatever candidates were gathered when +// the deadline hits, since a single slow candidate must not sink the offer. +function waitForIceComplete(pc: RTCPeerConnection): Promise { + return new Promise((resolve) => { + if (pc.iceGatheringState === 'complete') { + resolve(); + + return; + } + + const timer = setTimeout(resolve, WEBRTC_TIMEOUTS.ICE_GATHER_MS); + + pc.addEventListener('icegatheringstatechange', () => { + if (pc.iceGatheringState === 'complete') { + clearTimeout(timer); + resolve(); + } + }); + }); +} + +// --------------------------------------------------------------------------- +// Tracker signaling (WebTorrent WS tracker protocol) +// --------------------------------------------------------------------------- + +type TrackerMsg = Record; + +class Tracker { + onAnswer?: (offerId: string, answer: RTCSessionDescriptionInit) => void; + onClose?: () => void; + private readonly infoHash: string; + + private readonly peerId: string; + private ws: WebSocket | null = null; + + announce( + opts: { + numwant?: number; + offers?: Array<{ offer_id: string; offer: RTCSessionDescriptionInit }>; + } = {} + ): void { + const msg: TrackerMsg = { + action: 'announce', + info_hash: this.infoHash, + numwant: opts.numwant ?? 0, + peer_id: this.peerId + }; + + if (opts.offers) msg.offers = opts.offers; + + this.send(msg); + } + + close(): void { + this.ws?.close(); + this.ws = null; + } + + connect(url: string): Promise { + return new Promise((resolve, reject) => { + const ws = new WebSocket(url); + + this.ws = ws; + const timer = setTimeout( + () => reject(new Error('tracker connect timeout')), + WEBRTC_TIMEOUTS.TRACKER_CONNECT_MS + ); + + ws.onopen = () => { + clearTimeout(timer); + resolve(); + }; + ws.onerror = () => { + clearTimeout(timer); + reject(new Error('tracker WebSocket error')); + }; + ws.onclose = () => { + this.onClose?.(); + }; + ws.onmessage = (event) => { + try { + const msg = JSON.parse(event.data as string) as TrackerMsg; + + if (msg.answer && msg.offer_id) { + this.onAnswer?.(msg.offer_id as string, msg.answer as RTCSessionDescriptionInit); + } + } catch { + // ignore malformed tracker messages + } + }; + }); + } + + constructor(infoHash: string, peerId: string) { + this.infoHash = infoHash; + this.peerId = peerId; + } + + private send(msg: TrackerMsg): void { + if (this.ws?.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify(msg)); + } + } +} + +// --------------------------------------------------------------------------- +// Tunnel message types (JSON, sent over RTCDataChannel) +// --------------------------------------------------------------------------- + +interface ReqMsg { + type: 'req'; + id: string; + method: string; + path: string; + headers: Record; + body: string | null; // base64 or null +} +interface ResStartMsg { + type: 'res_start'; + id: string; + status: number; + headers: Record; +} +interface ResChunkMsg { + type: 'res_chunk'; + id: string; + data: string; // base64 +} +interface ResEndMsg { + type: 'res_end'; + id: string; +} +interface ResErrMsg { + type: 'res_err'; + id: string; + message: string; +} +interface CancelMsg { + type: 'cancel'; + id: string; +} +interface AuthMsg { + type: 'auth'; + pass: string; +} +interface AuthOkMsg { + type: 'auth_ok'; +} +interface AuthFailMsg { + type: 'auth_fail'; +} + +type TunnelMsg = + | ReqMsg + | ResStartMsg + | ResChunkMsg + | ResEndMsg + | ResErrMsg + | CancelMsg + | AuthMsg + | AuthOkMsg + | AuthFailMsg; + +function uint8ToBase64(bytes: Uint8Array): string { + let binary = ''; + + for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]); + + return btoa(binary); +} + +function base64ToUint8(b64: string): Uint8Array { + return Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)); +} +// --------------------------------------------------------------------------- +// ClientTunnel +// --------------------------------------------------------------------------- + +export type ClientCallbacks = { + onConnected?: () => void; + onDisconnected?: () => void; +}; + +type PendingReq = { + onStart: (status: number, headers: Record) => void; + onChunk: (data: string) => void; + onEnd: () => void; + onError: (message: string) => void; +}; + +export class ClientTunnel { + private readonly callbacks: ClientCallbacks; + private channel: RTCDataChannel | null = null; + private readonly infoHash: string; + private readonly passCode: string; + + private pc: RTCPeerConnection | null = null; + private readonly peerId: string; + private pending = new Map(); + private trackers: Tracker[] = []; + + get isConnected(): boolean { + return this.channel?.readyState === 'open'; + } + + async connect(): Promise { + const offerId = randomStr(ID_LENGTHS.OFFER); + const pc = new RTCPeerConnection(STUN_CONFIG); + + this.pc = pc; + + const channel = pc.createDataChannel('tunnel', { ordered: true }); + + this.channel = channel; + + const offer = await pc.createOffer(); + + await pc.setLocalDescription(offer); + await waitForIceComplete(pc); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error('connection timed out waiting for host')); + }, WEBRTC_TIMEOUTS.CONNECT_MS); + + channel.onopen = () => { + channel.send(JSON.stringify({ pass: this.passCode, type: 'auth' } satisfies AuthMsg)); + }; + + channel.onmessage = (event) => { + try { + const msg = JSON.parse(event.data as string) as TunnelMsg; + + if (msg.type === 'auth_ok') { + clearTimeout(timer); + this.callbacks.onConnected?.(); + resolve(); + } else if (msg.type === 'auth_fail') { + clearTimeout(timer); + reject(new Error('authentication failed: invalid passcode')); + } else { + this.routeResponseMsg(msg); + } + } catch { + // ignore + } + }; + + channel.onclose = () => { + this.callbacks.onDisconnected?.(); + this.rejectAllPending('connection closed'); + }; + + channel.onerror = () => { + clearTimeout(timer); + reject(new Error('data channel error')); + }; + + // The host announces on every tracker, so the same offer goes out to + // all of them at once and the first answer back wins. + let answered = false; + let unreachable = 0; + + for (const url of TRACKER_URLS) { + const tracker = new Tracker(this.infoHash, this.peerId); + + this.trackers.push(tracker); + + tracker.onAnswer = async (answeredOfferId, answer) => { + if (answeredOfferId !== offerId || answered) return; + + answered = true; + + try { + await pc.setRemoteDescription(new RTCSessionDescription(answer)); + } catch (e) { + clearTimeout(timer); + reject(e); + } + }; + + tracker + .connect(url) + .then(() => { + tracker.announce({ + numwant: 1, + offers: [{ offer: pc.localDescription!, offer_id: offerId }] + }); + }) + .catch(() => { + unreachable++; + + if (unreachable === TRACKER_URLS.length) { + clearTimeout(timer); + reject(new Error('no tracker reachable')); + } + }); + } + }); + } + + constructor(roomCode: string, passCode: string, callbacks: ClientCallbacks = {}) { + this.passCode = passCode; + this.infoHash = roomToInfoHash(roomCode); + this.peerId = randomStr(ID_LENGTHS.PEER); + this.callbacks = callbacks; + } + + disconnect(): void { + this.rejectAllPending('disconnected'); + this.cleanupConnection(); + } + + async fetch(input: RequestInfo | URL, init?: RequestInit): Promise { + if (!this.channel || this.channel.readyState !== 'open') { + throw new Error('tunnel not connected'); + } + + const request = new Request(input, init); + const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined); + const id = randomStr(ID_LENGTHS.REQUEST); + + if (signal?.aborted) { + return Promise.reject(new DOMException('Aborted', 'AbortError')); + } + + // Requests reach the tunnel addressed at the server's tunnel origin, so + // the path is already the one the host's server serves: nothing to strip. + const reqUrl = new URL(request.url); + const path = reqUrl.pathname + reqUrl.search; + const headers: Record = {}; + + request.headers.forEach((v, k) => { + headers[k] = v; + }); + + let bodyB64: string | null = null; + + const bodyBytes = await request.arrayBuffer(); + + if (bodyBytes.byteLength > 0) { + bodyB64 = uint8ToBase64(new Uint8Array(bodyBytes)); + } + + return new Promise((resolve, reject) => { + let streamController!: ReadableStreamDefaultController; + + const stream = new ReadableStream({ + start(ctrl) { + streamController = ctrl; + } + }); + const abortHandler = () => { + this.pending.delete(id); + try { + streamController.error(new DOMException('Aborted', 'AbortError')); + } catch { + // stream may already be closed + } + reject(new DOMException('Aborted', 'AbortError')); + + // Tell the host to stop the in-flight fetch + if (this.channel?.readyState === 'open') { + this.channel.send(JSON.stringify({ id, type: 'cancel' } satisfies CancelMsg)); + } + }; + + signal?.addEventListener('abort', abortHandler, { once: true }); + + this.pending.set(id, { + onChunk: (data) => { + streamController.enqueue(base64ToUint8(data)); + }, + onEnd: () => { + signal?.removeEventListener('abort', abortHandler); + streamController.close(); + }, + onError: (message) => { + signal?.removeEventListener('abort', abortHandler); + try { + streamController.error(new Error(message)); + } catch { + // stream may already be closed + } + reject(new Error(message)); + this.pending.delete(id); + }, + onStart: (status, resHeaders) => { + resolve(new Response(stream, { headers: resHeaders, status })); + } + }); + + this.channel!.send( + JSON.stringify({ + body: bodyB64, + headers, + id, + method: request.method, + path, + type: 'req' + } satisfies ReqMsg) + ); + }); + } + + private cleanupConnection(): void { + this.channel?.close(); + this.pc?.close(); + for (const tracker of this.trackers) tracker.close(); + this.channel = null; + this.pc = null; + this.trackers = []; + } + + private rejectAllPending(reason: string): void { + for (const req of this.pending.values()) req.onError(reason); + this.pending.clear(); + } + + private routeResponseMsg(msg: TunnelMsg): void { + if ( + msg.type !== 'res_start' && + msg.type !== 'res_chunk' && + msg.type !== 'res_end' && + msg.type !== 'res_err' + ) + return; + + const req = this.pending.get(msg.id); + + if (!req) return; + + if (msg.type === 'res_start') { + req.onStart(msg.status, msg.headers); + } else if (msg.type === 'res_chunk') { + req.onChunk(msg.data); + } else if (msg.type === 'res_end') { + req.onEnd(); + this.pending.delete(msg.id); + } else if (msg.type === 'res_err') { + req.onError(msg.message); + this.pending.delete(msg.id); + } + } +} diff --git a/ui/apps/chat/src/routes/+layout.svelte b/ui/apps/chat/src/routes/+layout.svelte index 8b7cb80db335..f3f58fb65816 100644 --- a/ui/apps/chat/src/routes/+layout.svelte +++ b/ui/apps/chat/src/routes/+layout.svelte @@ -23,6 +23,9 @@ conversationsStore, deviceStore, modelsStore, + remoteAccessStore, + serversModelsStore, + serversStore, serverStore, settingsStore, tabsStore, @@ -30,8 +33,9 @@ versionStore } from '#lib/stores/index.ts'; import { initStores } from '#lib/stores/init.ts'; - import { withBase } from '#lib/utils/base-path.ts'; + import { apiUrl } from '#lib/utils/api-base.ts'; import { applySurfaceTheme, hapticTap, resolveTabView } from '#lib/utils/index.ts'; + import { isLocalServer, isRemoteServer } from '#lib/utils/servers.ts'; import { browser } from '$app/env'; import { afterNavigate, goto } from '$app/navigation'; import { page } from '$app/state'; @@ -204,7 +208,7 @@ [HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKey.trim()}` }; - fetch(withBase('/props'), { headers }) + fetch(apiUrl('/props'), { headers }) .then((response) => { if (response.status === 401 || response.status === 403) { window.location.reload(); @@ -219,7 +223,7 @@ onMount(() => { updateFavicon(); - // snapshot of every backend running stream on first load, populates the sidebar spinners + // snapshot of every server running stream on first load, populates the sidebar spinners // so the user sees each conv that has a live inference, even ones not opened yet void chatStore.syncRemoteRunningStreams(); }); @@ -230,6 +234,10 @@ if (document.visibilityState !== 'visible') return; void chatStore.syncRemoteRunningStreams(); + + // mobile browsers drop a peer connection when the tab is backgrounded, so a + // tunnel that was open is asked for again on the way back + remoteAccessStore.recoverChannels(); } $effect(() => { @@ -238,6 +246,11 @@ updateFavicon(); }); + // turning the experimental remote pairing off closes the tunnels it opened + $effect(() => { + if (!remoteAccessStore.isEnabled) untrack(() => remoteAccessStore.syncServers()); + }); + // Initialize server properties on app load (run once) $effect(() => { // Only fetch if we don't already have props @@ -265,37 +278,34 @@ node.textContent = (settingsStore.config.customCss as string | undefined) ?? ''; } - // Fetch router models when in router mode (for status and modalities) - // Wait for models to be loaded first, run only once - let routerModelsFetched = false; - + // Live model status and load progress via the /models/sse feed (router mode). + // The feed is kept for the session: switching to an external server and back + // must not tear it down and reconnect on every tab switch. $effect(() => { - const isRouter = serverStore.isRouterMode; - const modelsCount = modelsStore.models.length; + if (!browser) return; - // Only fetch router models once when we have models loaded and in router mode - if (isRouter && modelsCount > 0 && !routerModelsFetched) { - routerModelsFetched = true; + if (!serverStore.localIsRouter) return; - untrack(() => { - modelsStore.fetchRouterModels(); - }); - } + untrack(() => { + modelsStore.status.subscribe(); + }); }); - // Live model status and load progress via the /models/sse feed (router mode) + // The active server's own feed, when it is not the built-in one: a llama.cpp + // server serves /models/sse at its origin, so a router behind a tunnel animates + // its loads the way a local one does. It follows that server being reachable, + // and the operations that wait on a load keep their polling as the fallback. $effect(() => { if (!browser) return; - if (!serverStore.isRouterMode) return; - - untrack(() => { - modelsStore.status.subscribe(); - }); + const server = serversStore.active; + const reachable = + !isLocalServer(server) && + (isRemoteServer(server) + ? remoteAccessStore.isConnected(server.id) + : serversModelsStore.get(server.id).loaded); - return () => { - modelsStore.status.unsubscribe(); - }; + untrack(() => modelsStore.status.subscribeServer(reachable ? server.id : null)); }); // Monitor API key changes and redirect to error page if removed or changed when required diff --git a/ui/apps/chat/src/routes/[extensionPage]/+page.ts b/ui/apps/chat/src/routes/[extensionPage]/+page.ts index af674fcc8f60..98440b7e2176 100644 --- a/ui/apps/chat/src/routes/[extensionPage]/+page.ts +++ b/ui/apps/chat/src/routes/[extensionPage]/+page.ts @@ -1,12 +1,14 @@ import type { PageLoad } from './$types'; import { error } from '@sveltejs/kit'; import { extensions } from '#lib/extensions/index.ts'; -import { loadAppRoute } from '#lib/stores/init.ts'; +import { loadAppRoute, startExtensions } from '#lib/stores/init.ts'; export const load: PageLoad = async ({ fetch, params }) => { await loadAppRoute(fetch); + // the rest of the startup runs after this load, so the extensions are started here + await startExtensions(); - // the extensions are active by now: a path none of them has a page for is the app's 404 + // a path none of them has a page for is the app's 404 if (!extensions.contributions('pages').some((page) => page.path === params.extensionPage)) { error(404, 'Not Found'); } diff --git a/ui/apps/chat/src/routes/models/+page.svelte b/ui/apps/chat/src/routes/models/+page.svelte index e8295430845b..0a4e3643a3ad 100644 --- a/ui/apps/chat/src/routes/models/+page.svelte +++ b/ui/apps/chat/src/routes/models/+page.svelte @@ -1,10 +1,15 @@ + + + Servers - {APP_NAME} + + + +
+
+

Servers

+ +

+ {remoteAccessStore.isEnabled + ? 'Pair a remote server by its access code, or point at one by its URL.' + : 'Point at a llama-server by its URL.'} +

+
+ + +
diff --git a/ui/apps/chat/src/routes/servers/+page.ts b/ui/apps/chat/src/routes/servers/+page.ts new file mode 100644 index 000000000000..3c3deac5178a --- /dev/null +++ b/ui/apps/chat/src/routes/servers/+page.ts @@ -0,0 +1,4 @@ +import type { PageLoad } from './$types'; +import { loadAppRoute } from '#lib/stores/init.ts'; + +export const load: PageLoad = ({ fetch }) => loadAppRoute(fetch); diff --git a/ui/apps/chat/static/backend-presets/deepseek.svg b/ui/apps/chat/static/backend-presets/deepseek.svg new file mode 100644 index 000000000000..185c108b458c --- /dev/null +++ b/ui/apps/chat/static/backend-presets/deepseek.svg @@ -0,0 +1 @@ +DeepSeek \ No newline at end of file diff --git a/ui/apps/chat/static/backend-presets/google.svg b/ui/apps/chat/static/backend-presets/google.svg new file mode 100644 index 000000000000..4545c8331f9e --- /dev/null +++ b/ui/apps/chat/static/backend-presets/google.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/ui/apps/chat/static/backend-presets/huggingface.svg b/ui/apps/chat/static/backend-presets/huggingface.svg new file mode 100644 index 000000000000..6b3b4f687eb3 --- /dev/null +++ b/ui/apps/chat/static/backend-presets/huggingface.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + diff --git a/ui/apps/chat/static/backend-presets/kimi.png b/ui/apps/chat/static/backend-presets/kimi.png new file mode 100644 index 000000000000..213b9b83d6cd Binary files /dev/null and b/ui/apps/chat/static/backend-presets/kimi.png differ diff --git a/ui/apps/chat/static/backend-presets/openrouter-dark.svg b/ui/apps/chat/static/backend-presets/openrouter-dark.svg new file mode 100644 index 000000000000..5df05ac31558 --- /dev/null +++ b/ui/apps/chat/static/backend-presets/openrouter-dark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/apps/chat/static/backend-presets/openrouter-light.svg b/ui/apps/chat/static/backend-presets/openrouter-light.svg new file mode 100644 index 000000000000..8fbbb8717851 --- /dev/null +++ b/ui/apps/chat/static/backend-presets/openrouter-light.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/ui/apps/chat/static/backend-presets/qwen.png b/ui/apps/chat/static/backend-presets/qwen.png new file mode 100644 index 000000000000..87a98a952624 Binary files /dev/null and b/ui/apps/chat/static/backend-presets/qwen.png differ diff --git a/ui/apps/chat/static/backend-presets/zai.png b/ui/apps/chat/static/backend-presets/zai.png new file mode 100644 index 000000000000..70c33481a4ce Binary files /dev/null and b/ui/apps/chat/static/backend-presets/zai.png differ diff --git a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts index 058c7e02ed98..d35aa0cf3272 100644 --- a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts +++ b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts @@ -12,7 +12,7 @@ import { VOICE_MODE_MESSAGE } from '#lib/constants/index.ts'; import { ServerRole, VoiceInputMode } from '#lib/enums/index.ts'; -import { voiceTranscriptsStore } from '#lib/stores/index.ts'; +import { modelsStore, voiceTranscriptsStore } from '#lib/stores/index.ts'; import { parseFilesToMessageExtras } from '#lib/utils/browser-only.ts'; import { tick } from 'svelte'; import { afterEach, describe, expect, it } from 'vitest'; @@ -46,8 +46,7 @@ function seed( /** A router whose chat model (Qwen3) can't hear while another (Gemma) can, none of them loaded. */ function seedRouter(transcriptionModel = '') { const [chat, hearing] = routerModels; - - restore = seedStores({ + const restoreStores = seedStores({ config: { autoMicOnEmpty: true, [SETTINGS_KEYS.VOICE_INPUT_MODE]: VoiceInputMode.AUTO, @@ -63,6 +62,14 @@ function seedRouter(transcriptionModel = '') { selectedModelId: chat.id, serverRole: ServerRole.ROUTER }); + // the microphone looks among the active server's models for one that hears + const activeModels = modelsStore.activeModels; + + modelsStore.activeModels = routerModels; + restore = () => { + modelsStore.activeModels = activeModels; + restoreStores(); + }; } async function renderForm(options: { isLoading?: boolean; voiceMode?: boolean } = {}) { diff --git a/ui/apps/chat/tests/client/components/DialogServersWrapper.svelte b/ui/apps/chat/tests/client/components/DialogServersWrapper.svelte new file mode 100644 index 000000000000..4f1d3ae5b2b4 --- /dev/null +++ b/ui/apps/chat/tests/client/components/DialogServersWrapper.svelte @@ -0,0 +1,15 @@ + + + + + + diff --git a/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte b/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte index 494ccd81ae6b..92f55e9127e7 100644 --- a/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte +++ b/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte @@ -12,7 +12,7 @@ let { onSelect, option, selected = false }: Props = $props(); - + false} diff --git a/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte b/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte index 26af1bf324af..7d12eb29ab99 100644 --- a/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte +++ b/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte @@ -3,7 +3,7 @@ import ModelsManager from '#lib/components/app/models/ModelsManager/ModelsManager.svelte'; - + diff --git a/ui/apps/chat/tests/client/components/ServerCardWrapper.svelte b/ui/apps/chat/tests/client/components/ServerCardWrapper.svelte new file mode 100644 index 000000000000..200dd144995d --- /dev/null +++ b/ui/apps/chat/tests/client/components/ServerCardWrapper.svelte @@ -0,0 +1,18 @@ + + + + + + diff --git a/ui/apps/chat/tests/client/components/TestWrapper.svelte b/ui/apps/chat/tests/client/components/TestWrapper.svelte index 3f050cdec8f5..f09265d7f87c 100644 --- a/ui/apps/chat/tests/client/components/TestWrapper.svelte +++ b/ui/apps/chat/tests/client/components/TestWrapper.svelte @@ -4,8 +4,8 @@ diff --git a/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts b/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts index 32da9e692af3..646623708648 100644 --- a/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts +++ b/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts @@ -4,8 +4,10 @@ import ModelsManagerRowWrapper from './components/ModelsManagerRowWrapper.svelte'; import ModelsManagerRepoRow from '#lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte'; import type { ModelQuantGroup } from '#lib/components/app/models/ModelsManager/utils.ts'; +import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; import { ModelGroupKind, ServerRole } from '#lib/enums/index.ts'; import { serverStore } from '#lib/stores/llama-server.svelte.ts'; +import { serversStore } from '#lib/stores/servers.svelte.ts'; import type { ModelOption } from '#lib/types/models.js'; import { beforeEach, describe, expect, it } from 'vitest'; import { userEvent } from 'vitest/browser'; @@ -15,7 +17,8 @@ const option: ModelOption = { capabilities: [], id: 'org/Qwen3-8B:Q4_K_M', model: 'org/Qwen3-8B:Q4_K_M', - name: 'Qwen3-8B' + name: 'Qwen3-8B', + serverId: LOCAL_BACKEND_ID }; const entry: ModelQuantGroup = { base: option, @@ -34,7 +37,9 @@ describe('manager model row', () => { beforeEach(() => { selected = []; - // the row only offers a load control on a router server + // the row resolves the server that serves it, and only offers a load control on + // a router server + serversStore.initialize(); serverStore.role = ServerRole.ROUTER; }); diff --git a/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts b/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts index 1bcc556a7ba3..40f7124311dd 100644 --- a/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts +++ b/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts @@ -15,7 +15,7 @@ import type { ApiModelDataEntry, ModelOption } from '#lib/types/index.ts'; import { beforeEach, describe, expect, it } from 'vitest'; function option(model: string, id = model): ModelOption { - return { capabilities: [], id, model, name: model }; + return { capabilities: [], id, model, name: model, serverId: LOCAL_BACKEND_ID }; } function routerEntry(id: string, status: ServerModelStatus): ApiModelDataEntry { @@ -43,8 +43,8 @@ describe('groupModelQuants', () => { ]); expect(groups.map((group) => [group.key, group.kind, group.quants.length])).toEqual([ - ['org/Qwen3-8B', ModelGroupKind.QUANTS, 2], - ['org/Llama-3-8B', ModelGroupKind.QUANTS, 1] + [`${LOCAL_BACKEND_ID}::org/Qwen3-8B`, ModelGroupKind.QUANTS, 2], + [`${LOCAL_BACKEND_ID}::org/Llama-3-8B`, ModelGroupKind.QUANTS, 1] ]); }); @@ -61,8 +61,8 @@ describe('groupModelQuants', () => { ]); expect(groups.map((group) => [group.key, group.kind])).toEqual([ - ['org/Qwen3-8B::first', ModelGroupKind.VARIANTS], - ['org/Qwen3-8B::second', ModelGroupKind.VARIANTS] + [`${LOCAL_BACKEND_ID}::org/Qwen3-8B::first`, ModelGroupKind.VARIANTS], + [`${LOCAL_BACKEND_ID}::org/Qwen3-8B::second`, ModelGroupKind.VARIANTS] ]); }); }); @@ -102,9 +102,9 @@ describe('downloadGroups', () => { // the sub-lines carry the sidecars, so the inline badges stay empty expect(groups[0].base).toMatchObject({ - backendId: LOCAL_BACKEND_ID, draftSidecars: [], - model: 'org/Qwen3-8B:Q4_K_M' + model: 'org/Qwen3-8B:Q4_K_M', + serverId: LOCAL_BACKEND_ID }); expect(groups[0].sidecarBadges).toEqual([ { kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: 'org/Qwen3-8B' } diff --git a/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts b/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts index a6785edc231a..fb9dfacb79fd 100644 --- a/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts +++ b/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts @@ -3,6 +3,7 @@ // can be hidden and selected at the same time. import ModelsSelectorHookHarness from './components/ModelsSelectorHookHarness.svelte'; +import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; import { ServerModelStatus, ServerRole } from '#lib/enums/index.ts'; import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; @@ -11,7 +12,7 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { render } from 'vitest-browser-svelte'; function option(model: string): ModelOption { - return { capabilities: [], id: model, model, name: model }; + return { capabilities: [], id: model, model, name: model, serverId: LOCAL_BACKEND_ID }; } function routerEntry(id: string, status: ServerModelStatus): ApiModelDataEntry { diff --git a/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts b/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts index 2c1d61284d07..24b70a0a6789 100644 --- a/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts +++ b/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts @@ -34,13 +34,14 @@ describe('model picker drawer on a phone', () => { vi.spyOn(ModelsService, 'list').mockResolvedValue({ data: [{ id: option.model }, { id: other.model }] } as never); - modelsStore.models = [option, other]; + // the models read through the servers data layer on this layer of the stack + modelsStore.activeModels = [option, other]; await modelsStore.selectModelById(option.id); await page.viewport(PHONE.width, PHONE.height); }); afterEach(async () => { - modelsStore.models = []; + modelsStore.activeModels = []; vi.restoreAllMocks(); await page.viewport(DESKTOP.width, DESKTOP.height); }); diff --git a/ui/apps/chat/tests/client/qr-code.svelte.test.ts b/ui/apps/chat/tests/client/qr-code.svelte.test.ts new file mode 100644 index 000000000000..91756b180e8d --- /dev/null +++ b/ui/apps/chat/tests/client/qr-code.svelte.test.ts @@ -0,0 +1,56 @@ +// The two ends of the code handoff, exercised against each other: the QR the +// host screen draws is read back by the same native detector the phone's camera +// scanner uses. A change to the payload or to the rendering shows up here, and +// only here - everything else about the code is a unit test. + +import { CODE_CHARS, CODE_LENGTHS } from '#lib/constants/index.ts'; +import { qrCodeDataUrl } from '#lib/utils/qr-code.ts'; +import { supportsQrScan } from '#lib/utils/qr-scan.ts'; +import { expect, it } from 'vitest'; + +/** A code of the right length, drawn from the alphabet llama-server generates from. */ +const CODE = Array.from( + { length: CODE_LENGTHS.SHARE }, + (_, index) => CODE_CHARS[index % CODE_CHARS.length] +).join(''); +/** Browsers that ship the detector the scanner uses; the rest skip the round trip. */ +const canReadCodes = supportsQrScan(); + +interface BarcodeDetectorLike { + detect(source: CanvasImageSource): Promise>; +} + +/** The payload the scanner reports for a rendered code, as a phone would see it. */ +async function readBack(url: string): Promise { + const image = new Image(); + + image.src = url; + await image.decode(); + + const canvas = document.createElement('canvas'); + + canvas.width = 256; + canvas.height = 256; + + const context = canvas.getContext('2d')!; + + context.fillStyle = '#ffffff'; + context.fillRect(0, 0, canvas.width, canvas.height); + context.drawImage(image, 0, 0, canvas.width, canvas.height); + + const Detector = ( + window as unknown as { + BarcodeDetector: new (options: { formats: string[] }) => BarcodeDetectorLike; + } + ).BarcodeDetector; + const [found] = await new Detector({ formats: ['qr_code'] }).detect(canvas); + + return found?.rawValue ?? null; +} + +it.skipIf(!canReadCodes)('reads back the code the host screen shows', async () => { + const url = qrCodeDataUrl(CODE); + + expect(url).not.toBeNull(); + expect(await readBack(url!)).toBe(CODE); +}); diff --git a/ui/apps/chat/tests/client/remote-servers.svelte.test.ts b/ui/apps/chat/tests/client/remote-servers.svelte.test.ts new file mode 100644 index 000000000000..523cf290d0dc --- /dev/null +++ b/ui/apps/chat/tests/client/remote-servers.svelte.test.ts @@ -0,0 +1,416 @@ +// Guards the transport rule the whole remote-server feature leans on: a request +// addressed at a server's tunnel origin is answered by that server's tunnel or +// refused out loud - it never quietly reaches the server serving this UI. The +// card is checked here too: what it says about a channel is what the user acts on. + +import ServerCardWrapper from './components/ServerCardWrapper.svelte'; +import { + API_MODELS, + CODE_CHARS, + CODE_LENGTHS, + LOCAL_BACKEND_ID, + SETTINGS_KEYS, + WEBRTC_TIMEOUTS +} from '#lib/constants/index.ts'; +import { TunnelStatus } from '#lib/enums/index.ts'; +import { + modelsStore, + remoteAccessStore, + serversStore, + serverStore, + settingsStore +} from '#lib/stores/index.ts'; +import type { ApiLlamaCppServerProps } from '#lib/types/index.ts'; +import { createRemoteServer, tunnelOrigin } from '#lib/utils/servers.ts'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { render } from 'vitest-browser-svelte'; + +const { FakeTunnel } = vi.hoisted(() => { + /** + * Stands in for the transport: a handshake the test settles by hand, so an + * attempt the user gave up on can be examined without a live tracker. + */ + class FakeTunnel { + disconnected = false; + static instances: FakeTunnel[] = []; + onConnected: (() => void) | undefined; + settle: (() => void) | undefined; + + get isConnected(): boolean { + return !this.disconnected; + } + + connect(): Promise { + return new Promise((resolve) => { + this.settle = resolve; + }); + } + + constructor(_room: string, _pass: string, callbacks: { onConnected?: () => void } = {}) { + this.onConnected = callbacks.onConnected; + FakeTunnel.instances.push(this); + } + + disconnect(): void { + this.disconnected = true; + } + } + + return { FakeTunnel }; +}); + +vi.mock('#lib/utils/webrtc-tunnel.ts', () => ({ ClientTunnel: FakeTunnel })); + +/** A code of the right length, drawn from the alphabet llama-server generates from. */ +const CODE = Array.from( + { length: CODE_LENGTHS.SHARE }, + (_, index) => CODE_CHARS[index % CODE_CHARS.length] +).join(''); +const originalFetch = window.fetch; + +let passthrough: string[] = []; + +/** + * Register one remote server, swap `window.fetch` for a recorder, and install + * the tunnel transport over it. Returns the recorded requests. + */ +function withTransport(url: string): Promise { + settingsStore.updateConfig( + SETTINGS_KEYS.SERVERS, + JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }]) + ); + serversStore.initialize(); + + passthrough = []; + + window.fetch = ((input: RequestInfo | URL) => { + passthrough.push(String(input instanceof Request ? input.url : input)); + + return Promise.resolve(new Response('ok', { status: 200 })); + }) as typeof window.fetch; + + remoteAccessStore.installInterceptor(); + + return fetch(url); +} + +// pairing is an experimental opt-in; these tests exercise it switched on +beforeEach(() => { + settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, true); +}); + +afterEach(() => { + remoteAccessStore.uninstallInterceptor(); + window.fetch = originalFetch; + settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, false); +}); + +it('starts a paired server idle', () => { + settingsStore.updateConfig( + SETTINGS_KEYS.SERVERS, + JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }]) + ); + serversStore.initialize(); + + expect(remoteAccessStore.statusOf('r1')).toBe(TunnelStatus.IDLE); + expect(remoteAccessStore.isConnected('r1')).toBe(false); + expect(remoteAccessStore.isConnecting('r1')).toBe(false); +}); + +it('refuses a server it has no tunnel for', async () => { + const request = withTransport(`${tunnelOrigin('r1')}/props`); + + await expect(request).rejects.toThrow(/is not connected/); + + // the point of the whole exercise: nothing leaked to the serving origin + expect(passthrough).toEqual([]); +}); + +it('refuses a tunnel origin no server owns', async () => { + const request = withTransport('https://ghost.remote.llamacpp.invalid/v1/models'); + + await expect(request).rejects.toThrow(/No remote server is registered/); + expect(passthrough).toEqual([]); +}); + +it('leaves a real URL to the browser', async () => { + await withTransport('https://example.com/v1/models'); + + expect(passthrough).toEqual(['https://example.com/v1/models']); +}); + +it('leaves the page origin to the browser', async () => { + await withTransport('/props'); + + expect(passthrough).toEqual(['/props']); +}); + +/** The remote server the card tests below are about. */ +const REMOTE = createRemoteServer(CODE, 'r1', 'Home'); + +/** A tunnel status the card reads back for `r1`, without a transport behind it. */ +function withTunnel(tunnel: TunnelStatus, error = '') { + remoteAccessStore.status['r1'] = tunnel; + remoteAccessStore.errorMessage['r1'] = error; +} + +describe('server card', () => { + beforeEach(() => { + // the local card only reports the host of the server the app opens with + serversStore.setActive(LOCAL_BACKEND_ID); + }); + + afterEach(() => { + delete remoteAccessStore.status['r1']; + delete remoteAccessStore.errorMessage['r1']; + serverStore.props = null; + }); + + it('renders the host this machine runs, and pairs with its code', async () => { + serverStore.props = { connect_code: CODE } as ApiLlamaCppServerProps; + const paired: string[] = []; + const screen = render(ServerCardWrapper, { + isLocal: true, + onPair: (code: string) => paired.push(code), + server: serversStore.local + }); + + await expect.element(screen.getByText('Remote access: on')).toBeVisible(); + await expect + .element(screen.getByText('This server can be reached through its access code.')) + .toBeVisible(); + + await screen.getByRole('button', { name: 'Pair' }).click(); + + expect(paired).toEqual([CODE]); + }); + + it('renders no host side for a machine without one', async () => { + serverStore.props = {} as ApiLlamaCppServerProps; + + const screen = render(ServerCardWrapper, { isLocal: true, server: serversStore.local }); + + await expect.element(screen.getByText('Remote access: on')).not.toBeInTheDocument(); + }); + + it('offers the host code as a QR the other device can scan', async () => { + serverStore.props = { connect_code: CODE } as ApiLlamaCppServerProps; + + const screen = render(ServerCardWrapper, { isLocal: true, server: serversStore.local }); + + await expect.element(screen.getByText('Remote access: on')).toBeVisible(); + + // the QR is behind its own control: the card stays compact until it is asked for + expect(document.querySelector('img[alt="Access code QR code"]')).toBeNull(); + + await screen.getByRole('button', { name: 'Show the access code as a QR code' }).click(); + + const qr = screen.getByAltText('Access code QR code'); + + await expect.element(qr).toBeVisible(); + expect((qr.element() as HTMLImageElement).src.startsWith('data:image/svg+xml')).toBe(true); + }); + + it('offers a way out of a handshake that stalls', async () => { + withTunnel(TunnelStatus.CONNECTING); + + const screen = render(ServerCardWrapper, { server: REMOTE }); + + await expect.element(screen.getByText('Connecting')).toBeVisible(); + await expect + .element(screen.getByRole('button', { name: 'Disconnect this server' })) + .toBeVisible(); + }); + + it('reports the reason a channel dropped, not just that it did', async () => { + withTunnel(TunnelStatus.ERROR, 'authentication failed: invalid passcode'); + + const screen = render(ServerCardWrapper, { server: REMOTE }); + + await expect.element(screen.getByText('Channel down')).toBeVisible(); + await expect.element(screen.getByText('authentication failed: invalid passcode')).toBeVisible(); + await expect + .element(screen.getByRole('button', { name: 'Reconnect to the host' })) + .toBeVisible(); + }); + + it('keeps an idle server a choice, with the code in reach', async () => { + const writeText = vi.spyOn(navigator.clipboard, 'writeText').mockResolvedValue(); + const screen = render(ServerCardWrapper, { server: REMOTE }); + + await expect.element(screen.getByText('Not connected')).toBeVisible(); + await expect + .element(screen.getByText('Paste the code the host prints, or press Connect.')) + .toBeVisible(); + + await screen.getByRole('button', { name: 'Copy the access code' }).click(); + + expect(writeText).toHaveBeenCalledWith(CODE); + + await expect.element(screen.getByText('Not connected')).toBeVisible(); + }); + + it('reveals the code a server stores', async () => { + const screen = render(ServerCardWrapper, { server: REMOTE }); + + await screen.getByTitle('Show the access code').click(); + + await expect.element(screen.getByText(CODE)).toBeVisible(); + }); +}); + +describe('the status feed of a tunneled router', () => { + const calls: string[] = []; + + beforeEach(() => { + calls.length = 0; + settingsStore.updateConfig( + SETTINGS_KEYS.SERVERS, + JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }]) + ); + serversStore.initialize(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: RequestInfo | URL) => { + calls.push(String(input instanceof Request ? input.url : input)); + + return new Response(': ping\n\n', { status: 200 }); + }); + }); + + afterEach(() => { + modelsStore.status.subscribeServer(null); + vi.restoreAllMocks(); + }); + + it('reads the feed at the origin of the server it was given', async () => { + modelsStore.status.subscribeServer('r1'); + + await vi.waitFor(() => expect(calls.length).toBeGreaterThan(0)); + + // the tunnel carries the feed like any other request: the server's own + // origin, never the one serving this UI + expect(calls[0]).toBe(`${tunnelOrigin('r1')}${API_MODELS.SSE}`); + }); + + it('opens one reader per server', async () => { + modelsStore.status.subscribeServer('r1'); + modelsStore.status.subscribeServer('r1'); + + await vi.waitFor(() => expect(calls.length).toBe(1)); + + expect(calls.length).toBe(1); + }); + + it('opens nothing for no server', async () => { + modelsStore.status.subscribeServer(null); + + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(calls).toEqual([]); + }); +}); + +describe('channels lost to a backgrounded tab', () => { + beforeEach(() => { + vi.useFakeTimers(); + settingsStore.updateConfig( + SETTINGS_KEYS.SERVERS, + JSON.stringify([ + { code: CODE, id: 'r1', kind: 'remote', name: 'Home' }, + { code: CODE, id: 'r2', kind: 'remote', name: 'Office' } + ]) + ); + serversStore.initialize(); + // one channel was open before the tab went away, the other was never asked for + withTunnel(TunnelStatus.ERROR, 'Disconnected from the host'); + remoteAccessStore.status['r2'] = TunnelStatus.IDLE; + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + delete remoteAccessStore.status['r1']; + delete remoteAccessStore.errorMessage['r1']; + delete remoteAccessStore.status['r2']; + }); + + it('asks again, and only for a channel that was wanted', async () => { + const connect = vi.spyOn(remoteAccessStore, 'connect').mockResolvedValue(); + + remoteAccessStore.recoverChannels(); + + // the network is not back the moment the tab is: nothing happens at once + expect(connect).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(WEBRTC_TIMEOUTS.RECOVER_MS); + + expect(connect).toHaveBeenCalledWith('r1'); + expect(connect).not.toHaveBeenCalledWith('r2'); + }); + + it('opens one handshake per burst of tab switches', async () => { + const connect = vi.spyOn(remoteAccessStore, 'connect').mockResolvedValue(); + + remoteAccessStore.recoverChannels(); + await vi.advanceTimersByTimeAsync(WEBRTC_TIMEOUTS.RECOVER_MS / 2); + remoteAccessStore.recoverChannels(); + await vi.advanceTimersByTimeAsync(WEBRTC_TIMEOUTS.RECOVER_MS); + + expect(connect).toHaveBeenCalledTimes(1); + }); +}); + +describe('a handshake the user gives up on', () => { + beforeEach(() => { + FakeTunnel.instances = []; + settingsStore.updateConfig( + SETTINGS_KEYS.SERVERS, + JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }]) + ); + serversStore.initialize(); + }); + + afterEach(() => { + remoteAccessStore.disconnect('r1'); + }); + + it('leaves the server idle when the answer lands late', async () => { + const attempt = remoteAccessStore.connect('r1'); + const [tunnel] = FakeTunnel.instances; + + remoteAccessStore.disconnect('r1'); + + // the host answers after the user stopped waiting + tunnel.onConnected?.(); + tunnel.settle?.(); + + await attempt; + + expect(remoteAccessStore.statusOf('r1')).toBe(TunnelStatus.IDLE); + expect(remoteAccessStore.isConnected('r1')).toBe(false); + expect(tunnel.disconnected).toBe(true); + }); + + it('starts a fresh handshake when asked again', async () => { + const dropped = remoteAccessStore.connect('r1'); + + remoteAccessStore.disconnect('r1'); + + const kept = remoteAccessStore.connect('r1'); + const [first, second] = FakeTunnel.instances; + + expect(second).not.toBe(first); + + second.onConnected?.(); + second.settle?.(); + await kept; + + expect(remoteAccessStore.isConnected('r1')).toBe(true); + + // the dropped attempt landing now must not close the one that is up + first.settle?.(); + await dropped; + + expect(remoteAccessStore.isConnected('r1')).toBe(true); + expect(first.disconnected).toBe(true); + }); +}); diff --git a/ui/apps/chat/tests/client/servers-dialog.svelte.test.ts b/ui/apps/chat/tests/client/servers-dialog.svelte.test.ts new file mode 100644 index 000000000000..56837e8b6425 --- /dev/null +++ b/ui/apps/chat/tests/client/servers-dialog.svelte.test.ts @@ -0,0 +1,166 @@ +// Guards the phone's door to servers. The models dialog is desktop-only, so a +// phone reaches its server list from the sidebar; without that entry there is no +// way to pair a server, and a phone with no paired server has nothing to pick a +// model from. + +import DialogServersWrapper from './components/DialogServersWrapper.svelte'; +import { ChatScreenServerError, SidebarNavigationActions } from '#lib/components/app/index.ts'; +import { + CODE_CHARS, + CODE_LENGTHS, + SETTINGS_KEYS, + SIDEBAR_FOOTER_ITEMS +} from '#lib/constants/index.ts'; +import { SidebarAction } from '#lib/enums/index.ts'; +import { serversStore, serverStore, settingsStore, uiStore } from '#lib/stores/index.ts'; +import { getAuthHeadersForServer } from '#lib/utils/api-headers.ts'; +import { createRemoteServer, createUrlServer } from '#lib/utils/servers.ts'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { page } from 'vitest/browser'; +import { render } from 'vitest-browser-svelte'; + +const PHONE = { height: 844, width: 390 }; +const DESKTOP = { height: 900, width: 1280 }; +/** A code of the right length, drawn from the alphabet llama-server generates from. */ +const CODE = Array.from( + { length: CODE_LENGTHS.SHARE }, + (_, index) => CODE_CHARS[index % CODE_CHARS.length] +).join(''); + +describe('servers on a phone', () => { + const opened: boolean[] = []; + + beforeEach(async () => { + opened.length = 0; + settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, true); + await page.viewport(PHONE.width, PHONE.height); + }); + + afterEach(async () => { + settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, false); + await page.viewport(DESKTOP.width, DESKTOP.height); + }); + + it('offers the servers list in the sidebar', async () => { + // the sidebar's footer carries the entry, and a phone lists it with the top actions + const screen = render(SidebarNavigationActions, { + class: '', + items: SIDEBAR_FOOTER_ITEMS, + onAction: (action) => opened.push(action === SidebarAction.SERVERS) + }); + const item = screen.getByRole('button', { name: 'Servers' }); + + await expect.element(item).toBeVisible(); + + await item.click(); + + expect(opened).toEqual([true]); + }); + + it('opens the list as a dialog of its own', async () => { + const screen = render(DialogServersWrapper); + + await expect.element(screen.getByText('Servers')).toBeVisible(); + await expect.element(screen.getByRole('button', { name: 'Close' })).toBeVisible(); + }); + + it('reaches the pairing form through the add control', async () => { + const screen = render(DialogServersWrapper); + + await screen.getByRole('button', { name: 'Add new server' }).click(); + + await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible(); + + await screen.getByRole('tab', { name: 'Remote server' }).click(); + + await expect.element(screen.getByText('Not paired yet')).toBeVisible(); + }); + + it('offers only a URL while pairing is switched off', async () => { + settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, false); + + const screen = render(DialogServersWrapper); + + await screen.getByRole('button', { name: 'Add new server' }).click(); + + await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible(); + expect(screen.getByRole('tab', { name: 'Remote server' }).query()).toBeNull(); + }); + + it('opens the form as a bottom drawer, and as a dialog on a desktop', async () => { + const screen = render(DialogServersWrapper); + + await screen.getByRole('button', { name: 'Add new server' }).click(); + await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible(); + + const drawer = document.querySelector('[data-slot="drawer-content"]'); + + expect(drawer?.getAttribute('data-vaul-drawer-direction')).toBe('bottom'); + + await page.viewport(DESKTOP.width, DESKTOP.height); + + expect(document.querySelector('[data-slot="drawer-content"]')).toBeNull(); + await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible(); + }); + + it('offers the API key of a paired server too', async () => { + const screen = render(DialogServersWrapper); + + await screen.getByRole('button', { name: 'Add new server' }).click(); + await screen.getByRole('tab', { name: 'Remote server' }).click(); + + // a llama-server started with --connect can still guard its API with a key, + // and the tunnel carries the header through + await expect.element(screen.getByLabelText(/API key/)).toBeVisible(); + }); +}); + +describe('server credentials', () => { + afterEach(() => { + settingsStore.updateConfig(SETTINGS_KEYS.SERVERS, '[]'); + }); + + it('keeps the API key of every server it stores', () => { + serversStore.addServer(createUrlServer('https://example.com', 'u1', 'Office', 'sk-url')); + serversStore.addRemoteServer(CODE, 'Home', 'r1', 'sk-tunnel'); + + // the list a request is built from is the parsed one: a key that does not + // survive the round trip is a key that never reaches the server + const stored = serversStore.external; + const url = stored.find((server) => server.id === 'u1'); + const remote = stored.find((server) => server.id === 'r1'); + + expect(url?.apiKey).toBe('sk-url'); + expect(remote?.apiKey).toBe('sk-tunnel'); + expect(getAuthHeadersForServer(remote!).Authorization).toBe('Bearer sk-tunnel'); + }); + + it('keeps the code of a remote server', () => { + serversStore.addRemoteServer(CODE, 'Home', 'r1'); + + expect(serversStore.external.find((server) => server.id === 'r1')?.code).toBe(CODE); + expect(createRemoteServer(CODE, 'r2', 'Office').code).toBe(CODE); + }); +}); + +describe('a chat with no server at all', () => { + beforeEach(() => { + settingsStore.updateConfig(SETTINGS_KEYS.SERVERS, '[]'); + serverStore.error = 'Connection refused'; + }); + + afterEach(() => { + serverStore.error = null; + uiStore.serversOpen = false; + }); + + it('offers pairing as the way out', async () => { + const screen = render(ChatScreenServerError); + + await expect.element(screen.getByText('Server unavailable')).toBeVisible(); + + await screen.getByRole('button', { name: 'Pair a remote server' }).click(); + + expect(uiStore.serversOpen).toBe(true); + }); +}); diff --git a/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte b/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte index 3aa5c6045df4..c960d6b8401c 100644 --- a/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte +++ b/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte @@ -72,7 +72,8 @@ items: [availableModels[4]], orgName: 'intel' } - ] + ], + servers: [] }; function handleSelect(modelId: string) { diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts index fc573064ccbf..6729dbae4c39 100644 --- a/ui/apps/chat/tests/unit/cli.test.ts +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -1,4 +1,13 @@ -import { buildStatic, createChatServer, DEFAULTS, parseCliArgs } from '../../cli/server.js'; +import { + buildStatic, + createChatServer, + DEFAULTS, + parseCliArgs, + parseServerUrl, + PRESET_FILE, + PRESET_META, + withPresetMeta +} from '../../cli/server.js'; import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import http from 'node:http'; import type { AddressInfo } from 'node:net'; @@ -14,6 +23,9 @@ function listen(server: http.Server): Promise { }); } +/** A built index.html in short: the CLI adds its marker for llama-chat.json before ``. */ +const INDEX = 'Llama Chat'; + function close(server: http.Server): Promise { return new Promise((resolve) => server.close(() => resolve())); } @@ -25,6 +37,7 @@ describe('llama-chat CLI arguments', () => { expect(options.host).toBe(DEFAULTS.host); expect(options.port).toBe(DEFAULTS.port); expect(options.open).toBe(true); + expect(options.servers).toEqual([]); }); it('reads the documented flags', () => { @@ -42,8 +55,30 @@ describe('llama-chat CLI arguments', () => { expect(() => parseCliArgs(['--nope'])).toThrow(); expect(() => parseCliArgs(['--port', 'abc'])).toThrow(/invalid --port/); expect(() => parseCliArgs(['--port', '70000'])).toThrow(/invalid --port/); - // it forwards nothing, so there is no llama-server to name - expect(() => parseCliArgs(['--server', 'http://127.0.0.1:9931'])).toThrow(); + }); + + it('takes --server more than once, for serving and for build alike', () => { + expect( + parseCliArgs(['--server', 'localhost:9931', '--server', 'https://llm.example.com/api/']) + .servers + ).toEqual(['http://localhost:9931', 'https://llm.example.com/api']); + expect(parseCliArgs(['build', '--server', '127.0.0.1:9931']).servers).toEqual([ + 'http://127.0.0.1:9931' + ]); + }); + + it('puts the marker for llama-chat.json in the head of index.html once', () => { + const once = withPresetMeta(INDEX); + + expect(once).toContain(`${PRESET_META}\n\t`); + expect(withPresetMeta(once)).toBe(once); + }); + + it('refuses a server address the app could not use', () => { + expect(() => parseServerUrl('ftp://localhost')).toThrow(/invalid --server/); + expect(() => parseServerUrl('http://localhost:9931/?key=1')).toThrow(/invalid --server/); + expect(() => parseServerUrl('http://user:pass@localhost:9931')).toThrow(/invalid --server/); + expect(() => parseServerUrl('http://')).toThrow(/invalid --server/); }); it('reads build with its own flags, and keeps serving flags out of it', () => { @@ -85,7 +120,7 @@ describe('llama-chat build', () => { dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-')); work = await mkdtemp(join(tmpdir(), 'llama-chat-out-')); await mkdir(join(dist, '_app'), { recursive: true }); - await writeFile(join(dist, 'index.html'), ''); + await writeFile(join(dist, 'index.html'), INDEX); await writeFile(join(dist, '_app', 'app.js'), 'export {};'); await writeFile(join(dist, '_app', 'app.js.map'), '{}'); }); @@ -103,6 +138,23 @@ describe('llama-chat build', () => { join('_app', 'app.js'), 'index.html' ]); + // without servers there is no llama-chat.json, so nothing points the app at one + expect(await readFile(join(out, 'index.html'), 'utf8')).toBe(INDEX); + }); + + it('writes the servers to llama-chat.json when given', async () => { + const out = await buildStatic({ + distDir: dist, + out: join(work, 'with-servers'), + servers: ['http://127.0.0.1:9931'] + }); + + expect(JSON.parse(await readFile(join(out, PRESET_FILE), 'utf8'))).toEqual({ + servers: [{ url: 'http://127.0.0.1:9931' }] + }); + expect(await readFile(join(out, 'index.html'), 'utf8')).toContain(PRESET_META); + // the copy in dist keeps no marker + expect(await readFile(join(dist, 'index.html'), 'utf8')).toBe(INDEX); }); it('refuses a folder that is not empty unless forced', async () => { @@ -114,7 +166,7 @@ describe('llama-chat build', () => { await expect(buildStatic({ distDir: dist, out })).rejects.toThrow(/not empty/); await buildStatic({ distDir: dist, force: true, out }); expect(await readFile(join(out, 'keep.txt'), 'utf8')).toBe('mine'); - expect(await readFile(join(out, 'index.html'), 'utf8')).toBe(''); + expect(await readFile(join(out, 'index.html'), 'utf8')).toBe(INDEX); }); it('refuses to write into the app files themselves', async () => { @@ -135,7 +187,7 @@ describe('llama-chat server', () => { beforeAll(async () => { dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-')); await mkdir(join(dist, '_app', 'immutable'), { recursive: true }); - await writeFile(join(dist, 'index.html'), 'Llama Chat'); + await writeFile(join(dist, 'index.html'), INDEX); await writeFile(join(dist, 'sw.js'), 'self;'); await writeFile(join(dist, 'service-worker.js'), 'self;'); await writeFile(join(dist, '_app', 'immutable', 'app.abc123.js'), 'export {};'); @@ -157,7 +209,7 @@ describe('llama-chat server', () => { expect(res.headers.get('cache-control')).toBe('no-cache'); expect(res.headers.get('cross-origin-embedder-policy')).toBe('require-corp'); expect(res.headers.get('cross-origin-opener-policy')).toBe('same-origin'); - expect(await res.text()).toContain('Llama Chat'); + expect(await res.text()).toContain(PRESET_META); const again = await fetch(`${chatUrl}/`, { headers: { 'If-None-Match': res.headers.get('etag') ?? '' } @@ -181,6 +233,31 @@ describe('llama-chat server', () => { expect(worker.headers.get('cache-control')).toBe('no-cache'); }); + it('serves llama-chat.json, revalidated, with no servers when none are given', async () => { + const empty = await fetch(`${chatUrl}/${PRESET_FILE}`); + + expect(empty.status).toBe(200); + expect(await empty.json()).toEqual({ servers: [] }); + + const withServers = createChatServer({ + distDir: dist, + servers: ['http://127.0.0.1:9931', 'https://llm.example.com'] + }); + const url = await listen(withServers); + + try { + const res = await fetch(`${url}/${PRESET_FILE}`); + + expect(res.status).toBe(200); + expect(res.headers.get('cache-control')).toBe('no-cache'); + expect(await res.json()).toEqual({ + servers: [{ url: 'http://127.0.0.1:9931' }, { url: 'https://llm.example.com' }] + }); + } finally { + await close(withServers); + } + }); + it('forwards nothing: API paths are not found and other methods are refused', async () => { const props = await fetch(`${chatUrl}/props`); diff --git a/ui/apps/chat/tests/unit/extension-page-route.test.ts b/ui/apps/chat/tests/unit/extension-page-route.test.ts index 7cb28d3344e1..595387e2285e 100644 --- a/ui/apps/chat/tests/unit/extension-page-route.test.ts +++ b/ui/apps/chat/tests/unit/extension-page-route.test.ts @@ -4,11 +4,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('$app/env', () => ({ browser: true })); -vi.mock('#lib/stores/init.ts', () => ({ loadAppRoute: vi.fn(async () => {}) })); +vi.mock('#lib/stores/init.ts', () => ({ + loadAppRoute: vi.fn(async () => {}), + startExtensions: vi.fn(async () => {}) +})); import { load } from '../../src/routes/[extensionPage]/+page'; import { extensions } from '#lib/extensions/index.ts'; -import { loadAppRoute } from '#lib/stores/init.ts'; +import { loadAppRoute, startExtensions } from '#lib/stores/init.ts'; import type { Component } from 'svelte'; const Page = (() => {}) as unknown as Component; @@ -51,8 +54,9 @@ describe('extension page route', () => { await expect(loadPath('fake-page')).resolves.toBeUndefined(); await expect(loadPath('other-page')).rejects.toMatchObject({ status: 404 }); - // the stores start, and the API key is checked, before the path is looked up + // the stores hydrate, the API key is checked and the extensions start before the path is looked up expect(loadAppRoute).toHaveBeenCalledWith(fetch); + expect(startExtensions).toHaveBeenCalled(); await extensions.setEnabled('fake-pages', false); diff --git a/ui/apps/chat/tests/unit/model-list.test.ts b/ui/apps/chat/tests/unit/model-list.test.ts index 6a7c18fec146..793faa2ae235 100644 --- a/ui/apps/chat/tests/unit/model-list.test.ts +++ b/ui/apps/chat/tests/unit/model-list.test.ts @@ -33,7 +33,8 @@ function groups(): GroupedModelOptions { available: [ { items: [row('org/a'), row('org/b'), row('org/c')], orgName: 'org' }, { items: [row('other/d')], orgName: 'other' } - ] + ], + servers: [] }; } diff --git a/ui/apps/chat/tests/unit/qr-code.test.ts b/ui/apps/chat/tests/unit/qr-code.test.ts new file mode 100644 index 000000000000..c9b1c91058bb --- /dev/null +++ b/ui/apps/chat/tests/unit/qr-code.test.ts @@ -0,0 +1,66 @@ +// Guards the other end of the code scanner: what the host screen shows must be +// the payload the camera accepts, and nothing else. A QR that carries a URL, or +// a half-typed code, scans into a pairing that cannot work. + +import { CODE_CHARS, CODE_LENGTHS } from '#lib/constants/index.ts'; +import { qrCodeDataUrl } from '#lib/utils/qr-code.ts'; +import { describe, expect, it } from 'vitest'; + +/** A code of the right length, drawn from the alphabet llama-server generates from. */ +const CODE = Array.from( + { length: CODE_LENGTHS.SHARE }, + (_, index) => CODE_CHARS[index % CODE_CHARS.length] +).join(''); +/** The same alphabet, shifted: a different code of the same shape. */ +const OTHER = Array.from( + { length: CODE_LENGTHS.SHARE }, + (_, index) => CODE_CHARS[(index + 7) % CODE_CHARS.length] +).join(''); +/** Module size the renderer draws with, in SVG units. */ +const PIXEL_SIZE = 4; + +function svgOf(url: string | null): string { + expect(url).not.toBeNull(); + + return decodeURIComponent(url!.split(',')[1]); +} + +describe('access code QR', () => { + it('renders the code as an SVG data URL', () => { + const svg = svgOf(qrCodeDataUrl(CODE)); + + expect(svg.startsWith(' { + const svg = svgOf(qrCodeDataUrl(CODE)); + + expect(svg).toContain('fill="#ffffff"'); + expect(svg).toContain('fill="#000000"'); + }); + + it('sizes the code for a whole access code, not a fragment', () => { + const svg = svgOf(qrCodeDataUrl(CODE)); + const side = Number(/viewBox="0 0 (\d+)/.exec(svg)?.[1]) / PIXEL_SIZE; + + // 40 bytes at ECC M land in version 3: 29 modules, plus the 4 module quiet zone + expect(side).toBe(37); + }); + + it('carries the code the scanner accepts, spacing and all', () => { + const spaced = `${CODE.slice(0, 8)} ${CODE.slice(8, 20)}\n${CODE.slice(20)}`; + + expect(qrCodeDataUrl(spaced)).toBe(qrCodeDataUrl(CODE)); + }); + + it('encodes the code it was handed', () => { + expect(qrCodeDataUrl(OTHER)).not.toBe(qrCodeDataUrl(CODE)); + }); + + it('renders nothing for a value that is not a code', () => { + expect(qrCodeDataUrl('')).toBeNull(); + expect(qrCodeDataUrl('not a code')).toBeNull(); + expect(qrCodeDataUrl(CODE.slice(0, CODE_LENGTHS.SHARE - 1))).toBeNull(); + }); +}); diff --git a/ui/apps/chat/tests/unit/remote-pairing-setting.test.ts b/ui/apps/chat/tests/unit/remote-pairing-setting.test.ts new file mode 100644 index 000000000000..ed36c4f061d0 --- /dev/null +++ b/ui/apps/chat/tests/unit/remote-pairing-setting.test.ts @@ -0,0 +1,68 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const config: Record = {}; + +vi.mock('#lib/stores/settings/index.svelte.ts', () => ({ + settingsStore: { getConfig: (key: string) => config[key] } +})); + +vi.mock('#lib/utils/api-base.ts', () => ({ + getServers: () => [ + { + baseUrl: '', + code: 'AbCd2345' + 'x'.repeat(32), + enabled: true, + id: 'remote-1', + kind: 'remote', + name: 'Remote' + } + ] +})); + +const connect = vi.fn(); + +vi.mock('#lib/utils/webrtc-tunnel.ts', () => ({ + ClientTunnel: class { + connect = connect; + disconnect = vi.fn(); + } +})); + +import { + SETTING_CONFIG_DEFAULT, + SETTINGS_CHAT_SECTIONS, + SETTINGS_KEYS +} from '#lib/constants/index.ts'; +import { remoteAccessStore } from '#lib/stores/remote-access.svelte.ts'; + +describe('remote pairing setting', () => { + beforeEach(() => { + connect.mockReset().mockResolvedValue(undefined); + delete config[SETTINGS_KEYS.ENABLE_REMOTE_PAIRING]; + remoteAccessStore.disconnect('remote-1'); + }); + + it('is off by default, marked experimental and carries the security note', () => { + const field = SETTINGS_CHAT_SECTIONS.flatMap((section) => section.fields ?? []).find( + (candidate) => candidate.key === SETTINGS_KEYS.ENABLE_REMOTE_PAIRING + ); + + expect(SETTING_CONFIG_DEFAULT[SETTINGS_KEYS.ENABLE_REMOTE_PAIRING]).toBe(false); + expect(field?.isExperimental).toBe(true); + expect(field?.note).toMatch(/pass code/); + }); + + it('opens no tunnel while it is off', async () => { + await expect(remoteAccessStore.connect('remote-1')).rejects.toThrow(/turned off/); + expect(connect).not.toHaveBeenCalled(); + }); + + it('opens the tunnel once it is on', async () => { + config[SETTINGS_KEYS.ENABLE_REMOTE_PAIRING] = true; + + await remoteAccessStore.connect('remote-1'); + + expect(connect).toHaveBeenCalledOnce(); + expect(remoteAccessStore.isConnected('remote-1')).toBe(true); + }); +}); diff --git a/ui/apps/chat/tests/unit/server-props.test.ts b/ui/apps/chat/tests/unit/server-props.test.ts new file mode 100644 index 000000000000..2dde59e5e378 --- /dev/null +++ b/ui/apps/chat/tests/unit/server-props.test.ts @@ -0,0 +1,47 @@ +// Guards the one question that keeps a foreign endpoint out of the servers +// list: does /props answer like a llama.cpp server? A negative that is only a +// guess would lock a working server out of the UI, so "unknown" stays unknown. + +import { isLlamaCppProps } from '#lib/utils/server-props.ts'; +import { describe, expect, it } from 'vitest'; + +describe('llama.cpp props', () => { + it('accepts the props of a model server', () => { + expect( + isLlamaCppProps({ + default_generation_settings: { n_ctx: 4096, params: {} }, + model_path: '/models/qwen.gguf', + role: 'model', + total_slots: 1 + }) + ).toBe(true); + }); + + it('accepts the props of a router', () => { + expect( + isLlamaCppProps({ + build_info: 'b1234', + default_generation_settings: { n_ctx: 0, params: {} }, + role: 'router' + }) + ).toBe(true); + }); + + it('accepts a router that reports its role and build only', () => { + expect(isLlamaCppProps({ build_info: 'b1234', role: 'router' })).toBe(true); + }); + + it('refuses the answer of another runtime', () => { + expect(isLlamaCppProps({ data: [], object: 'list' })).toBe(false); + expect(isLlamaCppProps({ message: 'Not Found' })).toBe(false); + expect(isLlamaCppProps({ default_generation_settings: 'nope' })).toBe(false); + // a role or a build string on its own is a coincidence, not a llama.cpp server + expect(isLlamaCppProps({ role: 'router' })).toBe(false); + }); + + it('refuses anything that is not an object', () => { + expect(isLlamaCppProps(null)).toBe(false); + expect(isLlamaCppProps('llama.cpp')).toBe(false); + expect(isLlamaCppProps([])).toBe(false); + }); +}); diff --git a/ui/apps/chat/tests/unit/server.test.ts b/ui/apps/chat/tests/unit/server.test.ts new file mode 100644 index 000000000000..a96846e49ac6 --- /dev/null +++ b/ui/apps/chat/tests/unit/server.test.ts @@ -0,0 +1,290 @@ +// Guards the addressing of the server transport: a remote server is an +// address that cannot resolve on purpose, an access code is only usable in its +// exact shape, and the stored server list decides what the tunnel transport is +// allowed to route. A regression here silently sends a request to the wrong +// server, so these are cheap to check and expensive to lose. + +import { CODE_CHARS, CODE_LENGTHS, TUNNEL_HOST_SUFFIX } from '#lib/constants/index.ts'; +import { ServerCardState, ServerKind, TunnelStatus } from '#lib/enums/index.ts'; +import { type ServerCardFacts, serverCardStatusOf } from '#lib/utils/server-status.ts'; +import { + createLocalServer, + createRemoteServer, + createUrlServer, + isLocalServer, + isReachableServer, + isRemoteServer, + normalizeShareCode, + parsePresetServers, + parseServersSettings, + parseTunnelOrigin, + presetServersToAdd, + serverOrigin, + tunnelHostLabel, + tunnelOrigin +} from '#lib/utils/servers.ts'; +import { describe, expect, it } from 'vitest'; + +/** A code of the right length, drawn from the alphabet llama-server generates from. */ +const CODE = Array.from( + { length: CODE_LENGTHS.SHARE }, + (_, index) => CODE_CHARS[index % CODE_CHARS.length] +).join(''); + +describe('access codes', () => { + it('accepts a code the user pasted with spacing', () => { + expect(normalizeShareCode(` ${CODE.slice(0, 8)} ${CODE.slice(8)}\n`)).toBe(CODE); + }); + + it('refuses a code of the wrong length', () => { + expect(normalizeShareCode(CODE.slice(0, 39))).toBe(''); + expect(normalizeShareCode(`${CODE}x`)).toBe(''); + }); + + it('refuses characters the code alphabet leaves out', () => { + // `I` is not in CODE_CHARS: it is too easy to misread off a terminal + expect(normalizeShareCode(`${'I'.repeat(8)}${CODE.slice(8)}`)).toBe(''); + }); +}); + +describe('tunnel origins', () => { + it('addresses a server by a host that cannot resolve', () => { + expect(tunnelOrigin('server-1')).toBe('https://server-1.remote.llamacpp.invalid'); + }); + + it('reads the server back out of the address it wrote', () => { + expect(parseTunnelOrigin(tunnelOrigin('server-1'))).toBe('server-1'); + expect(parseTunnelOrigin(new URL(tunnelOrigin('r2')))).toBe('r2'); + }); + + it('leaves anything that is not a tunnel origin alone', () => { + expect(parseTunnelOrigin('https://example.com/v1/models')).toBeNull(); + + // a host that merely ends in the suffix without a label is not a server + expect(parseTunnelOrigin('https://remote.llamacpp.evil/props')).toBeNull(); + }); + + it('matches a host a server id was sanitized into', () => { + expect(tunnelHostLabel('Bad Id!')).toBe('bad-id'); + expect(parseTunnelOrigin(tunnelOrigin('Bad Id!'))).toBe('bad-id'); + }); +}); + +describe('server origins', () => { + it('serves the built-in server from the page origin', () => { + expect(serverOrigin(createLocalServer())).toBe(''); + expect(isLocalServer(createLocalServer())).toBe(true); + expect(isLocalServer(null)).toBe(true); + }); + + it('strips the trailing slash of a URL server', () => { + const server = createUrlServer('https://example.com/llama/', 'u1', 'Office'); + + expect(serverOrigin(server)).toBe('https://example.com/llama'); + expect(server.name).toBe('Office'); + }); + + it('addresses a remote server by its tunnel origin', () => { + const server = createRemoteServer(CODE, 'r1'); + + expect(isRemoteServer(server)).toBe(true); + expect(serverOrigin(server)).toBe(tunnelOrigin('r1')); + }); +}); + +describe('stored server list', () => { + it('reads a legacy entry as a URL server', () => { + const [server] = parseServersSettings([ + { baseUrl: 'https://example.com/llama/', id: 'u1', name: 'O' } + ]); + + expect(server.kind).toBe(ServerKind.URL); + expect(server.baseUrl).toBe('https://example.com/llama'); + }); + + it('keeps the API key of a stored server', () => { + const [server] = parseServersSettings([ + { apiKey: 'sk-secret', baseUrl: 'https://example.com', id: 'u1', name: 'Office' } + ]); + + // the key is the credential of that server: dropping it here turns every + // request into an unauthenticated one + expect(server.apiKey).toBe('sk-secret'); + }); + + it('keeps the API key of a stored remote server', () => { + const [server] = parseServersSettings([ + { apiKey: 'sk-secret', code: CODE, id: 'r1', kind: 'remote', name: 'Home' } + ]); + + expect(server.apiKey).toBe('sk-secret'); + }); + + it('carries the API key of a remote server the form builds', () => { + const server = createRemoteServer(CODE, 'r1', 'Home', 'sk-secret'); + + expect(server.apiKey).toBe('sk-secret'); + }); + + it('reads a remote entry and drops one without a usable code', () => { + const servers = parseServersSettings([ + { code: CODE, id: 'r1', kind: 'remote', name: 'Home' }, + { code: 'nope', id: 'r2', kind: 'remote' } + ]); + + expect(servers.map((server) => server.id)).toEqual(['r1']); + expect(servers[0].kind).toBe(ServerKind.REMOTE); + expect(servers[0].baseUrl).toBe(''); + }); + + it('refuses a URL server squatting on the tunnel host', () => { + expect( + parseServersSettings([{ baseUrl: 'https://r1.remote.llamacpp.invalid', id: 'u1' }]) + ).toEqual([]); + }); + + it('drops the built-in server, which is never persisted', () => { + expect(parseServersSettings([{ baseUrl: '', id: 'local', name: 'Local' }])).toEqual([]); + }); + + it('ignores a value it cannot parse', () => { + expect(parseServersSettings('{oops')).toEqual([]); + expect(parseServersSettings(undefined)).toEqual([]); + expect(parseServersSettings('not json')).toEqual([]); + }); +}); + +describe('reachability', () => { + it('always reaches the built-in server', () => { + expect(isReachableServer(createLocalServer())).toBe(true); + }); + + it('needs a code from a remote server and a URL from a URL one', () => { + expect(isReachableServer(createRemoteServer(CODE, 'r1'))).toBe(true); + expect(isReachableServer({ ...createRemoteServer(CODE, 'r1'), code: '' })).toBe(false); + expect(isReachableServer(createUrlServer('https://example.com', 'u1'))).toBe(true); + expect(isReachableServer(createUrlServer('not a url', 'u2'))).toBe(false); + expect(isReachableServer(createUrlServer('', 'u3'))).toBe(false); + }); +}); + +/** Facts of a server that was never listed and never connected. */ +function facts(overrides: Partial = {}): ServerCardFacts { + return { + enabled: true, + modelCount: 0, + modelsError: '', + modelsLoaded: false, + modelsLoading: false, + remote: false, + tunnel: TunnelStatus.IDLE, + tunnelError: '', + ...overrides + }; +} + +describe('card status', () => { + it('names a running handshake', () => { + const status = serverCardStatusOf(facts({ remote: true, tunnel: TunnelStatus.CONNECTING })); + + expect(status.state).toBe(ServerCardState.CONNECTING); + expect(status.label).toBe('Connecting'); + expect(status.detail).toBe('Opening the tunnel to the host.'); + }); + + it('keeps an idle remote server a choice, not a failure', () => { + const status = serverCardStatusOf(facts({ remote: true, tunnel: TunnelStatus.IDLE })); + + expect(status.state).toBe(ServerCardState.IDLE); + expect(status.label).toBe('Not connected'); + expect(status.detail).toBe('Paste the code the host prints, or press Connect.'); + }); + + it('carries the transport sentence of a dropped channel', () => { + const status = serverCardStatusOf( + facts({ + remote: true, + tunnel: TunnelStatus.ERROR, + tunnelError: 'authentication failed: invalid passcode' + }) + ); + + expect(status.state).toBe(ServerCardState.CHANNEL_DOWN); + expect(status.label).toBe('Channel down'); + expect(status.detail).toBe('authentication failed: invalid passcode'); + }); + + it('reports a listing that failed behind an open channel as unreachable', () => { + const status = serverCardStatusOf( + facts({ modelsError: 'host refused', remote: true, tunnel: TunnelStatus.CONNECTED }) + ); + + expect(status.state).toBe(ServerCardState.UNREACHABLE); + expect(status.label).toBe('Unreachable'); + expect(status.detail).toBe('host refused'); + }); + + it('counts the models of a server that answered', () => { + expect(serverCardStatusOf(facts({ modelCount: 1, modelsLoaded: true })).label).toBe('1 model'); + expect(serverCardStatusOf(facts({ modelCount: 4, modelsLoaded: true })).label).toBe('4 models'); + }); + + it('says what is being checked, tunnel or not', () => { + const direct = serverCardStatusOf(facts({ modelsLoading: true })); + const tunneled = serverCardStatusOf( + facts({ modelsLoading: true, remote: true, tunnel: TunnelStatus.CONNECTED }) + ); + + expect(direct.label).toBe('Checking'); + expect(direct.detail).toBe('Asking the server for the models it serves.'); + expect(tunneled.detail).toBe('Asking the host for the models it serves.'); + }); + + it('reports nothing for a server that is switched off', () => { + const status = serverCardStatusOf(facts({ enabled: false, modelsError: 'host refused' })); + + expect(status.state).toBe(ServerCardState.NONE); + expect(status.label).toBe(''); + expect(status.detail).toBe(''); + }); +}); + +describe('preset servers (llama-chat.json)', () => { + it('reads URLs and objects, trims the trailing slash and drops duplicates', () => { + expect( + parsePresetServers({ + servers: [ + { url: 'http://127.0.0.1:9931/' }, + 'https://llm.example.com/api', + { url: 'http://127.0.0.1:9931' } + ] + }) + ).toEqual(['http://127.0.0.1:9931', 'https://llm.example.com/api']); + }); + + it('drops what is not an http(s) URL, and anything but a servers list', () => { + expect( + parsePresetServers({ + servers: [ + { url: 'ftp://x' }, + { url: 42 }, + null, + 'not a url', + `https://a.${TUNNEL_HOST_SUFFIX}` + ] + }) + ).toEqual([]); + expect(parsePresetServers(null)).toEqual([]); + expect(parsePresetServers('')).toEqual([]); + expect(parsePresetServers({ servers: 'http://127.0.0.1:9931' })).toEqual([]); + }); + + it('adds only servers that are not stored and were not added before', () => { + const stored = [createUrlServer('http://127.0.0.1:9931', 'a')]; + const urls = ['http://127.0.0.1:9931', 'http://127.0.0.1:9932', 'http://127.0.0.1:9933']; + + expect(presetServersToAdd(urls, stored, ['http://127.0.0.1:9933'])).toEqual([ + 'http://127.0.0.1:9932' + ]); + }); +}); diff --git a/ui/apps/chat/tests/unit/servers-service.test.ts b/ui/apps/chat/tests/unit/servers-service.test.ts new file mode 100644 index 000000000000..341360f1eadc --- /dev/null +++ b/ui/apps/chat/tests/unit/servers-service.test.ts @@ -0,0 +1,84 @@ +// Guards the two questions a connectivity check answers: does the server serve +// models, and is it a llama.cpp server at all. The UI drives llama.cpp +// endpoints only, and an endpoint that speaks just the OpenAI API answers the +// listing happily, so the second question is what keeps it out. + +import { ServersService } from '#lib/services/servers.service.ts'; +import { createUrlServer } from '#lib/utils/servers.ts'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const SERVER = createUrlServer('https://example.com', 'u1', 'Office'); +const MODELS = JSON.stringify({ data: [{ id: 'qwen.gguf' }], object: 'list' }); +const PROPS = JSON.stringify({ + default_generation_settings: { n_ctx: 4096, params: {} }, + model_path: '/models/qwen.gguf', + role: 'model' +}); + +/** Serve the two endpoints a check reads, with the answers the test wants. */ +function stubEndpoint(options: { props?: Response; propsBody?: string } = {}) { + vi.stubGlobal('fetch', async (input: RequestInfo | URL) => { + const url = String(input instanceof Request ? input.url : input); + + if (url.endsWith('/props')) { + return options.props ?? new Response(options.propsBody ?? PROPS, { status: 200 }); + } + + return new Response(MODELS, { status: 200 }); + }); +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('server check', () => { + it('accepts a server that lists models and answers props', async () => { + stubEndpoint(); + + const result = await ServersService.test(SERVER); + + expect(result.ok).toBe(true); + expect(result.modelCount).toBe(1); + }); + + it('refuses an endpoint that is not a llama.cpp server', async () => { + stubEndpoint({ propsBody: JSON.stringify({ data: [], object: 'list' }) }); + + const result = await ServersService.test(SERVER); + + expect(result.ok).toBe(false); + expect(result.error).toMatch(/not a llama\.cpp server/); + }); + + it('keeps an endpoint whose props cannot be read', async () => { + // a reverse proxy that hides /props is not proof of a foreign server + stubEndpoint({ props: new Response('Not Found', { status: 404 }) }); + + const result = await ServersService.test(SERVER); + + expect(result.ok).toBe(true); + }); +}); + +describe('server probe', () => { + it('reports a key-protected endpoint as wanting one, and knows nothing else', async () => { + stubEndpoint({ props: new Response('Unauthorized', { status: 401 }) }); + + expect(await ServersService.probe(SERVER)).toEqual({ authRequired: true, llamaCpp: null }); + }); + + it('reports what the endpoint is when props answer', async () => { + stubEndpoint(); + + expect(await ServersService.probe(SERVER)).toEqual({ authRequired: false, llamaCpp: true }); + }); + + it('reports nothing for an endpoint that does not answer', async () => { + vi.stubGlobal('fetch', async () => { + throw new Error('Failed to fetch'); + }); + + expect(await ServersService.probe(SERVER)).toEqual({ authRequired: false, llamaCpp: null }); + }); +});