diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md
index 538485fe9086..49c6fcb4f3c5 100644
--- a/ui/apps/chat/README.md
+++ b/ui/apps/chat/README.md
@@ -19,21 +19,22 @@ It works with a llama-server in either mode: **MODEL** (one model) or **ROUTER**
Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app as static files on localhost, or writes them to a folder for any static file server (see [Self-hosting](#self-hosting)); it brings no server and no models of its own. It needs Node.js 20 or later.
```bash
-npx @ggml-org/llama-chat # serve it on http://127.0.0.1:3000 and open the browser
-npx llama-chat --host 0.0.0.0 --no-open # the same, by its short name, reachable from other devices
-npx llama-chat build # write the static app to ./llama-chat instead
+npx @ggml-org/llama-chat --server http://127.0.0.1:9931 # serve it on http://127.0.0.1:3000 with that server added
+npx llama-chat --host 0.0.0.0 --no-open # the same, by its short name, reachable from other devices
+npx llama-chat build --server https://llm.example.com # write the static app to ./llama-chat instead
```
| Flag | Default | What it does |
| --------------------------------- | ----------- | ------------------------------------------------------------------------------------------- |
+| `--server ` | | A llama-server the app adds on startup, once; repeatable. Its API key goes in the app |
| `--port ` | `3000` | Fixed, so the origin, and the chats the browser stores under it, stay the same between runs |
| `--host ` | `127.0.0.1` | `0.0.0.0` to open it to the LAN |
| `--open` / `--no-open` | open | Opens the default browser once it listens |
| `-v`, `--version`, `-h`, `--help` | | As usual |
-The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`) and `--force`, which lets it write into a folder that is not empty.
+The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`), `--force`, which lets it write into a folder that is not empty, and `--server`.
-How it works: the CLI only serves the files of `dist`, with llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good), and forwards nothing. The browser talks to each llama-server you add in the app by URL directly, so llama-server's own rules apply: start it with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled, and with `--api-key` when it is reachable from the LAN. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot.
+How it works: the CLI only serves the files of `dist`, with llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good), and forwards nothing. `--server` only puts the address in `llama-chat.json` next to `index.html`, which the app reads at startup to add the server; one you remove in the app stays removed. The CLI marks `index.html` with ``, and only a page with that tag asks for the file (`build --server` writes both). The browser talks to each llama-server directly, so llama-server's own rules apply: start it with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled, and with `--api-key` when it is reachable from the LAN. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot.
## Self-hosting
diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js
index 8ef0f557c3a4..70a7a33c80cd 100644
--- a/ui/apps/chat/cli/server.js
+++ b/ui/apps/chat/cli/server.js
@@ -6,12 +6,15 @@
*
* It only serves files and forwards nothing: the browser calls each llama-server the app is
* pointed at directly, so llama-server's own rules (--cors-origins, --api-key) apply as they do
- * to any other page.
+ * to any other page. `--server` only tells the app which servers to add: the CLI serves them in
+ * `llama-chat.json` next to index.html, and `build` writes that file, which the app reads at
+ * startup. index.html carries a marker for it, so a page served any other way, such as by
+ * llama-server or the dev server, does not ask for a file that is not there.
*/
import { spawn } from 'node:child_process';
import { createReadStream, readFileSync } from 'node:fs';
-import { cp, readdir, stat } from 'node:fs/promises';
+import { cp, readdir, readFile, stat, writeFile } from 'node:fs/promises';
import http from 'node:http';
import { extname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -28,6 +31,15 @@ export const DEFAULTS = Object.freeze({
port: 3000
});
+/** The file next to index.html that lists the servers the app adds at startup. */
+export const PRESET_FILE = 'llama-chat.json';
+
+/**
+ * The marker in index.html that sends the app to `llama-chat.json`. It names the file rather
+ * than the servers: the service worker keeps index.html, so servers in it would outlive a run.
+ */
+export const PRESET_META = ``;
+
// The same caching llama-server applies to the UI it embeds (tools/server/server-http.cpp):
// index.html and the PWA's update checks are revalidated, hashed assets are cached for good.
const NO_CACHE = new Set([
@@ -36,7 +48,8 @@ const NO_CACHE = new Set([
'/sw.js',
'/manifest.webmanifest',
'/_app/version.json',
- '/build.json'
+ '/build.json',
+ `/${PRESET_FILE}`
]);
const CACHE_IMMUTABLE = 'public, max-age=31536000, immutable';
const CACHE_REVALIDATE = 'no-cache';
@@ -65,13 +78,15 @@ const SERVE_OPTIONS = ['host', 'open', 'port'];
const BUILD_OPTIONS = ['force', 'out'];
export const USAGE = `Usage: llama-chat [options]
- llama-chat build [--out ] [--force]
+ llama-chat build [--out ] [--force] [--server ]
Serves Llama Chat on localhost as static files, or, with build, writes them to
a folder for any HTTP server. The browser talks to llama-server directly: start
it with --cors-origins set to this page's address when it restricts origins.
Options:
+ --server a llama-server the app adds on startup, repeatable; the API
+ key, if it needs one, goes in the app
--port port to listen on (default: ${DEFAULTS.port})
--host address to listen on, 0.0.0.0 for the LAN (default: ${DEFAULTS.host})
--open open the browser once it listens (default)
@@ -82,11 +97,12 @@ Options:
Build options:
--out folder to write to (default: ./${DEFAULTS.out})
--force write into a folder that is not empty
+ --server as above, written to ${PRESET_FILE} in the folder
Examples:
- npx @ggml-org/llama-chat
+ npx @ggml-org/llama-chat --server http://127.0.0.1:9931
npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open
- npx @ggml-org/llama-chat build --out ./public`;
+ npx @ggml-org/llama-chat build --out ./public --server https://llm.example.com`;
/** The version in this package's package.json. */
export function packageVersion() {
@@ -97,7 +113,7 @@ export function packageVersion() {
* Reads the command line. No verb serves the app, `start` means the same, and `build` writes it
* to a folder.
* @param {string[]} argv the arguments after the command
- * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, version: boolean }}
+ * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, servers: string[], version: boolean }}
*/
export function parseCliArgs(argv) {
const { positionals, values } = parseArgs({
@@ -111,6 +127,7 @@ export function parseCliArgs(argv) {
open: { type: 'boolean' },
out: { type: 'string' },
port: { type: 'string' },
+ server: { multiple: true, type: 'string' },
version: { short: 'v', type: 'boolean' }
},
strict: true
@@ -146,10 +163,54 @@ export function parseCliArgs(argv) {
open: values.open ?? true,
out: values.out ?? DEFAULTS.out,
port,
+ servers: (values.server ?? []).map(parseServerUrl),
version: values.version ?? false
};
}
+/**
+ * A llama-server address for the app, with or without `http://`, as the app stores it: no
+ * trailing slash, a path kept (for `--api-prefix`).
+ * @param {string} value
+ */
+export function parseServerUrl(value) {
+ let url;
+
+ try {
+ url = new URL(/^[a-z][a-z\d+.-]*:\/\//i.test(value) ? value : `http://${value}`);
+ } catch {
+ throw new Error(`invalid --server: ${value}`);
+ }
+
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+ throw new Error(`invalid --server: ${value} (use an http:// or https:// address)`);
+ }
+
+ if (url.search || url.hash || url.username || url.password) {
+ throw new Error(`invalid --server: ${value} (no query, fragment or credentials)`);
+ }
+
+ return url.href.replace(/\/+$/, '');
+}
+
+/**
+ * index.html with the marker for `llama-chat.json` at the end of its head, once.
+ * @param {string} html
+ */
+export function withPresetMeta(html) {
+ if (html.includes('name="llama-chat-servers"')) return html;
+
+ return html.replace('', `\t${PRESET_META}\n\t`);
+}
+
+/**
+ * The contents of `llama-chat.json`.
+ * @param {string[]} servers
+ */
+export function presetFile(servers) {
+ return `${JSON.stringify({ servers: servers.map((url) => ({ url })) }, null, '\t')}\n`;
+}
+
/**
* Maps a request path to a file under `root`, or null when it falls outside it.
* @param {string} root
@@ -181,7 +242,10 @@ function resolveStatic(root, pathname) {
*/
function sendFile(req, res, file, pathname, stats) {
const name = pathname === '/' ? '/index.html' : pathname;
- const etag = `W/"${stats.size.toString(16)}-${Math.floor(stats.mtimeMs).toString(16)}"`;
+ // index.html goes out with the marker for llama-chat.json, and an ETag of its own
+ const body =
+ name === '/index.html' ? Buffer.from(withPresetMeta(readFileSync(file, 'utf8'))) : null;
+ const etag = `W/"${(body?.length ?? stats.size).toString(16)}-${Math.floor(stats.mtimeMs).toString(16)}"`;
res.setHeader('ETag', etag);
res.setHeader('Cache-Control', NO_CACHE.has(name) ? CACHE_REVALIDATE : CACHE_IMMUTABLE);
@@ -201,7 +265,7 @@ function sendFile(req, res, file, pathname, stats) {
'Content-Type',
MIME_TYPES[extname(file).toLowerCase()] ?? 'application/octet-stream'
);
- res.setHeader('Content-Length', stats.size);
+ res.setHeader('Content-Length', body?.length ?? stats.size);
res.writeHead(200);
if (req.method === 'HEAD') {
@@ -210,6 +274,12 @@ function sendFile(req, res, file, pathname, stats) {
return;
}
+ if (body) {
+ res.end(body);
+
+ return;
+ }
+
createReadStream(file).pipe(res);
}
@@ -224,11 +294,13 @@ function sendError(res, status, message) {
}
/**
- * The HTTP server: the files of `dist`, read-only.
- * @param {{ distDir?: string }} [options]
+ * The HTTP server: the files of `dist`, read-only, and `llama-chat.json` with the servers given,
+ * if any: index.html always points the app at it.
+ * @param {{ distDir?: string, servers?: string[] }} [options]
*/
-export function createChatServer({ distDir = DIST_DIR } = {}) {
+export function createChatServer({ distDir = DIST_DIR, servers = [] } = {}) {
const root = normalize(distDir);
+ const preset = Buffer.from(presetFile(servers));
return http.createServer(async (req, res) => {
const { pathname } = new URL(req.url ?? '/', 'http://localhost');
@@ -240,6 +312,17 @@ export function createChatServer({ distDir = DIST_DIR } = {}) {
return;
}
+ if (pathname === `/${PRESET_FILE}`) {
+ res.writeHead(200, {
+ 'Cache-Control': CACHE_REVALIDATE,
+ 'Content-Length': preset.length,
+ 'Content-Type': MIME_TYPES['.json']
+ });
+ res.end(req.method === 'HEAD' ? undefined : preset);
+
+ return;
+ }
+
const file = resolveStatic(root, pathname);
const stats = file ? await stat(file).catch(() => null) : null;
@@ -255,11 +338,12 @@ export function createChatServer({ distDir = DIST_DIR } = {}) {
/**
* Writes the app to a folder: the files of `dist` without source maps, ready for any static
- * file server. Refuses a folder that is not empty unless `force` is set, and a folder in `dist`.
- * @param {{ distDir?: string, force?: boolean, out: string }} options
+ * file server, and `llama-chat.json` when servers are given, with the marker in index.html. Refuses a folder that is not empty
+ * unless `force` is set, and a folder in `dist`.
+ * @param {{ distDir?: string, force?: boolean, out: string, servers?: string[] }} options
* @returns {Promise} the absolute path of the folder
*/
-export async function buildStatic({ distDir = DIST_DIR, force = false, out }) {
+export async function buildStatic({ distDir = DIST_DIR, force = false, out, servers = [] }) {
const source = resolve(distDir);
const target = resolve(out);
const inside = relative(source, target);
@@ -276,6 +360,13 @@ export async function buildStatic({ distDir = DIST_DIR, force = false, out }) {
await cp(source, target, { filter: (file) => !file.endsWith('.map'), recursive: true });
+ if (servers.length > 0) {
+ const index = join(target, 'index.html');
+
+ await writeFile(join(target, PRESET_FILE), presetFile(servers));
+ await writeFile(index, withPresetMeta(await readFile(index, 'utf8')));
+ }
+
return target;
}
@@ -339,10 +430,17 @@ export async function main(argv) {
if (options.command === 'build') {
try {
- const target = await buildStatic({ force: options.force, out: options.out });
+ const target = await buildStatic({
+ force: options.force,
+ out: options.out,
+ servers: options.servers
+ });
const shown = relative(process.cwd(), target) || '.';
console.log(`Llama Chat ${packageVersion()} written to ${target}`);
+
+ for (const server of options.servers) console.log(`Adds the llama-server at ${server}`);
+
console.log(`Serve it with any static file server, e.g.: npx http-server ${shown}`);
} catch (error) {
console.error(`llama-chat: ${error instanceof Error ? error.message : error}`);
@@ -352,7 +450,7 @@ export async function main(argv) {
return;
}
- const server = createChatServer();
+ const server = createChatServer({ servers: options.servers });
return new Promise((resolve) => {
server.on('error', (error) => {
@@ -373,6 +471,9 @@ export async function main(argv) {
const url = `http://${shown.includes(':') ? `[${shown}]` : shown}:${address.port}/`;
console.log(`Llama Chat ${packageVersion()} on ${url}`);
+
+ for (const server of options.servers) console.log(`Adds the llama-server at ${server}`);
+
console.log(
`If llama-server restricts origins (tools or MCP on), start it with --cors-origins ${url.slice(0, -1)}`
);
diff --git a/ui/apps/chat/package-lock.json b/ui/apps/chat/package-lock.json
index a0005085ed9a..ed6e762d09ef 100644
--- a/ui/apps/chat/package-lock.json
+++ b/ui/apps/chat/package-lock.json
@@ -91,6 +91,7 @@
"typescript-eslint": "8.72.1",
"unified": "11.0.5",
"unist-util-visit": "5.1.0",
+ "uqr": "0.1.3",
"uuid": "13.0.2",
"vaul-svelte": "1.0.0-next.7",
"vite": "8.3.6",
@@ -12450,6 +12451,13 @@
"node": ">=18.12.0"
}
},
+ "node_modules/uqr": {
+ "version": "0.1.3",
+ "resolved": "https://registry.npmjs.org/uqr/-/uqr-0.1.3.tgz",
+ "integrity": "sha512-0rjE8iEJe4YmT9TOhwsZtqCMRLc5DXZUI2UEYUUg63ikBkqqE5EYWaI0etFe/5KUcmcYwLih2RND1kq+hrUJXA==",
+ "dev": true,
+ "license": "MIT"
+ },
"node_modules/uri-js": {
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
diff --git a/ui/apps/chat/package.json b/ui/apps/chat/package.json
index 2e9ef1849236..c38f70e10932 100644
--- a/ui/apps/chat/package.json
+++ b/ui/apps/chat/package.json
@@ -151,6 +151,7 @@
"typescript-eslint": "8.72.1",
"unified": "11.0.5",
"unist-util-visit": "5.1.0",
+ "uqr": "0.1.3",
"uuid": "13.0.2",
"vaul-svelte": "1.0.0-next.7",
"vite": "8.3.6",
diff --git a/ui/apps/chat/src/app.d.ts b/ui/apps/chat/src/app.d.ts
index 006a37a8cfaa..7b9e60c457f0 100644
--- a/ui/apps/chat/src/app.d.ts
+++ b/ui/apps/chat/src/app.d.ts
@@ -10,6 +10,7 @@ import type {
ApiChatCompletionStreamChunk,
ApiChatCompletionToolCall,
ApiChatCompletionToolCallDelta,
+ ApiChatCompletionUsage,
ApiChatMessageContentPart,
ApiChatMessageData,
ApiContextSizeError,
@@ -72,6 +73,7 @@ declare global {
ApiChatCompletionResponse,
ApiChatCompletionStreamChunk,
ApiChatCompletionToolCall,
+ ApiChatCompletionUsage,
ApiChatCompletionToolCallDelta,
ApiChatMessageData,
ApiChatMessageContentPart,
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte
index 802dd7e8d081..53c0820e1896 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte
@@ -104,7 +104,7 @@
{/snippet}
{#snippet body()}
- {#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded}
+ {#if gauge.canLoadActiveModel && gauge.activeModelId !== null && !gauge.isActiveModelLoaded}
{:else}
{@const source = toolsStore.getToolSource(toolName)}
- {@const providerName =
+ {@const serverName =
source === ToolSource.SERVER
? TOOL_SERVER_LABELS[ToolSource.SERVER]
: source === ToolSource.CUSTOM
? TOOL_SERVER_LABELS[ToolSource.CUSTOM]
: 'MCP Tools'}
onDecision(ToolPermissionDecision.ALWAYS_SERVER)}>
- Approve all tools from {providerName}
+ Approve all tools from {serverName}
{/if}
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
index 97c74861c62e..4b97863c4229 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte
@@ -26,6 +26,7 @@
CHAT_FORM_MOVE_TO_BOTTOM_IN_MS,
CHAT_FORM_MOVE_TO_BOTTOM_OUT_MS
} from '#lib/constants/index.ts';
+ import { useBackendAvailability } from '#lib/hooks/use-backend-availability.svelte.ts';
import { useChatScreenActiveModel } from '#lib/hooks/use-chat-screen-active-model.svelte.ts';
import { useChatScreenDragAndDrop } from '#lib/hooks/use-chat-screen-drag-and-drop.svelte.ts';
import { useChatScreenFileUpload } from '#lib/hooks/use-chat-screen-file-upload.svelte.ts';
@@ -77,8 +78,10 @@
let formWrapper: HTMLElement | undefined = $state();
let chatScreenForm: ChatScreenForm | undefined = $state();
let activeErrorDialog = $derived(chatStore.errorDialogState);
- let isServerLoading = $derived(serverStore.loading);
- let hasPropsError = $derived(!!serverStore.error);
+ // no local server in this deployment: never block on its probe
+ let isServerLoading = $derived(serverStore.loading && !serverStore.localServerMissing);
+ const availability = useBackendAvailability();
+ let hasPropsError = $derived(availability.isOffline);
let isCurrentConversationLoading = $derived(chatStore.isLoading || chatStore.isStreaming());
let chatFormBottomPosition = $derived.by(() => {
if (!deviceStore.isMobile) return CHAT_FORM_BOTTOM_OFFSET.desktop;
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte
index 22bde59ce057..720e397368ca 100644
--- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte
+++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte
@@ -63,8 +63,9 @@
Hello there
- {serverStore.props?.modalities?.audio ? 'Record audio, type a message ' : 'Type a message'} or upload
- files to get started
+ {serverStore.localProps?.modalities?.audio
+ ? 'Record audio, type a message '
+ : 'Type a message'} or upload files to get started
- Warning: This export will include sensitive data such as API keys and MCP server custom
- headers (e.g., authorization tokens). Do not share this file with anyone you don't
- trust.
+ Warning: This export will include sensitive data such as API keys, remote access codes
+ and MCP server custom headers (e.g., authorization tokens). Do not share this file with
+ anyone you don't trust.
{:else}
- Sensitive data (API keys, MCP server custom headers) will not be included in the export
- to protect your credentials.
+ Sensitive data (API keys, remote access codes, MCP server custom headers) will not be
+ included in the export to protect your credentials.
{/if}
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte
index eb20342ce3cd..994101386db8 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte
@@ -1,22 +1,29 @@
-{#if orgName && hubEnabled}
+{#if useServerIcon}
+
+
+ {#snippet fallback()}
+ {#if isLocal}
+
+ {:else}
+
+ {/if}
+ {/snippet}
+
+
+{:else if orgName && hubEnabled}
(isNearViewport = true))}
class={['inline-flex shrink-0', className]}
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte b/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte
index 023412df889d..cc978351d6e2 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte
@@ -1,4 +1,5 @@
+{#snippet serverMark(server: Server)}
+ {#if server.id === LOCAL_BACKEND_ID}
+
+ {#snippet fallback()}
+
+ {/snippet}
+
+ {:else}
+
+ {/if}
+{/snippet}
+
@@ -98,6 +136,51 @@
gapSize="2"
innerClass="items-center"
>
+ {#if availableServers.length > 1}
+
+
+ {#snippet child({ props })}
+
+ {/snippet}
+
+
+
+
+ Servers
+
+ {#each availableServers as server (server.id)}
+ toggleServer(server.id, checked)}
+ >
+ {@render serverMark(server)}
+
+ {server.name}
+
+ {serverCounts[server.id] ?? 0}
+
+ {/each}
+
+
+ {#if servers.length > 0}
+
+
+ (servers = [])}>Show every server
+ {/if}
+
+
+ {/if}
+
(contextLimit = Number(value))}
type="single"
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte
index 9f0183e3188f..f9dbab94f505 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte
@@ -14,7 +14,7 @@
import { conversationsStore, deviceStore, uiStore } from '#lib/stores/index.ts';
interface Props {
- /** A provider that cannot load or unload has no button to offer. */
+ /** A server that cannot load or unload has no button to offer. */
canToggleLoad?: boolean;
isDownload?: boolean;
isDownloadPaused?: boolean;
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte
index 1f25c3a5553c..de64c61f042e 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte
@@ -55,7 +55,7 @@
}
// the "more actions" dropdown and the right-click menu offer the same actions
- let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete, download));
+ let actions = $derived(modelRowActions(option, { download, favorite, isHidden }, { onDelete }));
let contextMenuOpen = $state(false);
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte
index e24f4e91b51b..9f66a93d75a9 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte
@@ -45,9 +45,10 @@
ModelCapability,
ModelDownloadConfirmAction,
ModelsTableGroupKind,
+ ModelsTableServerKind,
ModelsTableSortKey
} from '#lib/enums/index.ts';
- import { modelsStore, settingsStore } from '#lib/stores/index.ts';
+ import { modelsStore, serversModelsStore, settingsStore } from '#lib/stores/index.ts';
import type { ModelOption } from '#lib/types/models.js';
import { groupModelFamilies, type ModelFamilyGroup } from '#lib/utils/model-families.ts';
import type { Snippet } from 'svelte';
@@ -64,6 +65,10 @@
onSelect: (option: ModelOption) => void;
/** Modalities a model must support at least one of. */
modalities?: ModalityKey[];
+ /** Server ids to keep; empty keeps every server. */
+ servers?: string[];
+ /** Repos each server contributes to the current search, for the filter menu. */
+ serverCounts?: Record;
selectedId: string | null;
/** Rendered at the toolbar's right end, past the filters. */
toolbarEnd?: Snippet;
@@ -78,11 +83,15 @@
modalities = $bindable([]),
onSelect,
selectedId,
+ serverCounts = {},
+ servers = $bindable([]),
toolbarEnd
}: Props = $props();
let isEmpty = $derived(groups.every((group) => group.items.length === 0));
- let hasFilters = $derived(hasActiveFilters(contextLimit, modalities, capabilities));
+ let hasFilters = $derived(
+ hasActiveFilters(contextLimit, modalities, capabilities) || servers.length > 0
+ );
/** Noun the show-more row counts in, per unit of the grouped list. */
const SHOW_MORE_NOUNS: Record = {
@@ -95,7 +104,7 @@
/** Repos whose quants are folded away; the rest show them. */
const collapsedQuants = new SvelteSet();
/** Sections that list their models straight, without folding them into families. */
- const FLAT_SECTIONS = new Set([
+ const FLAT_SECTIONS = new Set([
ModelsTableGroupKind.DOWNLOADING,
ModelsTableGroupKind.FAVORITES,
ModelsTableGroupKind.LOADED
@@ -244,6 +253,7 @@
{onSelect}
option={quant}
selected={selectedId === quant.id}
+ showServer={entry.kind === 'servers'}
/>
{/each}
@@ -348,6 +358,8 @@
bind:contextLimit
bind:filter
bind:modalities
+ bind:servers
+ {serverCounts}
{toolbarEnd}
/>
@@ -382,15 +394,22 @@
{/if}
{/snippet}
+ {@const serverState = group.serverId ? serversModelsStore.get(group.serverId) : null}
+
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte
index 2099309e5e29..bb3e9b99cba0 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte
@@ -10,6 +10,7 @@
import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '#lib/constants/index.ts';
import { modelsStore } from '#lib/stores/index.ts';
import type { ModelOption } from '#lib/types/models.js';
+ import { getServer } from '#lib/utils/api-base.ts';
import { isAuxSidecar } from '#lib/utils/index.ts';
interface Props {
@@ -18,18 +19,28 @@
onDelete: (option: ModelOption) => void;
onSelect: (option: ModelOption) => void;
selected: boolean;
+ /** Badge the server name instead of the quant, for a server group. */
+ showServer?: boolean;
/** Left padding in px, from the nesting depth. */
indent?: number;
}
- let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props();
+ let {
+ indent = 0,
+ isFavorite,
+ onDelete,
+ onSelect,
+ option,
+ selected,
+ showServer = false
+ }: Props = $props();
let favorite = $derived(isFavorite(option));
let isHidden = $derived(modelsStore.isHidden(option.id));
let quant = $derived(option.parsedId?.quantization ?? option.model);
// the "more actions" dropdown and the right-click menu offer the same actions
- let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete));
+ let actions = $derived(modelRowActions(option, { favorite, isHidden }, { onDelete }));
let contextMenuOpen = $state(false);
@@ -59,7 +70,9 @@
style="padding-left: {indent}px"
type="button"
>
- {quant}
+
+ {showServer ? (getServer(option.serverId)?.name ?? quant) : quant}
+ !isAuxSidecar(badge.kind))}
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte
index eee8d9ebba44..bad443931284 100644
--- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte
@@ -20,10 +20,13 @@
let { entry, expanded, indent = 0, onToggle }: Props = $props();
+ let serverCount = $derived(new Set(entry.quants.map((option) => option.serverId ?? '')).size);
let groupLabel = $derived(
- entry.kind === ModelGroupKind.VARIANTS
- ? `${entry.quants.length} variants`
- : `${entry.quants.length} quants available`
+ entry.kind === ModelGroupKind.SERVERS
+ ? `${serverCount} server${serverCount === 1 ? '' : 's'}`
+ : entry.kind === ModelGroupKind.VARIANTS
+ ? `${entry.quants.length} variants`
+ : `${entry.quants.length} quants available`
);
let anyLoaded = $derived(entry.quants.some((quant) => modelsStore.isModelRunning(quant.model)));
// a repo row stands for its quants, so it reports what they agree on
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte
new file mode 100644
index 000000000000..896ebf3133bf
--- /dev/null
+++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte
@@ -0,0 +1,142 @@
+
+
+
+ {/if}
{/if}
diff --git a/ui/apps/chat/src/lib/components/app/settings/index.ts b/ui/apps/chat/src/lib/components/app/settings/index.ts
index 184a34fc7f51..d2422daf2e85 100644
--- a/ui/apps/chat/src/lib/components/app/settings/index.ts
+++ b/ui/apps/chat/src/lib/components/app/settings/index.ts
@@ -91,3 +91,8 @@ export { default as SettingsChatFields } from './SettingsChat/SettingsChatFields
* server favicons and permission management controls.
*/
export { default as SettingsChatToolsTab } from './SettingsChat/SettingsChatToolsTab.svelte';
+
+/**
+ * Servers panel for chat settings. Switches between the llama.cpp and
+ * OpenAI-compatible protocols and lists the endpoints configured for each.
+ */
diff --git a/ui/apps/chat/src/lib/constants/backend.constants.ts b/ui/apps/chat/src/lib/constants/backend.constants.ts
deleted file mode 100644
index 57f5c4213220..000000000000
--- a/ui/apps/chat/src/lib/constants/backend.constants.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-/** Marks the built-in local llama.cpp backend. */
-export const LOCAL_BACKEND_ID = 'local';
diff --git a/ui/apps/chat/src/lib/constants/index.ts b/ui/apps/chat/src/lib/constants/index.ts
index 9dfaa5e2bda7..6406b7eb621a 100644
--- a/ui/apps/chat/src/lib/constants/index.ts
+++ b/ui/apps/chat/src/lib/constants/index.ts
@@ -4,7 +4,8 @@
export * from './agentic.constants';
export * from './api-endpoints.constants';
export * from './app.constants';
-export * from './backend.constants';
+export * from './servers.constants';
+export * from './webrtc.constants';
export * from './chat-tabs.constants';
export * from './database.constants';
export * from './reasoning-effort.constants';
diff --git a/ui/apps/chat/src/lib/constants/routes.constants.ts b/ui/apps/chat/src/lib/constants/routes.constants.ts
index 1d7180540155..f01a5ac40b53 100644
--- a/ui/apps/chat/src/lib/constants/routes.constants.ts
+++ b/ui/apps/chat/src/lib/constants/routes.constants.ts
@@ -19,6 +19,8 @@ export const ROUTES = {
PROJECTS: '#/projects',
/** Search — mobile-only full-page conversation search. */
SEARCH: '#/search',
+ /** Servers - a page on a phone, a dialog elsewhere. */
+ SERVERS: '#/servers',
/** Settings - a page on a phone, a dialog elsewhere. */
SETTINGS: '#/settings',
/** Root — start of the app. */
@@ -31,6 +33,9 @@ export const MODELS_ROUTE_ID = '/models';
/** SvelteKit route id of every page an extension contributes, as `page.route.id` reports it. */
export const EXTENSION_PAGE_ROUTE_ID = '/[extensionPage]';
+/** SvelteKit route id of the servers page, as `page.route.id` reports it. */
+export const SERVERS_ROUTE_ID = '/servers';
+
/** SvelteKit route id of the settings page, as `page.route.id` reports it. */
export const SETTINGS_ROUTE_ID = '/settings';
diff --git a/ui/apps/chat/src/lib/constants/servers.constants.ts b/ui/apps/chat/src/lib/constants/servers.constants.ts
new file mode 100644
index 000000000000..3234bdbe8fa9
--- /dev/null
+++ b/ui/apps/chat/src/lib/constants/servers.constants.ts
@@ -0,0 +1,27 @@
+/** Prefix for generated ids of user-added servers. */
+export const SERVER_ID_PREFIX = 'server';
+
+/** Chat completions path every server serves. */
+export const DEFAULT_SERVER_CHAT_PATH = '/v1/chat/completions';
+
+/** Models listing path every server serves. */
+export const DEFAULT_SERVER_MODELS_PATH = '/v1/models';
+
+/** Id of the built-in backend that points at the server serving this UI. */
+export const LOCAL_BACKEND_ID = 'local';
+
+/** Favicon extract keyed by domain, for servers with no bundled mark. */
+export const FAVICON_SERVICE_URL = 'https://www.google.com/s2/favicons?domain=';
+
+/**
+ * File next to index.html listing servers the app adds on startup, as
+ * `{ "servers": [{ "url": "http://127.0.0.1:9931" }] }`. `llama-chat --server`
+ * serves it, and `llama-chat build --server` writes it.
+ */
+export const PRESET_SERVERS_FILE = 'llama-chat.json';
+
+/**
+ * Name of the meta tag in index.html that says {@link PRESET_SERVERS_FILE} is
+ * there: pages served any other way, such as by llama-server, never ask for it.
+ */
+export const PRESET_SERVERS_META = 'llama-chat-servers';
diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
index dcaed3da880a..545f5534b6cf 100644
--- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts
@@ -10,6 +10,7 @@ export const SETTINGS_KEYS = {
ALWAYS_SHOW_TOOL_CALL_CONTENT: 'alwaysShowToolCallContent',
API_KEY: 'apiKey',
AUTO_MIC_ON_EMPTY: 'autoMicOnEmpty',
+ // llama.cpp's own flag: "backend" here is the compute backend, not a server
BACKEND_SAMPLING: 'backend_sampling',
COMPACTION_PRESERVED_STEPS: 'compactionPreservedSteps',
COMPACTION_PROMPT: 'compactionPrompt',
@@ -29,11 +30,13 @@ export const SETTINGS_KEYS = {
DYNATEMP_RANGE: 'dynatemp_range',
ENABLE_CONTINUE_GENERATION: 'enableContinueGeneration',
ENABLE_DISCOVER_MODELS: 'enableDiscoverModels',
+ ENABLE_REMOTE_PAIRING: 'enableRemotePairing',
EXCLUDE_REASONING_FROM_CONTEXT: 'excludeReasoningFromContext',
FREQUENCY_PENALTY: 'frequency_penalty',
FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks',
GROUP_MODELS_BY_FAMILY: 'groupModelsByFamily',
LIVE_MARKDOWN_PREVIEW: 'liveMarkdownPreview',
+ LOCAL_BACKEND_ENABLED: 'localBackendEnabled',
MAX_IMAGE_RESOLUTION: 'maxImageMPixels',
MAX_TOKENS: 'max_tokens',
MCP_REQUEST_TIMEOUT_SECONDS: 'mcpRequestTimeoutSeconds',
@@ -54,6 +57,8 @@ export const SETTINGS_KEYS = {
REPEAT_PENALTY: 'repeat_penalty',
SAMPLERS: 'samplers',
SEND_ON_ENTER: 'sendOnEnter',
+ // the server list is stored under the key it shipped with, so the value is not its name
+ SERVERS: 'backends',
SHOW_AGENTIC_TURN_STATS: 'showAgenticTurnStats',
SHOW_BUILD_VERSION: 'showBuildVersion',
SHOW_FULL_PATH_IN_MENTIONS: 'showFullPathInMentions',
diff --git a/ui/apps/chat/src/lib/constants/settings.constants.ts b/ui/apps/chat/src/lib/constants/settings.constants.ts
index 454d00bde562..fb631642d722 100644
--- a/ui/apps/chat/src/lib/constants/settings.constants.ts
+++ b/ui/apps/chat/src/lib/constants/settings.constants.ts
@@ -188,6 +188,15 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
label: 'Show MCP',
type: SettingsFieldType.CHECKBOX
},
+ {
+ defaultValue: false,
+ help: 'Experimental: pair a llama-server started with --connect by its access code or QR code, and reach it peer-to-peer over WebRTC from any network. Servers added by URL work either way.',
+ isExperimental: true,
+ key: SETTINGS_KEYS.ENABLE_REMOTE_PAIRING,
+ label: 'Enable remote server pairing',
+ note: 'Known security gap: the pass code in the access code is sent to whichever peer answers on the public WebTorrent tracker, so someone watching that tracker could capture it. A mutual challenge-response, where both sides prove they know the code without ever sending it, is the planned fix.',
+ type: SettingsFieldType.CHECKBOX
+ },
{
defaultValue: true,
help: 'Choose how conversation titles are generated. The first non-empty line uses a fast deterministic rule; the LLM option uses a model-generated title from the first message exchange.',
@@ -879,6 +888,7 @@ function toSettingsSection(section: SettingsSectionEntry): SettingsSection {
label: s.label,
max: s.max,
min: s.min,
+ note: s.note,
options: s.options as SettingsFieldConfig['options'],
placeholder: s.placeholder,
radioOptions: s.radioOptions,
diff --git a/ui/apps/chat/src/lib/constants/storage.constants.ts b/ui/apps/chat/src/lib/constants/storage.constants.ts
index fa8d6a09dfb9..31cd6d0b9f04 100644
--- a/ui/apps/chat/src/lib/constants/storage.constants.ts
+++ b/ui/apps/chat/src/lib/constants/storage.constants.ts
@@ -16,6 +16,13 @@ export const DB_APP_NAME_DEPRECATED = 'LlamacppWebui';
export const ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.alwaysAllowedTools`;
+/** Id of the server the selector and new requests target, restored on page load. */
+/* Kept under the name it was first stored under; only the constant was renamed. */
+export const ACTIVE_SERVER_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.activeBackend`;
+
+/** Base URLs of the preset servers already added once, so one the user removed stays removed. */
+export const PRESET_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.presetServers`;
+
export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`;
export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`;
@@ -32,10 +39,16 @@ export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteMod
/** Open state the user set for a model list section or one of its families, by id. */
export const MODEL_GROUP_OPEN_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.modelGroupOpen`;
-/** Recently used model ids, most recent first, backend-qualified. */
+/** Per-model load and inference overrides, keyed by server-qualified model id. */
+export const MODEL_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.modelOverrides`;
+
+/** Model the user picked last, kept across reloads. Stores `{ id, model }`. */
+export const SELECTED_MODEL_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.selectedModel`;
+
+/** Recently used model ids, most recent first, server-qualified. */
export const RECENT_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.recentModels`;
-/** Model ids kept out of the selector, backend-qualified. */
+/** Model ids kept out of the selector, server-qualified. */
export const HIDDEN_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.hiddenModels`;
export const REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.reasoningEffortDefault`;
/** Sidebar conversation sections the user folded. */
@@ -47,11 +60,15 @@ export const SIDEBAR_EXPANDED_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.sidebarExp
export const SIDEBAR_WIDTH_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.sidebarWidth`;
export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`;
export const DISMISSED_RECOMMENDED_MCP_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.dismissedRecommendedMcpServers`;
+/* Likewise: the value is the key already written, the name is the one the list now has. */
+export const DISMISSED_RECOMMENDED_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.dismissedRecommendedBackends`;
/** Key prefix for per-conversation resumable stream state, conversationId is appended */
export const STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX = `${STORAGE_APP_NAME}.streamResume.`;
// Deprecated old key names (kept for backward compat while users migrate)
+/** @deprecated Remote servers are settings-backed now; the demo build kept them here */
+export const LEGACY_TUNNEL_SERVERS_LOCALSTORAGE_KEY = 'llama_servers';
/** @deprecated Use {@link ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY} instead */
export const DEPRECATED_ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME_DEPRECATED}.alwaysAllowedTools`;
/** @deprecated Use {@link CONFIG_LOCALSTORAGE_KEY} instead */
diff --git a/ui/apps/chat/src/lib/constants/stream.constants.ts b/ui/apps/chat/src/lib/constants/stream.constants.ts
index 64f67243c266..b590996c045c 100644
--- a/ui/apps/chat/src/lib/constants/stream.constants.ts
+++ b/ui/apps/chat/src/lib/constants/stream.constants.ts
@@ -2,6 +2,10 @@
// while the tab was hidden. covers brief background pauses without thrashing live streams
export const STREAM_VISIBILITY_KICK_MS = 3000;
+// minimum gap between synthesized live timing updates for servers that do not
+// stream their own, keeps the per-chunk state updates cheap
+export const STREAM_LIVE_TIMINGS_INTERVAL_MS = 500;
+
// separator joining a conversation id and its per-model stream identity
// suffix (conv::model) used by the server side replay buffer
export const CONVERSATION_ID_SEPARATOR = '::';
diff --git a/ui/apps/chat/src/lib/constants/ui.constants.ts b/ui/apps/chat/src/lib/constants/ui.constants.ts
index 90f06c68d8e0..fb9fe638183d 100644
--- a/ui/apps/chat/src/lib/constants/ui.constants.ts
+++ b/ui/apps/chat/src/lib/constants/ui.constants.ts
@@ -5,8 +5,13 @@ import {
SHORTCUT_SEARCH,
SHORTCUT_SETTINGS
} from './keyboard-shortcuts.constants';
-import { MODELS_ROUTE_ID, PROJECTS_ROUTE_ID, SETTINGS_ROUTE_ID } from './routes.constants';
-import { Search, Settings, SquarePen } from '@lucide/svelte';
+import {
+ MODELS_ROUTE_ID,
+ PROJECTS_ROUTE_ID,
+ SERVERS_ROUTE_ID,
+ SETTINGS_ROUTE_ID
+} from './routes.constants';
+import { Search, Server, Settings, SquarePen } from '@lucide/svelte';
import { SidebarAction, ToolSource } from '#lib/enums/index.ts';
import type { DesktopIconStripItem } from '#lib/types/index.ts';
@@ -96,8 +101,8 @@ export const STATS_UNITS = {
export const DEFAULT_MOBILE_BREAKPOINT = 768;
-/** Models listed per remote provider before the "+ X more" line; search covers the rest. */
-export const REMOTE_PROVIDER_MODEL_LIMIT = 12;
+/** Models listed per remote server before the "+ X more" line; search covers the rest. */
+export const REMOTE_SERVER_MODEL_LIMIT = 12;
/** Model rows mounted before the list scrolls; a local catalog can hold hundreds. */
export const MODEL_ROW_WINDOW = 50;
@@ -182,6 +187,12 @@ export const SIDEBAR_FOOTER_ITEMS: DesktopIconStripItem[] = [
keys: SHORTCUT_MODELS,
tooltip: 'Models'
},
+ {
+ action: SidebarAction.SERVERS,
+ activeRouteId: SERVERS_ROUTE_ID,
+ icon: Server,
+ tooltip: 'Servers'
+ },
{
action: SidebarAction.SETTINGS,
activeRouteId: SETTINGS_ROUTE_ID,
diff --git a/ui/apps/chat/src/lib/constants/webrtc.constants.ts b/ui/apps/chat/src/lib/constants/webrtc.constants.ts
new file mode 100644
index 000000000000..c5ccf643af4f
--- /dev/null
+++ b/ui/apps/chat/src/lib/constants/webrtc.constants.ts
@@ -0,0 +1,66 @@
+/**
+ * Remote server transport constants.
+ *
+ * A remote server is addressed by a synthetic origin whose host carries its
+ * id (`https://.remote.llamacpp.invalid`). The suffix is reserved
+ * by RFC 6761, so a request that ever escapes the tunnel transport dies in the
+ * browser instead of reaching the server that happens to serve this UI.
+ */
+
+/** Reserved host suffix of a remote server's synthetic origin. */
+export const TUNNEL_HOST_SUFFIX = 'remote.llamacpp.invalid';
+
+/**
+ * Public WebTorrent trackers used as the signaling rendezvous. The offer is
+ * announced on all of them at once and the first answer wins.
+ */
+export const TRACKER_URLS = [
+ 'wss://tracker.openwebtorrent.com',
+ 'wss://tracker.webtorrent.dev'
+] as const;
+
+/** STUN servers used to discover the public address for NAT traversal. */
+export const STUN_URLS = ['stun:stun.l.google.com:19302', 'stun:stun1.l.google.com:19302'] as const;
+
+/** Deadlines for each step of establishing a tunnel. */
+export const WEBRTC_TIMEOUTS = {
+ /** Deadline for the host to answer and the access code to be accepted */
+ CONNECT_MS: 30_000,
+ /** Deadline for gathering ICE candidates before the offer goes out */
+ ICE_GATHER_MS: 10_000,
+ /**
+ * Wait before reconnecting the channels a backgrounded tab lost. The network
+ * is not always back the moment the tab is.
+ */
+ RECOVER_MS: 2_000,
+ /** Deadline for the WebSocket handshake with a tracker */
+ TRACKER_CONNECT_MS: 10_000
+} as const;
+
+/**
+ * Lengths of the codes shared with the user. The share code is the room code
+ * followed by the pass code.
+ */
+export const CODE_LENGTHS = {
+ /** WebTorrent trackers require an info_hash of exactly this length */
+ INFO_HASH: 20,
+ /** Secret authenticating the client on the data channel */
+ PASS: 32,
+ /** Rendezvous key, padded to INFO_HASH */
+ ROOM: 8,
+ /** Room and pass code concatenated, what the user pastes */
+ SHARE: 40
+} as const;
+
+/** Lengths of the random identifiers exchanged over the wire. */
+export const ID_LENGTHS = {
+ /** Correlates an offer with its answer */
+ OFFER: 20,
+ /** Identifies this peer to the trackers */
+ PEER: 20,
+ /** Correlates a tunneled request with its response */
+ REQUEST: 16
+} as const;
+
+/** Characters that are unambiguous to read aloud or type. */
+export const CODE_CHARS = 'ABCDEFGHJKMNPQRSTUVWXYZabcdefghjkmnpqrstuvwxyz23456789';
diff --git a/ui/apps/chat/src/lib/enums/index.ts b/ui/apps/chat/src/lib/enums/index.ts
index 3a9e67de7bcc..be3d4270a015 100644
--- a/ui/apps/chat/src/lib/enums/index.ts
+++ b/ui/apps/chat/src/lib/enums/index.ts
@@ -16,6 +16,8 @@ export {
ToolCallType
} from './agentic.enums';
+export { ServerKind, ServerCardState, TunnelStatus } from './servers.enums';
+
export {
ChatMessageStatsView,
ChatMessageStatisticsMode,
@@ -84,6 +86,7 @@ export {
ModelGroupKind,
ModelRowDownloadState,
ModelsTableGroupKind,
+ ModelsTableServerKind,
ModelsTableSortKey
} from './model.enums';
diff --git a/ui/apps/chat/src/lib/enums/model.enums.ts b/ui/apps/chat/src/lib/enums/model.enums.ts
index e3a4d7b4f4f8..faceb6e5fe72 100644
--- a/ui/apps/chat/src/lib/enums/model.enums.ts
+++ b/ui/apps/chat/src/lib/enums/model.enums.ts
@@ -40,9 +40,10 @@ export enum ModelDownloadConfirmAction {
DELETE = 'delete'
}
-/** What a table row group folds: the quants of one repo, or its variants. */
+/** What a table row group folds: the servers of one repo, its quants, or its variants. */
export enum ModelGroupKind {
QUANTS = 'quants',
+ SERVERS = 'servers',
VARIANTS = 'variants'
}
@@ -67,3 +68,9 @@ export enum ModelsTableSortKey {
CONTEXT = 'context',
NAME = 'name'
}
+
+/** Kinds a server block carries on top of the manager's own sections. */
+export enum ModelsTableServerKind {
+ COMPAT = 'compat',
+ SERVER = 'server'
+}
diff --git a/ui/apps/chat/src/lib/enums/servers.enums.ts b/ui/apps/chat/src/lib/enums/servers.enums.ts
new file mode 100644
index 000000000000..d77a09cff9d7
--- /dev/null
+++ b/ui/apps/chat/src/lib/enums/servers.enums.ts
@@ -0,0 +1,52 @@
+/**
+ * Server transport enums.
+ *
+ * Every entry is a llama.cpp server the UI can talk to; only the way it is
+ * reached changes: its own base URL, or a WebRTC tunnel addressed by the access
+ * code its `llama-server --connect` prints.
+ */
+
+/** How a server is reached. */
+export enum ServerKind {
+ /** The llama-server serving this UI; it has no base URL of its own. */
+ LOCAL = 'local',
+ /** A remote llama.cpp server, reached through a WebRTC tunnel. */
+ REMOTE = 'remote',
+ /** A llama.cpp server answering on its own base URL. */
+ URL = 'url'
+}
+
+/** Tunnel state of a remote server. */
+export enum TunnelStatus {
+ /** Data channel open and authenticated. */
+ CONNECTED = 'connected',
+ /** Handshake in progress; requests for the server are held back. */
+ CONNECTING = 'connecting',
+ /** Handshake failed or the channel dropped; requests are refused. */
+ ERROR = 'error',
+ /** No tunnel for this server, none attempted. */
+ IDLE = 'idle'
+}
+
+/**
+ * State a server card reports: its tunnel and the server behind it composed.
+ * A remote server has two things that can fail, and the tunnel wins, since a
+ * closed tunnel makes the listing meaningless. A remote server nobody
+ * connected to yet is idle, which is a choice the user made, not a failure.
+ */
+export enum ServerCardState {
+ /** The channel dropped, or the handshake failed. */
+ CHANNEL_DOWN = 'channel_down',
+ /** The server is being asked for its models. */
+ CHECKING = 'checking',
+ /** Handshake in progress. */
+ CONNECTING = 'connecting',
+ /** Remote server with no channel, none attempted. */
+ IDLE = 'idle',
+ /** Nothing to report. */
+ NONE = 'none',
+ /** Listing answered. */
+ ONLINE = 'online',
+ /** Channel up, or no channel to fail, and the server did not answer. */
+ UNREACHABLE = 'unreachable'
+}
diff --git a/ui/apps/chat/src/lib/enums/ui.enums.ts b/ui/apps/chat/src/lib/enums/ui.enums.ts
index 4ef8428df603..02ce4b740725 100644
--- a/ui/apps/chat/src/lib/enums/ui.enums.ts
+++ b/ui/apps/chat/src/lib/enums/ui.enums.ts
@@ -22,6 +22,7 @@ export enum SidebarAction {
NEW_CHAT = 'new-chat',
PROJECTS = 'projects',
SEARCH = 'search',
+ SERVERS = 'servers',
SETTINGS = 'settings'
}
diff --git a/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts b/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts
index 48b330ee6e36..903cb3a38b88 100644
--- a/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts
+++ b/ui/apps/chat/src/lib/extensions/mcp/services/mcp.service.ts
@@ -28,6 +28,7 @@ import {
DEFAULT_IMAGE_MIME_TYPE,
DEFAULT_MCP_CONFIG,
HEADERS,
+ LOCAL_BACKEND_ID,
MCP_RECONNECT,
NEWLINE
} from '#lib/constants/index.ts';
@@ -534,7 +535,8 @@ export class MCPService {
if (useProxy) {
requestInit.headers = {
- ...getAuthHeaders(),
+ // the proxy is the local llama-server, so it takes the local key
+ ...getAuthHeaders(LOCAL_BACKEND_ID),
...(requestInit.headers as Record)
};
}
diff --git a/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts b/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts
new file mode 100644
index 000000000000..a1b8e75a9b9a
--- /dev/null
+++ b/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts
@@ -0,0 +1,46 @@
+/**
+ * Backend availability for connection error states.
+ *
+ * The server error banner and the offline affordances should only appear when
+ * nothing can serve requests. The local server failing is not enough: with an
+ * external server configured, the UI stays usable.
+ */
+
+import {
+ remoteAccessStore,
+ serversModelsStore,
+ serversStore,
+ serverStore
+} from '#lib/stores/index.ts';
+import { isLocalServer, isRemoteServer } from '#lib/utils/servers.ts';
+
+export interface BackendAvailability {
+ readonly hasAvailableBackend: boolean;
+ readonly isOffline: boolean;
+}
+
+export function useBackendAvailability(): BackendAvailability {
+ const hasAvailableBackend = $derived.by(() => {
+ // no connection error at all: nothing to report
+ if (!serverStore.error) return true;
+
+ // the local server failed, but another server may still be usable; a
+ // remote one is only usable while its tunnel is open
+ return serversStore.enabled.some(
+ (server) =>
+ !isLocalServer(server) &&
+ serversModelsStore.get(server.id).error === null &&
+ (!isRemoteServer(server) || remoteAccessStore.isConnected(server.id))
+ );
+ });
+
+ return {
+ get hasAvailableBackend() {
+ return hasAvailableBackend;
+ },
+
+ get isOffline() {
+ return !hasAvailableBackend;
+ }
+ };
+}
diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts
index d9f81b6adef3..410e1c14e09d 100644
--- a/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts
+++ b/ui/apps/chat/src/lib/hooks/use-chat-form-model.svelte.ts
@@ -8,7 +8,7 @@
* fetching model props from the server on demand if they are not cached yet.
*/
-import { conversationsStore, modelsStore, serverStore } from '#lib/stores/index.ts';
+import { conversationsStore, modelsStore, serversStore, serverStore } from '#lib/stores/index.ts';
import type { DatabaseMessage } from '#lib/types/database.js';
import { getConversationModel } from '#lib/utils/index.ts';
@@ -24,10 +24,15 @@ export function useChatFormModel() {
// points the selection at the model that produced it
$effect(() => {
if (conversationModel && conversationModel !== lastSyncedConversationModel) {
- if (modelsStore.models.some((m) => m.model === conversationModel)) {
+ const option = modelsStore.models.find((m) => m.model === conversationModel);
+
+ // only sync models served by the active server; a model from another
+ // server must not yank the active tab (and trigger a full server
+ // switch) just because the conversation used it
+ if (option && option.serverId === serversStore.active.id) {
modelsStore.selectedModelName = conversationModel;
modelsStore.selectModelByName(conversationModel);
- } else {
+ } else if (!option) {
modelsStore.selectedModelName = null;
modelsStore.clearSelection();
}
diff --git a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
index a06b88b8b0b1..724aae5adc89 100644
--- a/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
+++ b/ui/apps/chat/src/lib/hooks/use-chat-form-voice.svelte.ts
@@ -24,11 +24,11 @@ import {
voiceTranscriptsStore
} from '#lib/stores/index.ts';
import type { ChatUploadedFile } from '#lib/types/index.ts';
+import { apiUrl } from '#lib/utils/api-base.ts';
import { getAuthHeaders } from '#lib/utils/api-headers.ts';
import { describeMicrophoneError } from '#lib/utils/audio-recording.ts';
-import { withBase } from '#lib/utils/base-path.ts';
import { DictationSpan } from '#lib/utils/dictation-span.ts';
-import { uuid } from '#lib/utils/index.ts';
+import { getServer, isRemoteServer, uuid } from '#lib/utils/index.ts';
import { resolveVoiceMode, voiceModeLabel } from '#lib/utils/voice-mode.ts';
import {
resolveMicrophone,
@@ -97,18 +97,15 @@ function readAsDataUrl(blob: Blob): Promise {
});
}
-/** The microphone's server: its routes, its auth, and whether a websocket reaches it. */
+/** The microphone's server: the active one, its auth, and whether a websocket reaches it. */
function serverContext(): { context: VoiceServerContext; realtime: boolean } {
const headers = getAuthHeaders();
- // a browser websocket can not send the API key
+ // a browser websocket can send neither the API key nor a tunnel's transport
const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization');
return {
- context: {
- getHeaders: () => getAuthHeaders(),
- resolveUrl: (path) => withBase(path.replace(/^\./, ''))
- },
- realtime: !hasAuth
+ context: { getHeaders: () => getAuthHeaders(), resolveUrl: (path) => apiUrl(path) },
+ realtime: !isRemoteServer(getServer()) && !hasAuth
};
}
@@ -131,7 +128,7 @@ export function useChatFormVoice(options: UseChatFormVoiceOptions) {
if (!isRouter) return null;
return (
- modelsStore.models.find(
+ modelsStore.activeModels.find(
(option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model)
)?.model ?? null
);
@@ -151,7 +148,7 @@ export function useChatFormVoice(options: UseChatFormVoiceOptions) {
const anyAudioModel = $derived.by(() => {
void modelsStore.props.cacheVersion;
- return isRouter && modelsStore.models.some((option) => hasAudio(option.model));
+ return isRouter && modelsStore.activeModels.some((option) => hasAudio(option.model));
});
let hintSeen = $state(readHintSeen());
diff --git a/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts b/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts
index 66bd9058269b..9e6da6e29437 100644
--- a/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts
+++ b/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts
@@ -15,13 +15,16 @@ import {
chatStore,
contextStatsStore,
conversationsStore,
- modelsStore
+ modelsStore,
+ serverStore
} from '#lib/stores/index.ts';
import type { DatabaseMessage } from '#lib/types/index.ts';
import { page } from '$app/state';
export interface UseContextGaugeReturn {
readonly activeModelId: string | null;
+ /** Whether the active server can load the model it is serving. */
+ readonly canLoadActiveModel: boolean;
readonly isActiveModelLoaded: boolean;
readonly isActiveModelLoading: boolean;
readonly contextTotal: number | null;
@@ -100,6 +103,9 @@ export function useContextGauge(): UseContextGaugeReturn {
contextStatsStore.averageTokensPerSecond !== null ||
transientDetails.length > 0
);
+ // loading is a llama.cpp router feature, and the gauge tracks the active model,
+ // so the active server decides whether a load is possible at all
+ const canLoadActiveModel = $derived(serverStore.isRouterMode);
async function loadModel() {
const modelId = contextStatsStore.activeModelId;
@@ -120,6 +126,9 @@ export function useContextGauge(): UseContextGaugeReturn {
get averageTokensPerSecond() {
return contextStatsStore.averageTokensPerSecond;
},
+ get canLoadActiveModel() {
+ return canLoadActiveModel;
+ },
get colorLevel() {
return colorLevel;
},
diff --git a/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts b/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts
index 1f835051e2cd..e85909d9216e 100644
--- a/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts
+++ b/ui/apps/chat/src/lib/hooks/use-models-selector.svelte.ts
@@ -1,21 +1,39 @@
-import { CHAT_INPUT_FOCUS_SELECTOR } from '#lib/constants/index.ts';
-import { deviceStore, modelsStore, serverStore, uiStore } from '#lib/stores/index.ts';
+import {
+ CHAT_INPUT_FOCUS_SELECTOR,
+ LOCAL_BACKEND_ID,
+ REMOTE_SERVER_MODEL_LIMIT
+} from '#lib/constants/index.ts';
+import {
+ deviceStore,
+ modelsStore,
+ serversModelsStore,
+ serversStore,
+ serverStore,
+ uiStore
+} from '#lib/stores/index.ts';
import type { ModelOption } from '#lib/types/models.js';
import {
filterModelOptions,
+ type GroupedModelOptions,
groupFavoriteOptions,
groupModelOptions,
+ groupServerOptions,
type ModelItem
} from '#lib/utils/index.ts';
import { nameOf } from '#lib/utils/model-names.ts';
+import { rawModelId } from '#lib/utils/model-option-id.ts';
import { onMount } from 'svelte';
import { SvelteSet } from 'svelte/reactivity';
+/** Groups of the favorites tab, which lists favorites only. */
+const EMPTY_GROUPS: GroupedModelOptions = { available: [], servers: [] };
+
export interface UseModelsSelectorOptions {
currentModel: () => string | null;
useGlobalSelection?: () => boolean;
+ /** The caller's model outranks the active server's selection, e.g. a message's own. */
onModelChange?: () =>
- | ((modelId: string, modelName: string) => Promise | boolean | void)
+ | ((modelId: string, modelName: string, serverId?: string) => Promise | boolean | void)
| undefined;
onOpenChange?: (open: boolean) => void;
}
@@ -35,10 +53,14 @@ export interface UseModelsSelectorReturn {
readonly loadedItems: ModelItem[];
readonly filteredOptions: ModelOption[];
readonly isEmpty: boolean;
+ readonly isServerView: boolean;
readonly groupedFilteredOptions: ReturnType;
readonly isLoadingModel: boolean;
readonly searchTerm: string;
+ closeServer(): void;
+ openServer(serverId: string): void;
setSearchTerm(value: string): void;
+ showServerModels(serverId: string): Promise;
handleSelect(modelId: string): Promise;
handleOpenChange(open: boolean): void;
isFavorite(model: string): boolean;
@@ -53,20 +75,36 @@ export interface UseModelsSelectorReturn {
* selection handling, and model loading.
*/
export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSelectorReturn {
- let isLoadingModel = $state(false);
- let searchTerm = $state('');
-
- const options = $derived(
+ /**
+ * Current view: the favorites of every server, the local server's models, or
+ * the remote servers'. Favorites are the default while there is at least one.
+ */
+ /** Remote server drilled into from its section; null while browsing. */
+ let serverViewId = $state(null);
+
+ const isServerView = $derived(serverViewId !== null);
+ const isLocalOption = (option: ModelOption) => option.serverId === LOCAL_BACKEND_ID;
+ // every enabled server's models are one list: favorites, then the local
+ // server, then one section per remote server
+ const allOptions = $derived(
modelsStore.models.filter((option) => {
const modelProps = modelsStore.props.getModelProps(option.model);
return modelProps?.ui !== false;
})
);
+ const options = $derived(
+ serverViewId ? allOptions.filter((option) => option.serverId === serverViewId) : allOptions
+ );
const loading = $derived(modelsStore.loading);
const updating = $derived(modelsStore.updating);
const activeId = $derived(modelsStore.selectedModelId);
- // a lone llama.cpp server without a router has nothing to choose from
+ // Router mode and external servers both expose a selectable model list; only
+ // a lone llama.cpp server without a router has nothing to choose from.
+ const isMultiModel = $derived(
+ serverStore.isRouterMode ||
+ serversStore.enabled.some((server) => server.id !== LOCAL_BACKEND_ID)
+ );
const isRouter = $derived(serverStore.isRouterMode);
const serverModel = $derived(modelsStore.singleModelName);
const currentModel = $derived(opts.currentModel());
@@ -74,31 +112,57 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
const isHighlightedCurrentModelActive = $derived.by(() => {
if (!isRouter || !currentModel) return false;
- const currentOption = options.find((option) => option.model === currentModel);
+ const currentOption = allOptions.find((option) => option.model === currentModel);
return currentOption ? currentOption.id === activeId : false;
});
const isCurrentModelInCache = $derived.by(() => {
if (!isRouter || !currentModel) return true;
- return options.some((option) => option.model === currentModel);
+ return allOptions.some((option) => option.model === currentModel);
});
- // the search, the rows and the sections all read the visible set; only the current
- // model resolves against `options`, since it can be hidden and still selected
+
+ let isLoadingModel = $state(false);
+ let searchTerm = $state('');
+
+ // the search, the rows and the sections all read the visible set, while the
+ // favorites and loaded blocks span every server and read the full one
+ const allVisible = $derived(allOptions.filter((option) => !modelsStore.isHidden(option.id)));
const visibleOptions = $derived(options.filter((option) => !modelsStore.isHidden(option.id)));
- // one filter pass feeds the favorites, the loaded rows and the sections alike
+ // the sections read the view's own set; the favorites and the loaded rows span
+ // every server and read the full visible one
const filteredOptions = $derived(filterModelOptions(visibleOptions, searchTerm));
- const loadedItems = $derived(
- filteredOptions
- .filter((option) => modelsStore.isModelLoaded(option.model))
- .map((option) => ({ option }))
+ const filteredAllOptions = $derived(filterModelOptions(allVisible, searchTerm));
+ const remoteServers = $derived(
+ serversStore.enabled
+ .filter((server) => server.id !== LOCAL_BACKEND_ID)
+ .map((server) => {
+ const state = serversModelsStore.get(server.id);
+
+ return {
+ catalog: state.models.length,
+ error: state.error,
+ loading: state.loading,
+ name: server.name,
+ serverId: server.id
+ };
+ })
);
+ // loaded models lead the list, whichever server serves them
+ const isLoadedHere = (option: ModelOption) => modelsStore.isModelLoaded(option.model);
+ const loadedItems = $derived.by(() => {
+ if (isServerView) return [];
+
+ return filteredAllOptions
+ .filter((option) => isLoadedHere(option))
+ .map((option) => ({ option }));
+ });
const loadedIds = $derived(new SvelteSet(loadedItems.map((item) => item.option.id)));
// loaded models lead the list: their own sections list them once, so a
// loaded favorite shows there and not twice
const favoriteItems = $derived(
groupFavoriteOptions(
- filteredOptions.filter((option) => !loadedIds.has(option.id)),
+ filteredAllOptions.filter((option) => !loadedIds.has(option.id)),
modelsStore.favoriteModelIds
)
);
@@ -108,7 +172,26 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
(option) => !modelsStore.favoriteModelIds.has(option.model) && !loadedIds.has(option.id)
)
);
- const groupedFilteredOptions = $derived(groupModelOptions(sectionOptions));
+ const serverSections = $derived(
+ groupServerOptions(
+ sectionOptions,
+ remoteServers,
+ // a drill-in or a search reaches every model, the sections stay short
+ serverViewId || searchTerm ? Infinity : REMOTE_SERVER_MODEL_LIMIT,
+ modelsStore.recentModelIds
+ )
+ );
+ const groupedFilteredOptions = $derived.by(() => {
+ if (isServerView) {
+ const sections = serverSections.filter((section) => section.serverId === serverViewId);
+
+ return { ...EMPTY_GROUPS, servers: sections };
+ }
+
+ const local = groupModelOptions(sectionOptions.filter(isLocalOption));
+
+ return { ...local, servers: serverSections };
+ });
const isEmpty = $derived(
filteredOptions.length === 0 && favoriteItems.length === 0 && loadedItems.length === 0
);
@@ -123,15 +206,18 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
function handleOpenChange(open: boolean) {
if (loading || updating) return;
- // a single-model desktop server has no list: the trigger opens the manager
- // instead. a phone has no manager, so its drawer opens with the one model
- if (!isRouter && !deviceStore.isMobile) {
+ // a lone llama.cpp server without a router has no list: the trigger opens
+ // the manager instead. router mode and external servers both expose a
+ // selectable list, so their role does not decide here, and a phone has no
+ // manager at all: its drawer opens either way
+ if (!isMultiModel && !deviceStore.isMobile) {
if (open) uiStore.openModelsManager();
return;
}
searchTerm = '';
+ serverViewId = null;
if (open && isRouter) {
modelsStore.props.fetchModalitiesForLoadedModels();
@@ -140,15 +226,37 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
opts.onOpenChange?.(open);
}
+ /**
+ * Switch the rendered view. Views are display only: the server that serves
+ * requests follows the selected model, not the view.
+ */
+ /** Drill into one remote server's full model list. */
+ function openServer(serverId: string) {
+ serverViewId = serverId;
+ searchTerm = '';
+ }
+
+ function closeServer() {
+ serverViewId = null;
+ searchTerm = '';
+ }
+
+ /** Refresh a server's models, e.g. right after it was added. */
+ async function showServerModels(serverId: string): Promise {
+ await serversModelsStore.ensureLoaded(serverId);
+ }
+
async function handleSelect(modelId: string) {
- const option = options.find((opt) => opt.id === modelId);
+ // favorites live above the tabs and may belong to another server, so the
+ // lookup spans every enabled server
+ const option = allOptions.find((opt) => opt.id === modelId);
if (!option) return;
let shouldCloseMenu = true;
if (onModelChange) {
- const result = await onModelChange(option.id, option.model);
+ const result = await onModelChange(rawModelId(option.id), option.model, option.serverId);
if (result === false) {
shouldCloseMenu = false;
@@ -169,9 +277,11 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
// only the built-in server loads on request, and only in router mode; a model
// that is already loading is only selected, not loaded a second time
+ const canLoadHere = option.serverId === LOCAL_BACKEND_ID && isRouter;
+
if (
!onModelChange &&
- isRouter &&
+ canLoadHere &&
!modelsStore.isModelLoaded(option.model) &&
!modelsStore.isModelLoading(option.model)
) {
@@ -210,11 +320,11 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
};
}
- return options.find((option) => option.model === currentModel);
+ return allOptions.find((option) => option.model === currentModel);
}
if (activeId) {
- return options.find((option) => option.id === activeId);
+ return allOptions.find((option) => option.id === activeId);
}
return undefined;
@@ -225,6 +335,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
return activeId;
},
+ closeServer,
+
get emptyMessage() {
return emptyMessage;
},
@@ -267,13 +379,17 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
},
get isMultiModel() {
- return isRouter;
+ return isMultiModel;
},
get isRouter() {
return isRouter;
},
+ get isServerView() {
+ return isServerView;
+ },
+
get loadedItems() {
return loadedItems;
},
@@ -282,6 +398,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
return loading;
},
+ openServer,
+
get options() {
return options;
},
@@ -298,6 +416,8 @@ export function useModelsSelector(opts: UseModelsSelectorOptions): UseModelsSele
searchTerm = value;
},
+ showServerModels,
+
get updating() {
return updating;
}
diff --git a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts
index a99eaa39640b..166507492bc3 100644
--- a/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts
+++ b/ui/apps/chat/src/lib/hooks/use-voice-conversation.svelte.ts
@@ -17,9 +17,10 @@ import {
serverStore,
settingsStore
} from '#lib/stores/index.ts';
+import { apiUrl } from '#lib/utils/api-base.ts';
import { getAuthHeaders } from '#lib/utils/api-headers.ts';
import { describeMicrophoneError } from '#lib/utils/audio-recording.ts';
-import { withBase } from '#lib/utils/base-path.ts';
+import { getServer, isRemoteServer } from '#lib/utils/index.ts';
import { resolveVoiceMode } from '#lib/utils/voice-mode.ts';
import {
PauseDetector,
@@ -73,11 +74,8 @@ function serverContext(): { context: VoiceServerContext; realtime: boolean } {
const hasAuth = Object.keys(headers).some((key) => key.toLowerCase() === 'authorization');
return {
- context: {
- getHeaders: () => getAuthHeaders(),
- resolveUrl: (path) => withBase(path.replace(/^\./, ''))
- },
- realtime: !hasAuth
+ context: { getHeaders: () => getAuthHeaders(), resolveUrl: (path) => apiUrl(path) },
+ realtime: !isRemoteServer(getServer()) && !hasAuth
};
}
@@ -98,7 +96,7 @@ export function useVoiceConversation() {
void modelsStore.props.cacheVersion;
const loadedAudioModel = isRouter
- ? (modelsStore.models.find(
+ ? (modelsStore.activeModels.find(
(option) => hasAudio(option.model) && modelsStore.isModelLoaded(option.model)
)?.model ?? null)
: null;
@@ -115,7 +113,7 @@ export function useVoiceConversation() {
});
// the chat model reads its own replies when it can speak, else another model that can
const speechModel = $derived.by(() => {
- const options = modelsStore.models;
+ const options = modelsStore.activeModels;
const chat = options.find((option) => option.model === chatModel);
if (chat?.speaks) return chat.model;
diff --git a/ui/apps/chat/src/lib/services/chat-wire.ts b/ui/apps/chat/src/lib/services/chat-wire.ts
new file mode 100644
index 000000000000..f3e48da72e97
--- /dev/null
+++ b/ui/apps/chat/src/lib/services/chat-wire.ts
@@ -0,0 +1,119 @@
+/**
+ * llama.cpp chat wire format: turns the streamed chunks of a chat completion
+ * into canonical events, and reads back the model a payload reports.
+ *
+ * llama-server answers with `timings` and `prompt_progress` of its own, so the
+ * client keeps no timing fallback and asks for no usage chunk.
+ */
+
+import type {
+ ApiChatCompletionStreamChunk,
+ ApiChatCompletionToolCallDelta,
+ ApiChatCompletionUsage
+} from '#lib/types/api.js';
+import type { ChatMessagePromptProgress, ChatMessageTimings } from '#lib/types/chat.js';
+
+/** One canonical delta decoded from a server's stream payload. */
+export type ChatStreamEvent =
+ | { type: 'done' }
+ | { type: 'error'; message: string }
+ | { type: 'finish_reason'; reason: string }
+ | { type: 'id'; id: string }
+ | { type: 'model'; model: string }
+ | { type: 'prompt_progress'; progress: ChatMessagePromptProgress }
+ | { type: 'text'; text: string }
+ | { type: 'thinking'; text: string }
+ | { type: 'timings'; timings: ChatMessageTimings; promptProgress?: ChatMessagePromptProgress }
+ | { type: 'tool_calls'; deltas: ApiChatCompletionToolCallDelta[] }
+ | { type: 'usage'; usage: ApiChatCompletionUsage };
+
+/** Decodes one stream's payloads. Create one per request. */
+export interface ChatStreamReader {
+ /** Canonical deltas for one parsed SSE payload. */
+ readChunk(payload: unknown): ChatStreamEvent[];
+}
+
+/**
+ * Model name a payload reports. Streaming chunks carry it on the delta, final
+ * responses on the message, and some gateways on metadata or the choice itself.
+ */
+export function extractModelName(data: unknown): string | undefined {
+ const asRecord = (value: unknown): Record | undefined =>
+ typeof value === 'object' && value !== null ? (value as Record) : undefined;
+ const getTrimmedString = (value: unknown): string | undefined =>
+ typeof value === 'string' && value.trim() ? value.trim() : undefined;
+ const root = asRecord(data);
+
+ if (!root) return undefined;
+
+ const rootModel = getTrimmedString(root.model);
+
+ if (rootModel) return rootModel;
+
+ const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined;
+
+ if (!firstChoice) return undefined;
+
+ const metadataModel = getTrimmedString(asRecord(firstChoice.metadata)?.model);
+
+ if (metadataModel) return metadataModel;
+
+ const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model);
+
+ if (deltaModel) return deltaModel;
+
+ const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model);
+
+ if (messageModel) return messageModel;
+
+ return getTrimmedString(firstChoice.model);
+}
+
+function readChunk(payload: unknown): ChatStreamEvent[] {
+ if (!payload || typeof payload !== 'object') return [];
+
+ const chunk = payload as ApiChatCompletionStreamChunk;
+ const events: ChatStreamEvent[] = [];
+ const model = extractModelName(chunk);
+
+ if (chunk.id) events.push({ id: chunk.id, type: 'id' });
+
+ if (model) events.push({ model, type: 'model' });
+
+ if (chunk.prompt_progress) {
+ events.push({ progress: chunk.prompt_progress, type: 'prompt_progress' });
+ }
+
+ if (chunk.timings) {
+ events.push({
+ promptProgress: chunk.prompt_progress,
+ timings: chunk.timings,
+ type: 'timings'
+ });
+ }
+
+ if (chunk.usage) events.push({ type: 'usage', usage: chunk.usage });
+
+ const choice = chunk.choices?.[0];
+ const delta = choice?.delta;
+
+ if (choice?.finish_reason) events.push({ reason: choice.finish_reason, type: 'finish_reason' });
+
+ if (!delta) return events;
+
+ if (delta.content) events.push({ text: delta.content, type: 'text' });
+
+ // endpoints disagree on the reasoning field name; take the first one set
+ const reasoning = delta.reasoning_content ?? delta.reasoning ?? delta.reasoning_text;
+
+ if (reasoning) events.push({ text: reasoning, type: 'thinking' });
+
+ if (delta.tool_calls) events.push({ deltas: delta.tool_calls, type: 'tool_calls' });
+
+ return events;
+}
+
+/** New streaming reader for one request. */
+export function createChatStreamReader(): ChatStreamReader {
+ return { readChunk };
+}
diff --git a/ui/apps/chat/src/lib/services/chat.service.ts b/ui/apps/chat/src/lib/services/chat.service.ts
index 38c7eba144b1..98f3a789cfbb 100644
--- a/ui/apps/chat/src/lib/services/chat.service.ts
+++ b/ui/apps/chat/src/lib/services/chat.service.ts
@@ -31,6 +31,7 @@ import {
ReasoningFormat,
StreamConnectionState
} from '#lib/enums/index.ts';
+import { createChatStreamReader, extractModelName } from '#lib/services/chat-wire.ts';
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
import type {
@@ -45,6 +46,7 @@ import type {
} from '#lib/types/index.ts';
import type { OpenAIToolDefinition } from '#lib/types/mcp.js';
import { isAbortError } from '#lib/utils/abort.ts';
+import { apiChatUrl, apiUrl } from '#lib/utils/api-base.ts';
import { ApiError } from '#lib/utils/api-fetch.ts';
import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts';
import { formatAttachmentText } from '#lib/utils/formatters.ts';
@@ -97,7 +99,7 @@ export class ChatService {
static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise {
try {
const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST;
- const res = await fetch(url, { signal });
+ const res = await fetch(apiUrl(url), { signal });
if (!res.ok) return true;
@@ -518,7 +520,10 @@ export class ChatService {
let idEmitted = false;
let toolCallIndexOffset = 0;
let hasOpenToolCallBatch = false;
+ let firstTokenAt: number | null = null;
+ let usage: ApiChatCompletionUsage | undefined;
+ const streamReader = createChatStreamReader();
const finalizeOpenToolCallBatch = () => {
if (!hasOpenToolCallBatch) {
return;
@@ -558,6 +563,9 @@ export class ChatService {
onToolCallChunk?.(serializedToolCalls);
}
};
+ const markToken = () => {
+ firstTokenAt ??= Date.now();
+ };
const onVisibilityChange = () => {
if (typeof document === 'undefined') return;
@@ -659,59 +667,94 @@ export class ChatService {
continue;
}
+ let parsed: unknown;
+
try {
- const parsed: ApiChatCompletionStreamChunk = JSON.parse(data);
- const choice = parsed.choices?.[0];
+ parsed = JSON.parse(data);
+ } catch (parseError) {
+ console.error('Error parsing JSON chunk:', parseError);
- if (choice?.finish_reason) finishReason = choice.finish_reason;
+ continue;
+ }
- const content = choice?.delta?.content;
- const reasoningContent = choice?.delta?.reasoning_content;
- const toolCalls = choice?.delta?.tool_calls;
- const timings = parsed.timings;
- const promptProgress = parsed.prompt_progress;
- const chunkModel = ChatService.extractModelName(parsed);
+ for (const event of streamReader.readChunk(parsed)) {
+ switch (event.type) {
+ case 'done':
+ streamFinished = true;
- if (chunkModel && !modelEmitted) {
- modelEmitted = true;
- onModel?.(chunkModel);
- }
+ break;
- if (parsed.id && !idEmitted) {
- idEmitted = true;
- onCompletionId?.(parsed.id);
- }
+ case 'error':
+ throw new Error(event.message);
- if (promptProgress) {
- ChatService.notifyTimings(undefined, promptProgress, onTimings);
- }
+ case 'finish_reason':
+ finishReason = event.reason;
- if (timings) {
- ChatService.notifyTimings(timings, promptProgress, onTimings);
- lastTimings = timings;
- }
+ break;
- if (content) {
- finalizeOpenToolCallBatch();
- aggregatedContent += content;
+ case 'id':
+ if (!idEmitted) {
+ idEmitted = true;
+ onCompletionId?.(event.id);
+ }
- if (!abortSignal?.aborted) {
- onChunk?.(content);
- }
- }
+ break;
- if (reasoningContent) {
- finalizeOpenToolCallBatch();
- fullReasoningContent += reasoningContent;
+ case 'model':
+ if (!modelEmitted) {
+ modelEmitted = true;
+ onModel?.(event.model);
+ }
- if (!abortSignal?.aborted) {
- onReasoningChunk?.(reasoningContent);
- }
- }
+ break;
+
+ case 'prompt_progress':
+ ChatService.notifyTimings(undefined, event.progress, onTimings);
+
+ break;
+
+ case 'text':
+ finalizeOpenToolCallBatch();
+ aggregatedContent += event.text;
+
+ if (!abortSignal?.aborted) {
+ onChunk?.(event.text);
+ }
+
+ markToken();
+
+ break;
+
+ case 'thinking':
+ finalizeOpenToolCallBatch();
+ fullReasoningContent += event.text;
+
+ if (!abortSignal?.aborted) {
+ onReasoningChunk?.(event.text);
+ }
+
+ markToken();
+
+ break;
+
+ case 'timings':
+ ChatService.notifyTimings(event.timings, event.promptProgress, onTimings);
+ lastTimings = event.timings;
+
+ break;
+
+ case 'tool_calls':
+ processToolCallDelta(event.deltas);
+
+ break;
+
+ case 'usage':
+ // servers may split usage across chunks (some report input
+ // tokens on message_start and output tokens on message_delta)
+ usage = { ...usage, ...event.usage };
- processToolCallDelta(toolCalls);
- } catch (e) {
- console.error('Error parsing JSON chunk:', e);
+ break;
+ }
}
}
}
@@ -822,7 +865,7 @@ export class ChatService {
* conv::model identity when a model was bound at POST time.
*/
static async lookupStreamSessions(conversationIds: string[]): Promise {
- const resp = await fetch(API_STREAM.LOOKUP, {
+ const resp = await fetch(apiUrl(API_STREAM.LOOKUP), {
body: JSON.stringify({ conversation_ids: conversationIds }),
headers: getJsonHeaders(),
method: 'POST'
@@ -919,7 +962,7 @@ export class ChatService {
}
try {
- const response = await fetch(API_CHAT.COMPLETIONS, {
+ const response = await fetch(apiChatUrl(), {
body: JSON.stringify(requestBody),
headers: getJsonHeaders(),
method: 'POST',
@@ -1297,7 +1340,7 @@ export class ChatService {
// tag streaming requests with the conversation id, this single header is the opt in for the
// server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit
- // model the ::model suffix keeps the per model session distinct
+ // model the ::model suffix keeps the per model session distinct.
if (stream && conversationId) {
headers[HEADERS.X_CONVERSATION_ID_HEADER] = streamIdentity(conversationId, options.model);
// persist the pending stream before the fetch: a reload during the model load or
@@ -1305,7 +1348,7 @@ export class ChatService {
ChatService.saveStreamState(conversationId, 0, options.model ?? null);
}
- const response = await fetch(API_CHAT.COMPLETIONS, {
+ const response = await fetch(apiChatUrl(), {
body: JSON.stringify(requestBody),
headers,
method: 'POST',
@@ -1417,7 +1460,7 @@ export class ChatService {
if (model) body.model = model;
try {
- const res = await fetch(API_CHAT.CONTROL, {
+ const res = await fetch(apiUrl(API_CHAT.CONTROL), {
body: JSON.stringify(body),
headers: getJsonHeaders(),
method: 'POST'
@@ -1448,62 +1491,7 @@ export class ChatService {
const query = `${STREAM_QUERY_PARAMS.CONV_ID}=${encodeURIComponent(streamId)}`;
const offset = from === undefined ? '' : `&${STREAM_QUERY_PARAMS.FROM}=${from}`;
- return `${API_STREAM.BASE}?${query}${offset}`;
- }
-
- /**
- * Extracts model name from Chat Completions API response data.
- * Handles various response formats including streaming chunks and final responses.
- *
- * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name
- * in the response. We override it with the actual model name from serverStore.
- *
- * @param data - Raw response data from the Chat Completions API
- * @returns Model name string if found, undefined otherwise
- * @private
- */
- private static extractModelName(data: unknown): string | undefined {
- const asRecord = (value: unknown): Record | undefined => {
- return typeof value === 'object' && value !== null
- ? (value as Record)
- : undefined;
- };
- const getTrimmedString = (value: unknown): string | undefined => {
- return typeof value === 'string' && value.trim() ? value.trim() : undefined;
- };
- const root = asRecord(data);
-
- if (!root) return undefined;
-
- // 1) root (some implementations provide `model` at the top level)
- const rootModel = getTrimmedString(root.model);
-
- if (rootModel) {
- return rootModel;
- }
-
- // 2) streaming choice (delta) or final response (message)
- const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined;
-
- if (!firstChoice) {
- return undefined;
- }
-
- // priority: delta.model (first chunk) else message.model (final response)
- const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model);
-
- if (deltaModel) {
- return deltaModel;
- }
-
- const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model);
-
- if (messageModel) {
- return messageModel;
- }
-
- // avoid guessing from non-standard locations (metadata, etc.)
- return undefined;
+ return apiUrl(`${API_STREAM.BASE}?${query}${offset}`);
}
/**
@@ -1539,7 +1527,7 @@ export class ChatService {
}
const data: ApiChatCompletionResponse = JSON.parse(responseText);
- const responseModel = ChatService.extractModelName(data);
+ const responseModel = extractModelName(data);
if (responseModel) {
onModel?.(responseModel);
diff --git a/ui/apps/chat/src/lib/services/huggingface.service.ts b/ui/apps/chat/src/lib/services/huggingface.service.ts
index dd5e67820085..6eca16603ee9 100644
--- a/ui/apps/chat/src/lib/services/huggingface.service.ts
+++ b/ui/apps/chat/src/lib/services/huggingface.service.ts
@@ -404,7 +404,7 @@ export class HuggingFaceService {
* exist, so callers should provide a fallback.
*/
static getAvatarUrl(author: string): string {
- // OpenRouter-style model ids prefix the provider with a tilde
+ // OpenRouter-style model ids prefix the server with a tilde
// (`~openai/gpt-...`); the avatars endpoint only resolves bare names
return `${HF_AVATARS_URL}${PATH_SEPARATOR}${author.replace(/^~/, '')}`;
}
@@ -534,7 +534,7 @@ export class HuggingFaceService {
if (!response.ok) throw new Error(`Failed to fetch model details: ${response.status}`);
- // the hub answers with a list of provider entries for one repo; the gguf and
+ // the hub answers with a list of server entries for one repo; the gguf and
// card data sit on the entry that carries them
const payload = (await response.json()) as HfModelDetailInfo | HfModelDetailInfo[];
const entries = Array.isArray(payload) ? payload : [payload];
diff --git a/ui/apps/chat/src/lib/services/index.ts b/ui/apps/chat/src/lib/services/index.ts
index 04a63ad898de..c6d7a25c3433 100644
--- a/ui/apps/chat/src/lib/services/index.ts
+++ b/ui/apps/chat/src/lib/services/index.ts
@@ -19,6 +19,14 @@
*
*/
+/**
+ * **ServersService** - Server connectivity checks
+ *
+ * Probes an external server's models endpoint to validate its URL and
+ * credentials before it is saved. Stateless.
+ */
+export { ServersService } from './servers.service';
+
/**
* **ChatService** - Chat Completions API communication layer
*
diff --git a/ui/apps/chat/src/lib/services/migration.service.ts b/ui/apps/chat/src/lib/services/migration.service.ts
index 68c798db3187..c3c6132c1811 100644
--- a/ui/apps/chat/src/lib/services/migration.service.ts
+++ b/ui/apps/chat/src/lib/services/migration.service.ts
@@ -18,6 +18,7 @@ import {
IDXDB_VERSION,
LEGACY_AGENTIC_REGEX,
LEGACY_REASONING_TAGS,
+ LEGACY_TUNNEL_SERVERS_LOCALSTORAGE_KEY,
NEW_TO_DEPRECATED_MAP,
SEARCH_CHATS_TOOL_KEY,
SETTINGS_KEYS,
@@ -27,6 +28,12 @@ import {
import { BooleanString, MessageRole } from '#lib/enums/index.ts';
import type { McpServerOverride } from '#lib/types/database.js';
import { withSearchChatsDisabled } from '#lib/utils/search-chats.ts';
+import {
+ createRemoteServer,
+ normalizeShareCode,
+ parseServersSettings
+} from '#lib/utils/servers.ts';
+import { uuid } from '#lib/utils/uuid.ts';
import Dexie from 'dexie';
// Types
@@ -867,6 +874,64 @@ const searchChatsDefaultDisabledMigration: Migration = {
}
}
};
+const TUNNEL_SERVERS_LEGACY_KEY = LEGACY_TUNNEL_SERVERS_LOCALSTORAGE_KEY;
+/* A migration runs once per browser, so its id is written in stone: the name moved, the id did not. */
+const REMOTE_SERVERS_MIGRATION_ID = 'remote-providers-v1';
+/**
+ * Fold the `llama_servers` registry the first remote-access build kept in its own
+ * localStorage key into the settings-backed server list, as remote servers.
+ *
+ * The tunnel build stored exactly one shape - `{ type: 'tunnel', code }` - and a
+ * code it cannot validate is worthless, so unusable entries are dropped. Codes
+ * are credentials, so the legacy key is left in place: a user who downgrades
+ * still has what they pasted.
+ */
+const remoteServersMigration: Migration = {
+ description: 'Move tunnel servers onto the server list as remote servers',
+ id: REMOTE_SERVERS_MIGRATION_ID,
+
+ async run(): Promise {
+ const raw = localStorage.getItem(TUNNEL_SERVERS_LEGACY_KEY);
+
+ if (!raw) return;
+
+ let servers: unknown;
+
+ try {
+ servers = JSON.parse(raw);
+ } catch {
+ return;
+ }
+
+ if (!Array.isArray(servers)) return;
+
+ const migrated = servers.flatMap((server) => {
+ const entry = server as { code?: unknown; type?: unknown };
+
+ if (entry?.type !== 'tunnel' || typeof entry.code !== 'string') return [];
+
+ const code = normalizeShareCode(entry.code);
+
+ return code ? [createRemoteServer(code, uuid(), 'Remote server')] : [];
+ });
+
+ if (migrated.length === 0) return;
+
+ const configRaw = localStorage.getItem(CONFIG_LOCALSTORAGE_KEY);
+ const config = configRaw ? JSON.parse(configRaw) : {};
+ const kept = parseServersSettings(config[SETTINGS_KEYS.SERVERS]);
+ const codes = new Set(kept.map((server) => server.code));
+ const added = migrated.filter((server) => !codes.has(server.code));
+
+ if (added.length === 0) return;
+
+ config[SETTINGS_KEYS.SERVERS] = JSON.stringify([...kept, ...added]);
+ localStorage.setItem(CONFIG_LOCALSTORAGE_KEY, JSON.stringify(config));
+
+ if (import.meta.env.DEV && import.meta.env.VITE_DEBUG)
+ console.log(`[Migration] Remote servers: moved ${added.length} tunnel servers`);
+ }
+};
const migrations: Migration[] = [
localStorageMigration,
idxdbMigration,
@@ -879,7 +944,8 @@ const migrations: Migration[] = [
renderKeysMigration,
modelOrgNameKeyMigration,
mcpServerOverridesToToolPolicyMigration,
- searchChatsDefaultDisabledMigration
+ searchChatsDefaultDisabledMigration,
+ remoteServersMigration
];
export const MigrationService = {
diff --git a/ui/apps/chat/src/lib/services/models.service.ts b/ui/apps/chat/src/lib/services/models.service.ts
index 87abe1a94245..9bda05679b7c 100644
--- a/ui/apps/chat/src/lib/services/models.service.ts
+++ b/ui/apps/chat/src/lib/services/models.service.ts
@@ -6,23 +6,49 @@
* modelsStore and its status manager.
*/
-import { API_MODELS, MODEL_ID, type ModelSidecar } from '#lib/constants/index.ts';
+import {
+ API_MODELS,
+ LOCAL_BACKEND_ID,
+ MODEL_ID,
+ type ModelSidecar,
+ SIDECAR_TOKENS
+} from '#lib/constants/index.ts';
import { ServerModelStatus } from '#lib/enums/index.ts';
-import type { ModelSidecarBadge, ParsedModelId } from '#lib/types/models.js';
+import type { ModelSidecarBadge, ModelSidecarFile, ParsedModelId } from '#lib/types/models.js';
import { getAuthHeaders } from '#lib/utils/api-headers.ts';
-import { withBase } from '#lib/utils/base-path.ts';
import {
apiDelete,
apiFetch,
+ apiModelsUrl,
apiPost,
+ apiUrl,
extractSseDataPayload,
isAuxSidecar,
normalizeModelName,
repoOf,
sidecarFromFileToken,
+ sidecarFromTag,
splitSseRecords
} from '#lib/utils/index.ts';
+/** Sidecar token a file name carries, for the forms parsing an id as a model misses. */
+function sidecarTokenInFilename(modelId: string): ModelSidecar | null {
+ // the token can sit after a colon (`org/model:mtp`), a dash or an underscore
+ const name = modelId.toLowerCase();
+ const match = SIDECAR_TOKENS.find((token) =>
+ new RegExp(`(^|[-_:])${token}([-_.:]|$)`).test(name)
+ );
+
+ return (match as ModelSidecar | undefined) ?? null;
+}
+
+/** Parameter label a parsed id reports, e.g. `35B-A3B`. */
+function paramsLabel(parsed: ParsedModelId): string | null {
+ if (!parsed.params) return null;
+
+ return `${parsed.params}${parsed.activatedParams ? `-${parsed.activatedParams}` : ''}`;
+}
+
export class ModelsService {
private static readonly SSE_RECONNECT_MS = 1000;
@@ -122,6 +148,10 @@ export class ModelsService {
return model.status.value === ServerModelStatus.LOADED;
}
+ static isModelLoading(model: ApiModelDataEntry): boolean {
+ return model.status.value === ServerModelStatus.LOADING;
+ }
+
/**
*
*
@@ -130,8 +160,24 @@ export class ModelsService {
*
*/
- static isModelLoading(model: ApiModelDataEntry): boolean {
- return model.status.value === ServerModelStatus.LOADING;
+ /**
+ * True when a router entry id marks a downloaded sidecar file, e.g.
+ * `org/model:Q4_0-mtp` or `org/model:mmproj`, not a loadable model.
+ */
+ static isSidecarEntry(modelId: string): boolean {
+ const idx = modelId.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR);
+
+ if (idx !== MODEL_ID.NOT_FOUND && sidecarFromTag(modelId.slice(idx + 1))) return true;
+
+ // the router also lists projector and draft files by filename, e.g.
+ // `org/model-mmproj-F16.gguf` or `mmproj-model.gguf`
+ const name = modelId.split('/').pop() ?? modelId;
+
+ return (
+ MODEL_ID.SIDECAR_INFIX_REGEX.test(name) ||
+ MODEL_ID.SIDECAR_PREFIX_REGEX.test(name) ||
+ MODEL_ID.SIDECAR_SUFFIX_REGEX.test(name)
+ );
}
/**
@@ -141,7 +187,7 @@ export class ModelsService {
* @returns List of available models with basic metadata
*/
static async list(): Promise {
- return apiFetch(API_MODELS.LIST);
+ return apiFetch(apiModelsUrl());
}
/**
@@ -151,16 +197,21 @@ export class ModelsService {
*
* @param modelId - Model identifier to load
* @param extraArgs - Optional additional arguments to pass to the model instance
+ * @param serverId - Server serving the model; the active one when omitted
* @returns Load response from the server
*/
- static async load(modelId: string, extraArgs?: string[]): Promise {
+ static async load(
+ modelId: string,
+ extraArgs?: string[],
+ serverId?: string
+ ): Promise {
const payload: { model: string; extra_args?: string[] } = { model: modelId };
if (extraArgs && extraArgs.length > 0) {
payload.extra_args = extraArgs;
}
- return apiPost(API_MODELS.LOAD, payload);
+ return apiPost(API_MODELS.LOAD, payload, { serverId });
}
/**
@@ -334,16 +385,57 @@ export class ModelsService {
return result;
}
+ /**
+ * Sidecar files a listing carries as their own entries, keyed by the repo
+ * they belong to. The router lists a downloaded sidecar as a model of its
+ * own, so this keeps the pairing that the model list itself is filtered to
+ * drop.
+ */
+ static sidecarFilesByRepo(response: ApiModelsListResponse): Record {
+ const byRepo: Record = {};
+
+ for (const entry of response.data ?? []) {
+ const parsed = ModelsService.parseModelId(entry.id);
+ const sidecar = parsed.sidecar ?? sidecarTokenInFilename(entry.id);
+
+ if (!sidecar || isAuxSidecar(sidecar)) continue;
+
+ // the repo is the id with its quant tag and sidecar token taken off; naming
+ // parts of the id are not touched, a model name may carry `-4b` for instance
+ const model = entry.id
+ .split(MODEL_ID.QUANTIZATION_SEPARATOR)[0]
+ .replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, '')
+ .replace(new RegExp(`[-_ ]?${sidecar}([-_ ]?draft)?$`, 'i'), '');
+
+ if (!model) continue;
+
+ const files = (byRepo[model] ??= []);
+
+ if (files.some((file) => file.kind === sidecar)) continue;
+
+ files.push({
+ id: entry.id,
+ kind: sidecar,
+ model,
+ params: paramsLabel(parsed),
+ quant: parsed.quantization
+ });
+ }
+
+ return byRepo;
+ }
+
/**
* Unload a model (ROUTER mode only).
* Sends POST request to `/models/unload`. Note: the endpoint returns success
* before unloading completes — use polling to await actual unload status.
*
* @param modelId - Model identifier to unload
+ * @param serverId - Server serving the model; the active one when omitted
* @returns Unload response from the server
*/
- static async unload(modelId: string): Promise {
- return apiPost(API_MODELS.UNLOAD, { model: modelId });
+ static async unload(modelId: string, serverId?: string): Promise {
+ return apiPost(API_MODELS.UNLOAD, { model: modelId }, { serverId });
}
/**
@@ -354,14 +446,18 @@ export class ModelsService {
*/
static async watchModelEvents(
signal: AbortSignal,
- onEvent: (event: ApiModelsSseEvent) => void
+ onEvent: (event: ApiModelsSseEvent) => void,
+ serverId?: string
): Promise {
+ // a llama.cpp server serves its feed at its own origin, so the request is
+ // pinned to the server it is about
+ const id = serverId ?? LOCAL_BACKEND_ID;
const decoder = new TextDecoder();
while (!signal.aborted) {
try {
- const response = await fetch(withBase(API_MODELS.SSE), {
- headers: getAuthHeaders(),
+ const response = await fetch(apiUrl(API_MODELS.SSE, id), {
+ headers: getAuthHeaders(id),
signal
});
diff --git a/ui/apps/chat/src/lib/services/props.service.ts b/ui/apps/chat/src/lib/services/props.service.ts
index 4a2f4e884799..ae2443f5c180 100644
--- a/ui/apps/chat/src/lib/services/props.service.ts
+++ b/ui/apps/chat/src/lib/services/props.service.ts
@@ -15,17 +15,21 @@ export class PropsService {
* In ROUTER mode, returns server-wide settings without model-specific modalities.
*
* @param autoload - If false, prevents automatic model loading (default: false)
+ * @param serverId - Server to ask; defaults to the active one.
* @returns Server properties including default generation settings and capabilities
* @throws {Error} If the request fails or returns invalid data
*/
- static async fetch(autoload = false): Promise {
+ static async fetch(autoload = false, serverId?: string): Promise {
const params: Record = {};
if (!autoload) {
params.autoload = 'false';
}
- return apiFetchWithParams('./props', params, { authOnly: true });
+ return apiFetchWithParams('./props', params, {
+ authOnly: true,
+ serverId
+ });
}
/**
diff --git a/ui/apps/chat/src/lib/services/servers.service.ts b/ui/apps/chat/src/lib/services/servers.service.ts
new file mode 100644
index 000000000000..6bb655f25290
--- /dev/null
+++ b/ui/apps/chat/src/lib/services/servers.service.ts
@@ -0,0 +1,217 @@
+/**
+ * ServersService - Stateless server connectivity checks and model listing
+ *
+ * Probes a server's models endpoint to validate its URL and credentials, and
+ * normalizes the response into the UI model shape. No reactive state;
+ * consumed by the servers settings UI and the per-server model cache.
+ */
+
+import { API_MODELS, LOCAL_BACKEND_ID } from '#lib/constants/index.ts';
+import { ModelsService } from '#lib/services/models.service.ts';
+import type { ApiModelsListResponse, ModelOption, Server } from '#lib/types/index.ts';
+import { isAbortError } from '#lib/utils/abort.ts';
+import { apiUrl } from '#lib/utils/api-base.ts';
+import { getAuthHeadersForServer } from '#lib/utils/api-headers.ts';
+import { isLlamaCppProps } from '#lib/utils/server-props.ts';
+import {
+ isLocalServer,
+ isReachableServer,
+ isRemoteServer,
+ readModelContextLength,
+ serverModelsUrl,
+ serverOrigin
+} from '#lib/utils/servers.ts';
+
+/** What a check reports about an endpoint that is not a llama.cpp server. */
+const NOT_LLAMA_CPP = 'This endpoint is not a llama.cpp server: /props answered something else';
+
+/** Models returned by a server, plus the failure detail when the call fails. */
+export interface ServerModelsResult {
+ error?: string;
+ models: ModelOption[];
+ ok: boolean;
+ status: number | null;
+ /** Untouched list payload of the local server, kept so the router rows and their load statuses can be rebuilt without asking again. */
+ raw?: ApiModelsListResponse;
+}
+
+/** What probing a server's endpoint said about it. */
+export interface ServerProbe {
+ /** The endpoint refused the request for want of a key, so it wants one. */
+ authRequired: boolean;
+ /**
+ * Whether the endpoint answered /props like a llama.cpp server. Null while it
+ * did not answer at all: a key-protected or hidden /props decides nothing.
+ */
+ llamaCpp: boolean | null;
+}
+
+/** Outcome of a server connectivity check. */
+export interface ServerTestResult {
+ error?: string;
+ modelCount?: number;
+ ok: boolean;
+ status: number | null;
+}
+
+export class ServersService {
+ /**
+ * List the models a server exposes on its models endpoint.
+ *
+ * @param server - Server to query. Does not need to be registered yet.
+ * @param signal - Optional abort signal for a cancelled request.
+ */
+ static async listModels(server: Server, signal?: AbortSignal): Promise {
+ if (!isReachableServer(server)) {
+ return {
+ error: isRemoteServer(server) ? 'Access code is required' : 'Server URL is required',
+ models: [],
+ ok: false,
+ status: null
+ };
+ }
+
+ // the local server has no origin of its own; its models endpoint is base
+ // relative, and a remote one is answered over its tunnel
+ const url = isLocalServer(server)
+ ? apiUrl(API_MODELS.LIST, LOCAL_BACKEND_ID)
+ : serverModelsUrl(server);
+
+ try {
+ const response = await fetch(url, {
+ headers: getAuthHeadersForServer(server),
+ signal
+ });
+
+ if (!response.ok) {
+ return {
+ error: await describeFailure(response),
+ models: [],
+ ok: false,
+ status: response.status
+ };
+ }
+
+ const body = (await response.json()) as { data?: unknown };
+ const entries = Array.isArray(body?.data) ? body.data : [];
+ const models = entries.flatMap((entry) => normalizeServerModel(entry));
+ // the local rows carry load status, external ones carry the context size
+ const raw = body as ApiModelsListResponse;
+
+ return { models, ok: true, raw, status: response.status };
+ } catch (error) {
+ if (isAbortError(error)) {
+ return { models: [], ok: false, status: null };
+ }
+
+ return {
+ error: error instanceof Error ? error.message : String(error),
+ models: [],
+ ok: false,
+ status: null
+ };
+ }
+ }
+
+ /**
+ * Ask a server whether it wants an API key, and whether it is a llama.cpp
+ * server at all. A llama-server guards /props with the key it was started
+ * with, so a 401 there means the form needs one; an endpoint that answers
+ * something else there is not one this UI can drive.
+ */
+ static async probe(server: Server): Promise {
+ // a remote server has no URL to ask about: its code is tested by opening
+ // the tunnel, which is what the form's connection test does
+ if (isRemoteServer(server)) return { authRequired: false, llamaCpp: null };
+
+ const base = serverOrigin(server).replace(/\/+$/, '');
+
+ if (!base) return { authRequired: false, llamaCpp: null };
+
+ try {
+ const response = await fetch(`${base}/props`, {
+ headers: getAuthHeadersForServer(server),
+ signal: AbortSignal.timeout(5000)
+ });
+
+ if (response.status === 401) return { authRequired: true, llamaCpp: null };
+
+ if (!response.ok) return { authRequired: false, llamaCpp: null };
+
+ return { authRequired: false, llamaCpp: isLlamaCppProps(await response.json()) };
+ } catch {
+ return { authRequired: false, llamaCpp: null };
+ }
+ }
+
+ /**
+ * Check that a server answers on its models endpoint, and that it is a
+ * llama.cpp server.
+ *
+ * @param server - Server to probe. Does not need to be registered yet.
+ * @param signal - Optional abort signal for a cancelled test.
+ */
+ static async test(server: Server, signal?: AbortSignal): Promise {
+ const result = await ServersService.listModels(server, signal);
+
+ if (result.ok) {
+ const probe = await ServersService.probe(server);
+
+ // only a definite answer counts: an endpoint whose /props cannot be read
+ // is reported by the models listing above, not accused here
+ if (probe.llamaCpp === false) {
+ return { error: NOT_LLAMA_CPP, ok: false, status: result.status };
+ }
+ }
+
+ return {
+ error: result.error,
+ modelCount: result.models.length,
+ ok: result.ok,
+ status: result.status
+ };
+ }
+}
+
+/** Build a human-readable message from a non-OK response. */
+async function describeFailure(response: Response): Promise {
+ const status = `${response.status} ${response.statusText}`.trim();
+
+ try {
+ const body = (await response.json()) as { error?: { message?: string }; message?: string };
+ const message = body?.error?.message ?? body?.message;
+
+ if (message) return `${status}: ${message}`;
+ } catch {
+ // non-JSON error body, fall back to the status line
+ }
+
+ return status;
+}
+
+/**
+ * Normalize one entry of an OpenAI-compatible `/v1/models` response. External
+ * servers only guarantee an id, so that doubles as the display name.
+ */
+function normalizeServerModel(entry: unknown): ModelOption[] {
+ if (!entry || typeof entry !== 'object') return [];
+
+ const raw = entry as Record;
+ const id = typeof raw.id === 'string' ? raw.id.trim() : '';
+
+ if (!id) return [];
+
+ // a llama-compat server lists its projector and draft sidecars as models too
+ if (ModelsService.isSidecarEntry(id)) return [];
+
+ return [
+ {
+ capabilities: [],
+ contextLength: readModelContextLength(raw),
+ id,
+ model: id,
+ name: id,
+ status: raw.status as ApiModelDataEntry['status']
+ }
+ ];
+}
diff --git a/ui/apps/chat/src/lib/services/tools.service.ts b/ui/apps/chat/src/lib/services/tools.service.ts
index dd401f79d34d..5d5a2a0561ec 100644
--- a/ui/apps/chat/src/lib/services/tools.service.ts
+++ b/ui/apps/chat/src/lib/services/tools.service.ts
@@ -5,12 +5,17 @@
* No reactive state; consumed by toolsStore.
*/
-import { API_TOOLS, HEADERS, HOME_TILDE, SEARCH } from '#lib/constants/index.ts';
+import { API_TOOLS, HEADERS, HOME_TILDE, LOCAL_BACKEND_ID, SEARCH } from '#lib/constants/index.ts';
import { BuiltInTool, GlobSearchType, ToolResponseField } from '#lib/enums/index.ts';
import type { GlobEntry, ServerToolInfo, ToolExecutionResult } from '#lib/types/index.ts';
import { getJsonHeaders } from '#lib/utils/api-headers.ts';
-import { withBase } from '#lib/utils/base-path.ts';
-import { apiFetch, buildCaseInsensitiveGlob, joinPath, lastPathSegment } from '#lib/utils/index.ts';
+import {
+ apiFetch,
+ apiUrl,
+ buildCaseInsensitiveGlob,
+ joinPath,
+ lastPathSegment
+} from '#lib/utils/index.ts';
import { parseSseJsonStream, type SseJsonEvent } from '#lib/utils/sse.ts';
export class ToolsService {
@@ -31,6 +36,7 @@ export class ToolsService {
body: JSON.stringify({ params, tool: toolName }),
headers: cwd ? { [HEADERS.X_TOOL_CWD_HEADER]: cwd } : undefined,
method: 'POST',
+ serverId: LOCAL_BACKEND_ID,
signal
});
@@ -70,6 +76,7 @@ export class ToolsService {
body: JSON.stringify({ params, tool: toolName }),
headers: Object.keys(headers).length > 0 ? headers : undefined,
method: 'POST',
+ serverId: LOCAL_BACKEND_ID,
signal
});
}
@@ -80,7 +87,8 @@ export class ToolsService {
* @returns Array of tool definitions in OpenAI-compatible format
*/
static async list(): Promise {
- return apiFetch(API_TOOLS.LIST);
+ // pinned to the local server: server tools do not exist on other servers
+ return apiFetch(API_TOOLS.LIST, { serverId: LOCAL_BACKEND_ID });
}
/**
@@ -133,11 +141,13 @@ export class ToolsService {
signal?: AbortSignal,
cwd?: string
): AsyncGenerator {
- const headers = getJsonHeaders();
+ // pinned to the local server like the tool list and the non-streaming
+ // call: the tools offered to the model are the local server's
+ const headers = getJsonHeaders(LOCAL_BACKEND_ID);
if (cwd) headers[HEADERS.X_TOOL_CWD_HEADER] = cwd;
- const response = await fetch(withBase(API_TOOLS.EXECUTE), {
+ const response = await fetch(apiUrl(API_TOOLS.EXECUTE, LOCAL_BACKEND_ID), {
body: JSON.stringify({ params, stream: true, tool: toolName }),
headers,
method: 'POST',
diff --git a/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts b/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts
index de6fc2f06954..6a5524b9a006 100644
--- a/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/chat/activity.svelte.ts
@@ -4,7 +4,7 @@
* Single owner of the "is this conversation doing something" state:
* - `local` - this browser is piping a stream (send, server-stream attach,
* or resume-wait while the owning model loads)
- * - `remote` - the backend reports a running session, no local pipe yet
+ * - `remote` - the server reports a running session, no local pipe yet
* (global snapshot on mount / visibilitychange)
*
* The union of both drives the sidebar spinners (`loadingConvs`); `local`
@@ -21,7 +21,7 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity';
export class ChatActivityStore {
/** Convs this browser is piping a stream for (send, attach, resume-wait). */
private local = new SvelteSet();
- /** Convs the backend reports as having a running session (snapshot sync). */
+ /** Convs the server reports as having a running session (snapshot sync). */
private remote = new SvelteSet();
/** Convs with any activity, the union the sidebar spinners render. */
loadingConvs = $derived.by(() => {
@@ -40,7 +40,7 @@ export class ChatActivityStore {
private compacting = new SvelteMap();
/**
- * Apply a backend snapshot of running sessions (mount / visibilitychange).
+ * Apply a server snapshot of running sessions (mount / visibilitychange).
* Diffed so unchanged entries do not re-trigger reactivity.
*/
applyRemoteSnapshot(running: Iterable): void {
diff --git a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts
index 73c8d21d64e0..349025242a81 100644
--- a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts
@@ -956,6 +956,15 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
modelOverride?: string | null,
firstUserMessageContent?: string
): Promise {
+ // a conversation keeps the model that generated it, which can belong to
+ // another server; make that server active so the request is not sent to
+ // a server that does not serve the model
+ const requestedModel = modelOverride ?? getConversationModel(allMessages);
+
+ if (requestedModel) {
+ await modelsStore.ensureModelServer(requestedModel);
+ }
+
// the ::model suffix in the stream identity is only for router mode, where it routes to the
// owning child. in single-model mode the identity stays the bare conv id so that attach, stop
// and reattach all agree, regardless of fresh send vs regenerate passing a resolved model
@@ -1442,10 +1451,9 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost {
assistantContent: string,
convId: string
): Promise {
- const effectiveModel =
- serverStore.isRouterMode && modelsStore.selectedModelName
- ? modelsStore.selectedModelName
- : undefined;
+ const effectiveModel = serverStore.isRouterMode
+ ? (modelsStore.activeModelId ?? undefined)
+ : undefined;
const configValue = settingsStore.config;
const titlePromptTemplate =
typeof configValue.titleGenerationPrompt === 'string' &&
diff --git a/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts b/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts
index dfa8e5a5d24c..49b6c7a89ffa 100644
--- a/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/chat/streams.svelte.ts
@@ -171,7 +171,7 @@ export class ChatStreamManager {
}
/**
- * Resync the activity ledger's remote set from the backend. Called by the layout at mount and
+ * Resync the activity ledger's remote set from the server. Called by the layout at mount and
* on visibilitychange, no polling. A snapshot semantic: stale entries for sessions that
* finalized while the browser was elsewhere are dropped naturally.
*/
diff --git a/ui/apps/chat/src/lib/stores/index.ts b/ui/apps/chat/src/lib/stores/index.ts
index 5b011b0fcf2d..7cf7f0a5ce51 100644
--- a/ui/apps/chat/src/lib/stores/index.ts
+++ b/ui/apps/chat/src/lib/stores/index.ts
@@ -38,6 +38,13 @@ export { conversationsStore } from './conversations/index.svelte';
// GROUPS (named sidebar sections for chats)
export { groupsStore } from './groups.svelte';
+// SERVERS
+export { serversStore } from './servers.svelte';
+export { serversModelsStore } from './serversModels.svelte';
+
+// REMOTE ACCESS (WebRTC tunnels to remote servers)
+export { remoteAccessStore } from './remote-access.svelte';
+
// MODELS
export { modelsStore } from './models/index.svelte';
diff --git a/ui/apps/chat/src/lib/stores/init.ts b/ui/apps/chat/src/lib/stores/init.ts
index 532abfeb352a..c98b95f880df 100644
--- a/ui/apps/chat/src/lib/stores/init.ts
+++ b/ui/apps/chat/src/lib/stores/init.ts
@@ -1,8 +1,13 @@
// direct imports, not via the barrel, to avoid circular deps
import { conversationsStore } from './conversations/index.svelte';
import { groupsStore } from './groups.svelte';
+import { serverStore } from './llama-server.svelte';
+import { modelsStore } from './models/index.svelte';
import { permissionsStore } from './permissions.svelte';
import { projectsStore } from './projects.svelte';
+import { remoteAccessStore } from './remote-access.svelte';
+import { serversStore } from './servers.svelte';
+import { serversModelsStore } from './serversModels.svelte';
import { settingsStore } from './settings/index.svelte';
import { tabsStore } from './tabs.svelte';
import { toolsStore } from './tools.svelte';
@@ -11,8 +16,12 @@ import { BUILT_IN_EXTENSIONS } from '#lib/extensions/builtin.ts';
import { extensions } from '#lib/extensions/registry.svelte.ts';
import { MigrationService } from '#lib/services/migration.service.ts';
import { validateApiKey } from '#lib/utils/api-key-validation.ts';
+import { isLocalServer, isRemoteServer } from '#lib/utils/servers.ts';
import { browser } from '$app/env';
+let activeTunnel: Promise | null = null;
+let hydration: Promise | null = null;
+let extensionsStarted: Promise | null = null;
let startup: Promise | null = null;
/** Ask the browser not to evict conversations and settings under storage pressure, best effort. */
@@ -26,22 +35,126 @@ async function requestPersistentStorage(): Promise {
}
}
+/**
+ * Open the tunnel of the server the app opens with, when it is a remote one.
+ *
+ * Deliberately not awaited: the app paints with the server card reading
+ * Connecting, and its models arrive when the handshake lands. Servers that are
+ * merely enabled are left for the moment one of their models is picked.
+ *
+ * Runs once per session: startup asks twice - once before the local server is
+ * known to exist, once after - and the second ask is the same tunnel.
+ */
+function connectActiveServer(): void {
+ const active = serversStore.active;
+
+ if (activeTunnel || !isRemoteServer(active)) return;
+
+ activeTunnel = remoteAccessStore
+ .connect(active.id)
+ // the closed tunnel is exactly why the startup prefetch skipped this
+ // server, so both lists are asked for once it is open
+ .then(async () => {
+ await serversModelsStore.refresh(active.id);
+ await modelsStore.switchServer();
+ })
+ .catch(() => {
+ // the server card carries the failure and offers a retry
+ });
+}
+
+/**
+ * Read the stored state the first request needs: migrations, the settings that
+ * carry the API key, and the server that request goes to.
+ *
+ * Separate from {@link initStores} so a `load` can await it. The requests
+ * `initStores` starts belong after a load, where plain `window.fetch` is the
+ * intended one and SvelteKit raises no warning.
+ */
+export function hydrateStores(): Promise {
+ if (!browser) return Promise.resolve();
+
+ hydration ??= (async () => {
+ await MigrationService.runAllMigrations();
+
+ settingsStore.initialize();
+
+ // the tunnel transport goes in before anything can ask for a model list: a
+ // request aimed at a server's tunnel origin must never reach the server
+ // that happens to serve this UI
+ remoteAccessStore.installInterceptor();
+
+ serversStore.initialize();
+
+ connectActiveServer();
+ })();
+
+ return hydration;
+}
+
+/**
+ * Register the built-in extensions and activate the ones that are switched
+ * on, once. The startup does it; the load of an extension's page does too,
+ * as it runs before the startup and needs the page to be there.
+ */
+export function startExtensions(): Promise {
+ if (!browser) return Promise.resolve();
+
+ extensionsStarted ??= (async () => {
+ // extensions start once the settings they read are in place
+ await hydrateStores();
+
+ for (const manifest of BUILT_IN_EXTENSIONS) extensions.register(manifest);
+
+ await extensions.activateEnabled();
+ })();
+
+ return extensionsStarted;
+}
+
export function initStores(): Promise {
if (!browser) return Promise.resolve();
startup ??= (async () => {
- await MigrationService.runAllMigrations();
+ await hydrateStores();
void requestPersistentStorage();
- settingsStore.initialize();
+ // the servers the page was published with, asked for alongside the rest of
+ // the startup rather than ahead of it, and loaded as they join the list
+ const presets = serversStore
+ .addPresetServers()
+ .then((ids) => Promise.all(ids.map((id) => serversModelsStore.ensureLoaded(id))));
+
+ // prefetch every server's model list in the background; failures are
+ // per-server and never block startup
+ void serversModelsStore.loadAll().then(() => modelsStore.warmHubDetails());
+
permissionsStore.initialize();
toolsStore.initialize();
- // extensions start once the settings they read are in place
- for (const manifest of BUILT_IN_EXTENSIONS) extensions.register(manifest);
+ // the local server state backs the installation facts and decides whether
+ // /tools exists at all, so probe it first and only then list the tools;
+ // otherwise they stay empty until the tools menu is opened
+ void Promise.all([serverStore.prefetchLocalState(), presets]).then(async () => {
+ // a build served without a local server has no local server to fall
+ // back on, so the active server is only settled once that is known
+ connectActiveServer();
- await extensions.activateEnabled();
+ void toolsStore.fetchServerTools();
+
+ // the page asked the local server for its models before it was known to
+ // be missing; the server the app falls back on, such as one the page was
+ // published with, answers instead (a remote one does once its tunnel opens)
+ const active = serversStore.active;
+
+ if (serverStore.localServerMissing && !isLocalServer(active) && !isRemoteServer(active)) {
+ await modelsStore.switchServer();
+ await modelsStore.ensureFirstModelSelected();
+ }
+ });
+
+ await startExtensions();
void versionStore.initialize();
void groupsStore.initialize();
@@ -57,10 +170,11 @@ export function initStores(): Promise {
return startup;
}
-/** Load shared by the app's pages: start the stores, then check the API key. */
+/** Load shared by the app's pages: read the stored state, then check the API key. */
export async function loadAppRoute(fetch: typeof globalThis.fetch): Promise {
// loads run before the root layout script, so the stored API key reaches
- // the probe only once the settings store has read localStorage
- await initStores();
+ // the probe only once the settings store has read localStorage; the rest of
+ // the startup belongs to the layout, after this load
+ await hydrateStores();
await validateApiKey(fetch);
}
diff --git a/ui/apps/chat/src/lib/stores/llama-server.svelte.ts b/ui/apps/chat/src/lib/stores/llama-server.svelte.ts
index 84968528f359..f9820d33b413 100644
--- a/ui/apps/chat/src/lib/stores/llama-server.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/llama-server.svelte.ts
@@ -6,8 +6,10 @@
* PropsService for the /props fetch.
*/
+import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts';
import { ServerRole } from '#lib/enums/index.ts';
import { PropsService } from '#lib/services/props.service.ts';
+import { getServer } from '#lib/utils/api-base.ts';
import { ApiError } from '#lib/utils/index.ts';
const LOADING_RETRY_INTERVAL_MS = 1000;
@@ -18,7 +20,16 @@ class ServerStore {
props = $state(null);
role = $state(null);
status = $state(null);
+ private fetchBackendId: string | undefined;
private fetchPromise: Promise | null = null;
+ /**
+ * Whether a local llama.cpp server answered. Null until the first probe
+ * resolves: the bundled build is served by one, the hosted PWA is not.
+ */
+ private localAvailable: boolean | null = null;
+ /** Local server state kept alive while an external server is active. */
+ private localState: { props: ApiLlamaCppServerProps | null; role: ServerRole | null } | null =
+ null;
private retryTimer: ReturnType | null = null;
get contextSize(): number | null {
@@ -28,7 +39,12 @@ class ServerStore {
}
get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null {
- return this.props?.default_generation_settings?.params || null;
+ return this.localProps?.default_generation_settings?.params || null;
+ }
+
+ /** A local llama.cpp server answered, either now or earlier this session. */
+ get hasLocalServer(): boolean {
+ return this.localAvailable === true;
}
get isModelMode(): boolean {
@@ -39,10 +55,52 @@ class ServerStore {
return this.role === ServerRole.ROUTER;
}
+ /**
+ * Local server state. The local server is the base of the installation, so
+ * its props and role come from the mirror while another server is active
+ * instead of following the selected model.
+ */
+ private get localFacts(): { props: ApiLlamaCppServerProps | null; role: ServerRole | null } {
+ // no resolvable server (early startup, non-browser call): props belong to
+ // the origin serving this UI, which is the local server
+ const serverId = getServer()?.id ?? LOCAL_BACKEND_ID;
+
+ if (serverId === LOCAL_BACKEND_ID) return { props: this.props, role: this.role };
+
+ return this.localState ?? { props: null, role: null };
+ }
+
+ /** The local server runs as a router, independent of the active server. */
+ get localIsRouter(): boolean {
+ return this.localFacts.role === ServerRole.ROUTER;
+ }
+
+ /** Props of the local server, from the active server or the cache. */
+ get localProps(): ApiLlamaCppServerProps | null {
+ return this.localFacts.props;
+ }
+
+ /** The local server is known to be unreachable: local-only UI stays out. */
+ get localServerMissing(): boolean {
+ return this.localAvailable === false;
+ }
+
get uiSettings(): Record | undefined {
return this.props?.ui_settings ?? this.props?.webui_settings;
}
+ /**
+ * Keep the local server state before switching to an external server, so
+ * switching back restores it instead of asking the server again.
+ */
+ cacheLocalState(): void {
+ // props only exist while a llama.cpp server is active; an external to
+ // external switch must not overwrite the kept local state with blanks
+ if (!this.props) return;
+
+ this.localState = { props: this.props, role: this.role };
+ }
+
clear(): void {
this.clearRetryTimer();
this.props = null;
@@ -51,6 +109,7 @@ class ServerStore {
this.loading = false;
this.role = null;
this.fetchPromise = null;
+ this.fetchBackendId = undefined;
}
/**
@@ -59,7 +118,12 @@ class ServerStore {
* splash and the chat screen every retry tick.
*/
async fetch({ background = false }: { background?: boolean } = {}): Promise {
- if (this.fetchPromise) return this.fetchPromise;
+ // props and role describe one server. a fetch started for another server
+ // must not be reused, and its response must not commit once the active
+ // server has changed while it was in flight
+ const serverId = getServer()?.id;
+
+ if (this.fetchPromise && this.fetchBackendId === serverId) return this.fetchPromise;
this.clearRetryTimer();
@@ -73,19 +137,33 @@ class ServerStore {
this.error = null;
}
- const fetchPromise = (async () => {
+ const promise = (async () => {
try {
const props = await PropsService.fetch();
+ // the active server changed while this request was in flight
+ if (getServer()?.id !== serverId) return;
+
this.props = props;
this.error = null;
this.status = null;
this.detectRole(props);
+
+ if (serverId === LOCAL_BACKEND_ID) {
+ this.localAvailable = true;
+ // mirror the local facts as they arrive: a later clear() must not
+ // be able to wipe what the local server told us
+ this.localState = { props, role: this.role };
+ }
} catch (error: unknown) {
+ if (getServer()?.id !== serverId) return;
+
this.error = error instanceof Error ? error.message : String(error);
this.status = error instanceof ApiError ? error.status : null;
console.error('Error fetching server properties:', error);
+ if (serverId === LOCAL_BACKEND_ID) this.localAvailable = false;
+
if (this.status === 503) {
this.scheduleRetry();
}
@@ -93,13 +171,56 @@ class ServerStore {
if (!background) {
this.loading = false;
}
-
- this.fetchPromise = null;
}
})();
- this.fetchPromise = fetchPromise;
- await fetchPromise;
+ this.fetchPromise = promise;
+ this.fetchBackendId = serverId;
+
+ // a server switch clears the in-flight handle; only the fetch that is
+ // still the current one may release it
+ void promise
+ .catch(() => {})
+ .finally(() => {
+ if (this.fetchPromise === promise) {
+ this.fetchPromise = null;
+ this.fetchBackendId = undefined;
+ }
+ });
+
+ await promise;
+ }
+
+ /**
+ * Load the local server state in the background at startup. The local tab
+ * then opens from memory instead of asking for props on the first click.
+ */
+ async prefetchLocalState(): Promise {
+ if (this.localState) return;
+
+ try {
+ const props = await PropsService.fetch(false, LOCAL_BACKEND_ID);
+
+ this.localState = {
+ props,
+ role: props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL
+ };
+ this.localAvailable = true;
+ } catch {
+ // no local server in this deployment: mark it missing so the local
+ // server drops out of the enabled list
+ this.localAvailable = false;
+ }
+ }
+
+ /** Restore the state kept by {@link cacheLocalState}; no request is made. */
+ restoreLocalState(): void {
+ if (!this.localState) return;
+
+ this.props = this.localState.props;
+ this.role = this.localState.role;
+ this.error = null;
+ this.status = null;
}
private clearRetryTimer(): void {
diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts
index 7928defaad30..8d765ba61ef7 100644
--- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts
@@ -10,24 +10,30 @@
import {
FAVORITE_MODELS_LOCALSTORAGE_KEY,
HIDDEN_MODELS_LOCALSTORAGE_KEY,
+ LOCAL_BACKEND_ID,
MODEL_GROUP_OPEN_LOCALSTORAGE_KEY,
MODEL_REMOVE_BLOCKED_FALLBACK,
MODEL_REMOVE_BLOCKED_REASONS,
RECENT_MODEL_LIMIT,
- RECENT_MODELS_LOCALSTORAGE_KEY
+ RECENT_MODELS_LOCALSTORAGE_KEY,
+ SELECTED_MODEL_LOCALSTORAGE_KEY
} from '#lib/constants/index.ts';
import { ReasoningEffort, ServerModelSource, ServerModelStatus } from '#lib/enums/index.ts';
import { HuggingFaceService } from '#lib/services/huggingface.service.ts';
import { ModelsService } from '#lib/services/models.service.ts';
-// direct imports between stores, not via the barrel, to avoid circular deps
import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts';
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
import { type ModelPropsHost, ModelPropsManager } from '#lib/stores/models/props.svelte.ts';
import { type ModelStatusHost, ModelStatusManager } from '#lib/stores/models/status.svelte.ts';
+// direct imports between stores, not via the barrel, to avoid circular deps
+import { serversStore } from '#lib/stores/servers.svelte.ts';
+import { serversModelsStore } from '#lib/stores/serversModels.svelte.ts';
import type { ApiModelDataEntry } from '#lib/types/index.ts';
import type { ParsedModelId } from '#lib/types/models.js';
import { getConversationModel } from '#lib/utils/conversation-utils.ts';
import { repoOf } from '#lib/utils/model-names.ts';
+import { qualifyModelId, rawModelId, serverIdFromModelId } from '#lib/utils/model-option-id.ts';
+import { readModelContextLength } from '#lib/utils/servers.ts';
import { browser } from '$app/env';
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
import { toast } from 'svelte-sonner';
@@ -41,14 +47,39 @@ const MAX_GROUP_OPEN_ENTRIES = 200;
*/
const HUB_WARM_LIMIT = 12;
+/** Selection kept from the last session, so a reload does not drop the picked model. */
+function loadStoredSelection(): { id: string; model: string | null } | null {
+ if (!browser) return null;
+
+ try {
+ const raw = localStorage.getItem(SELECTED_MODEL_LOCALSTORAGE_KEY);
+
+ if (!raw) return null;
+
+ const parsed = JSON.parse(raw) as { id?: unknown; model?: unknown };
+
+ if (typeof parsed?.id !== 'string' || !parsed.id) return null;
+
+ return { id: parsed.id, model: typeof parsed.model === 'string' ? parsed.model : null };
+ } catch {
+ return null;
+ }
+}
+
+const storedSelection = loadStoredSelection();
+
/** Models kept out of the selector. */
function loadHiddenModels(): Set {
if (!browser) return new SvelteSet();
try {
const raw = localStorage.getItem(HIDDEN_MODELS_LOCALSTORAGE_KEY);
+ const ids = raw ? (JSON.parse(raw) as string[]) : [];
- return raw ? new SvelteSet(JSON.parse(raw) as string[]) : new SvelteSet();
+ // ids saved before models were qualified by server all belonged to the local one
+ return new SvelteSet(
+ ids.map((id) => (serverIdFromModelId(id) ? id : qualifyModelId(LOCAL_BACKEND_ID, id)))
+ );
} catch {
return new SvelteSet();
}
@@ -72,7 +103,7 @@ function loadGroupOpenState(): SvelteMap {
}
}
-/** Recently used backend-qualified ids, most recent first. */
+/** Recently used server-qualified ids, most recent first. */
function loadRecentModels(): string[] {
if (!browser) return [];
@@ -92,16 +123,59 @@ function loadRecentModels(): string[] {
}
class ModelsStore implements ModelPropsHost, ModelStatusHost {
+ activeModels = $state([]);
error = $state(null);
favoriteModelIds = $state>(this.loadFavoritesFromStorage());
groupOpenState = $state>(loadGroupOpenState());
hiddenModelIds = $state>(loadHiddenModels());
loading = $state(false);
- models = $state([]);
+ /**
+ * Every selectable model across enabled servers. The active server's
+ * models come from {@link activeModels}; the rest come from the background
+ * prefetch cache. Ids are server-qualified so the same model name on two
+ * servers stays distinct.
+ */
+ /**
+ * Computed once per state change. Rows reach this through per-model props
+ * lookups, so a getter that rebuilt the list on every read made opening the
+ * selector quadratic in the size of the catalog.
+ */
+ models = $derived.by((): ModelOption[] => {
+ const activeServerId = serversStore.active.id;
+ const merged: ModelOption[] = [];
+ const seen = new SvelteSet();
+ const push = (option: ModelOption, serverId: string) => {
+ const id = qualifyModelId(serverId, rawModelId(option.id));
+
+ // a server can be listed twice while a switch is in flight: the rows
+ // of the previous server are still in activeModels
+ if (seen.has(id)) return;
+
+ seen.add(id);
+ merged.push({ ...option, id, serverId });
+ };
+
+ for (const option of this.activeModels) {
+ // keep the server an option was built for: rows from the previous
+ // server must not be relabelled while a switch is in flight
+ push(option, option.serverId ?? activeServerId);
+ }
+
+ for (const server of serversStore.enabled) {
+ if (server.id === activeServerId) continue;
+
+ for (const option of serversModelsStore.get(server.id).models) {
+ push(option, option.serverId ?? server.id);
+ }
+ }
+
+ return merged;
+ });
recentModelIds = $state(loadRecentModels());
routerModels = $state([]);
- selectedModelId = $state(null);
- selectedModelName = $state(null);
+ selectedModelId = $state(storedSelection?.id ?? null);
+
+ selectedModelName = $state(storedSelection?.model ?? null);
updating = $state(false);
@@ -115,6 +189,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
// Without this, ?model= URL handler races an in-progress fetch and sees an empty list.
private inflightFetch: Promise | null = null;
+ /** A restored selection loses to the active conversation's model, a fresh pick does not. */
+ private selectionFromStorage = storedSelection !== null;
+
/**
* Model the active conversation view resolves to. Router mode: the user's
* selection first, then the conversation's own model. Otherwise the single
@@ -122,14 +199,20 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
*/
get activeModelId(): string | null {
if (!serverStore.isRouterMode) {
+ // external servers expose a selectable list; prefer the user's pick
+ const selected = this.selectedModelId
+ ? this.models.find((m) => m.id === this.selectedModelId)
+ : undefined;
+
+ if (selected) return selected.model;
+
return this.models.length > 0 ? this.models[0].model : this.singleModelName;
}
- if (this.selectedModelId) {
- const selected = this.models.find((m) => m.id === this.selectedModelId);
+ const picked = this.selectedModelId && !this.selectionFromStorage ? this.selectedModelId : null;
+ const selected = picked ? this.models.find((m) => m.id === picked) : undefined;
- if (selected) return selected.model;
- }
+ if (selected) return selected.model;
const conversationModel = getConversationModel(conversationsStore.activeMessages);
@@ -139,7 +222,11 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
if (model) return model.model;
}
- return null;
+ const restored = this.selectedModelId
+ ? this.models.find((m) => m.id === this.selectedModelId)
+ : undefined;
+
+ return restored?.model ?? null;
}
get loadedModelIds(): string[] {
@@ -192,6 +279,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
clearSelection(): void {
this.selectedModelId = null;
this.selectedModelName = null;
+ this.persistSelection();
}
/** Family keys folded away under one section, for a list that restores them. */
@@ -265,7 +353,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
return;
}
- // favorites are shared across backends, so a stored id may belong to another one
+ // favorites are shared across servers, so a stored id may belong to another one
const favorite = this.favoriteModelIds.values().next()?.value;
const favoriteOption = favorite
? availableModels.find((m) => m.id === favorite || m.model === favorite)
@@ -281,6 +369,26 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
await this.selectModelById(availableModels[0].id);
}
+ /**
+ * Make the server serving `modelName` active when it is not already.
+ * A conversation keeps the model that generated it, which can belong to a
+ * server other than the active one.
+ */
+ async ensureModelServer(modelName: string): Promise {
+ const options = this.models.filter((model) => model.model === modelName && model.serverId);
+
+ // the same model name can be served by several servers: stay on the active one
+ // when it has it, rather than switching to whichever server is listed first
+ if (
+ options.length === 0 ||
+ options.some((model) => model.serverId === serversStore.active.id)
+ ) {
+ return;
+ }
+
+ await this.selectModelById(options[0].id);
+ }
+
/**
* Fetch list of models from server and detect server role.
* Also fetches modalities for MODEL mode (single model).
@@ -288,7 +396,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
async fetch(force = false): Promise {
if (this.inflightFetch) return this.inflightFetch;
- if (this.models.length > 0 && !force) return;
+ if (this.activeModels.length > 0 && !force) return;
this.inflightFetch = this.runFetch();
try {
@@ -315,7 +423,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
this._status.syncPausedDownloads();
// keep the selector options in sync: a downloaded / deleted model shows
// up here too, not only in the router model rows
- this.models = this.buildModelOptions(response);
+ this.activeModels = this.buildModelOptions(response);
this.warmHubDetails();
await this.props.fetchModalitiesForLoadedModels();
@@ -375,10 +483,26 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
return null;
}
+ /**
+ * Load state a model's own server reports. Only the local server has a status
+ * feed, so an external server answers from its own model listing.
+ */
getModelStatus(modelId: string): ServerModelStatus | null {
+ // the local feed reports the models its router serves: a same-named entry
+ // on a remote server must not shadow it
const model = this.routerModels.find((m) => m.id === modelId);
- return (model?.status?.value as ServerModelStatus) ?? null;
+ if (model) return (model.status?.value as ServerModelStatus) ?? null;
+
+ // a model the local router does not serve belongs to a server, and the
+ // first enabled server listing it owns the state
+ for (const server of serversStore.enabled) {
+ const option = serversModelsStore.get(server.id).models.find((m) => m.model === modelId);
+
+ if (option) return (option.status?.value as ServerModelStatus) ?? null;
+ }
+
+ return null;
}
/**
@@ -386,8 +510,13 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
* hidden by the user, so an automatic pick never lands on one it doesn't show.
*/
getVisibleModels(): ModelOption[] {
- return this.models.filter(
- (option) => this.props.getModelProps(option.model)?.ui !== false && !this.isHidden(option.id)
+ const activeServerId = serversStore.active.id;
+
+ // the active server's rows carry raw ids, hidden ones are saved server-qualified
+ return this.activeModels.filter(
+ (option) =>
+ this.props.getModelProps(option.model)?.ui !== false &&
+ !this.isHidden(qualifyModelId(option.serverId ?? activeServerId, rawModelId(option.id)))
);
}
@@ -450,13 +579,27 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
async selectModelById(modelId: string, options?: { recordRecent?: boolean }): Promise {
if (!modelId || this.updating) return;
- if (this.selectedModelId === modelId) {
- if (options?.recordRecent) this.recordRecentModel(modelId);
+ const serverId = serverIdFromModelId(modelId) ?? serversStore.active.id;
+ const rawId = rawModelId(modelId);
+ // the selection is stored server-qualified, matching the aggregated
+ // model list, no matter which form the caller passed
+ const qualifiedId = qualifyModelId(serverId, rawId);
+
+ // a model from another server makes that server active first; a remote
+ // one is only reachable once its tunnel is open, so the switch asks for it
+ if (serverId !== serversStore.active.id) {
+ serversStore.setActive(serverId);
+ await serversModelsStore.connectAndLoad(serverId);
+ await this.switchServer();
+ }
+
+ if (this.selectedModelId === qualifiedId) {
+ if (options?.recordRecent) this.recordRecentModel(qualifiedId);
return;
}
- const option = this.models.find((model) => model.id === modelId);
+ const option = this.activeModels.find((model) => model.id === rawId);
if (!option) throw new Error('Selected model is not available');
@@ -464,10 +607,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
this.error = null;
try {
- this.selectedModelId = option.id;
+ this.selectedModelId = qualifiedId;
this.selectedModelName = option.model;
+ this.selectionFromStorage = false;
+ this.persistSelection();
- if (options?.recordRecent) this.recordRecentModel(modelId);
+ if (options?.recordRecent) this.recordRecentModel(qualifiedId);
// reasoning levels belong to the model that answered: another model in the chat starts at Default
const conversationModel = getConversationModel(conversationsStore.activeMessages);
@@ -582,6 +727,53 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
}
}
+ /**
+ * Activate a server for the selector tabs. Everything comes from memory:
+ * the model list and router rows are prefetched at startup and the local
+ * server state is kept while an external server is active. The selection
+ * is left alone, switching tabs must not pick a model.
+ */
+ async switchServer(): Promise {
+ this.error = null;
+
+ const server = serversStore.active;
+
+ // local props describe the server the UI is served from; keep them while
+ // an external server is active instead of dropping and refetching
+ if (server.id === LOCAL_BACKEND_ID) {
+ serverStore.restoreLocalState();
+ } else {
+ serverStore.cacheLocalState();
+ serverStore.clear();
+ }
+
+ const cached = serversModelsStore.get(server.id);
+
+ if (!cached.loaded) {
+ // nothing prefetched for this server (startup prefetch failed): load it once
+ await this.fetch(true);
+
+ return;
+ }
+
+ if (server.id === LOCAL_BACKEND_ID && !serverStore.props) {
+ // first visit to the local tab in this session
+ await serverStore.fetch({ background: true });
+ }
+
+ this.activeModels = cached.models;
+ this.loading = false;
+
+ // the local router rows carry the load statuses; the startup prefetch
+ // already returned them, so a tab switch rebuilds the list from memory
+ if (server.id === LOCAL_BACKEND_ID && this.routerModels.length === 0 && cached.raw) {
+ this.routerModels = cached.raw.data;
+ this.activeModels = this.buildModelOptions(cached.raw);
+ }
+
+ this.warmHubDetails();
+ }
+
toDisplayName(id: string): string {
const segments = id.split(/\\|\//);
const candidate = segments.pop();
@@ -621,10 +813,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
const repos: string[] = [];
- for (const option of this.models) {
- const repo = repoOf(option.model);
+ for (const server of serversStore.enabled) {
+ for (const option of serversModelsStore.get(server.id).models) {
+ const repo = repoOf(option.model);
- if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo);
+ if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo);
+ }
}
// the Hub rate limits aggressively and an unauthenticated burst of requests
@@ -674,13 +868,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
capabilities: rawCapabilities.filter((value: unknown): value is string =>
Boolean(value)
),
- // 0 is the server's way of leaving the trained context unknown; a
- // model-mode listing reports it as meta.n_ctx_train instead
- contextLength:
- item.context_length ||
- (typeof item.meta?.n_ctx_train === 'number' && item.meta.n_ctx_train > 0
- ? item.meta.n_ctx_train
- : undefined),
+ // external servers report the context in their listing, so the
+ // gauge keeps working when the list is rebuilt on reload
+ contextLength: readModelContextLength(item),
description: details?.description,
details: details?.details,
draftSidecars: this._status.getDraftSidecars(modelId),
@@ -690,6 +880,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
model: modelId,
name: this.toDisplayName(displayNameSource),
parsedId: ModelsService.parseModelId(modelId),
+ // stamp the server here so the option keeps its origin even
+ // after another server becomes active
+ serverId: serversStore.active.id,
speaks: item.architecture?.output_modalities?.includes('audio') ?? false,
tags: item.tags ?? []
};
@@ -715,6 +908,23 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
}
}
+ private persistSelection(): void {
+ if (!browser) return;
+
+ try {
+ if (!this.selectedModelId) {
+ localStorage.removeItem(SELECTED_MODEL_LOCALSTORAGE_KEY);
+ } else {
+ localStorage.setItem(
+ SELECTED_MODEL_LOCALSTORAGE_KEY,
+ JSON.stringify({ id: this.selectedModelId, model: this.selectedModelName })
+ );
+ }
+ } catch {
+ console.warn('[ModelsStore] Failed to persist the model selection');
+ }
+ }
+
/** Move a model to the front of the recently used list. */
private recordRecentModel(qualifiedId: string): void {
this.recentModelIds = [
@@ -746,7 +956,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
const response = await ModelsService.list();
this.routerModels = response.data;
- this.models = this.buildModelOptions(response);
+ this.activeModels = this.buildModelOptions(response);
await this.props.fetchModalitiesForLoadedModels();
@@ -756,10 +966,10 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost {
this.selectModelById(visible[0].id);
}
} else {
- this.models = await this.fetchModelModeInternal();
+ this.activeModels = await this.fetchModelModeInternal();
}
} catch (error) {
- this.models = [];
+ this.activeModels = [];
this.error = error instanceof Error ? error.message : 'Failed to load models';
throw error;
diff --git a/ui/apps/chat/src/lib/stores/models/props.svelte.ts b/ui/apps/chat/src/lib/stores/models/props.svelte.ts
index d7c89c438624..61f8c6a8256d 100644
--- a/ui/apps/chat/src/lib/stores/models/props.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/models/props.svelte.ts
@@ -18,15 +18,20 @@ import {
} from '#lib/constants/index.ts';
import { FileTypeCategory, ModelModality, ReasoningEffort } from '#lib/enums/index.ts';
import { PropsService } from '#lib/services/props.service.ts';
-// direct imports between stores, not via the barrel, to avoid circular deps
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
+// direct imports between stores, not via the barrel, to avoid circular deps
+import { serversStore } from '#lib/stores/servers.svelte.ts';
import type { ReasoningEffortLevel } from '#lib/types/index.ts';
// deep imports, not the '#lib/utils/index.ts' barrel: it re-exports modules that reach back
// into the stores, and going through it here would read a half-built module
import { TTLCache } from '#lib/utils/cache-ttl.ts';
import { detectThinkingSupport } from '#lib/utils/chat-template-thinking-detector.ts';
+import { rawModelId } from '#lib/utils/model-option-id.ts';
import { SvelteMap } from 'svelte/reactivity';
+/** Grace period before /props is retried for a model after a failed fetch. */
+const PROPS_FETCH_RETRY_COOLDOWN_MS = 30_000;
+
/**
* The slice of modelsStore the manager reads. Kept narrow on purpose so it
* cannot reach around the host's full surface; modelsStore implements this
@@ -34,7 +39,9 @@ import { SvelteMap } from 'svelte/reactivity';
*/
export interface ModelPropsHost {
/** Model rows the manager mirrors fetched modalities onto. */
- models: ModelOption[];
+ activeModels: ModelOption[];
+ /** Every enabled server's models, used to resolve a model's owner. */
+ readonly models: ModelOption[];
readonly selectedModelName: string | null;
readonly loadedModelIds: string[];
isModelLoaded(modelId: string): boolean;
@@ -51,6 +58,8 @@ export class ModelPropsManager {
maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES,
ttlMs: MODEL_PROPS_CACHE.TTL_MS
});
+ /** Last failed /props fetch per model; requests stay on cooldown after it. */
+ private failedPropsAt = new SvelteMap();
/** Props requests in flight, a second caller awaits the same request. */
private fetching = new SvelteMap>();
@@ -130,7 +139,7 @@ export class ModelPropsManager {
try {
const results = await Promise.all(propsPromises);
- this.host.models = this.host.models.map((model) => {
+ this.host.activeModels = this.host.activeModels.map((model) => {
const modelIndex = loadedModelIds.indexOf(model.model);
if (modelIndex === -1) return model;
@@ -159,14 +168,33 @@ export class ModelPropsManager {
* @returns Props data or null if fetch failed or model not loaded
*/
async fetchModelProps(modelId: string): Promise {
+ // props not loaded yet (mid server switch): the server role and model
+ // load state are not trustworthy, so do not guess
+ if (!serverStore.props) return null;
+
const cached = this.cache.get(modelId);
if (cached) return cached;
+ // never ask a server about a model another server serves; a name that
+ // exists on two servers resolves to the active one's entry first
+ const option = this.host.models.find(
+ (m) => m.model === modelId || m.id === modelId || rawModelId(m.id) === modelId
+ );
+
+ if (option?.serverId && option.serverId !== serversStore.active.id) return null;
+
if (serverStore.isRouterMode && !this.host.isModelLoaded(modelId)) {
return null;
}
+ // a failing model would otherwise be refetched on every reactive update
+ const failedAt = this.failedPropsAt.get(modelId);
+
+ if (failedAt !== undefined && Date.now() - failedAt < PROPS_FETCH_RETRY_COOLDOWN_MS) {
+ return null;
+ }
+
const inFlight = this.fetching.get(modelId);
if (inFlight) return inFlight;
@@ -175,10 +203,12 @@ export class ModelPropsManager {
.then((props) => {
this.cache.set(modelId, props);
this.cacheVersion++;
+ this.failedPropsAt.delete(modelId);
return props;
})
.catch((error) => {
+ this.failedPropsAt.set(modelId, Date.now());
console.warn(`Failed to fetch props for model ${modelId}:`, error);
return null;
@@ -194,7 +224,12 @@ export class ModelPropsManager {
const props = this.getModelProps(modelId);
const nCtx = props?.default_generation_settings?.n_ctx;
- return typeof nCtx === 'number' ? nCtx : null;
+ if (typeof nCtx === 'number') return nCtx;
+
+ // external servers report the context in their model listing, when they report one
+ const model = this.host.activeModels.find((m) => m.model === modelId || m.id === modelId);
+
+ return model?.contextLength ?? null;
}
getModelModalities(modelId: string): ModelModalities | null {
@@ -202,7 +237,7 @@ export class ModelPropsManager {
return this.buildModalities(serverStore.props.modalities);
}
- const model = this.host.models.find((m) => m.model === modelId || m.id === modelId);
+ const model = this.host.activeModels.find((m) => m.model === modelId || m.id === modelId);
if (model?.modalities) {
return model.modalities;
@@ -304,7 +339,7 @@ export class ModelPropsManager {
if (!props?.modalities) return;
- this.host.models = this.host.models.map((model) =>
+ this.host.activeModels = this.host.activeModels.map((model) =>
model.model === modelId
? { ...model, modalities: this.buildModalities(props.modalities!) }
: model
diff --git a/ui/apps/chat/src/lib/stores/models/status.svelte.ts b/ui/apps/chat/src/lib/stores/models/status.svelte.ts
index ade2b701a881..46ceaa6d3921 100644
--- a/ui/apps/chat/src/lib/stores/models/status.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/models/status.svelte.ts
@@ -8,6 +8,7 @@
import {
CLI_FLAGS,
HF_UD_QUANT_PREFIX_REGEX,
+ LOCAL_BACKEND_ID,
MODEL_ID,
type ModelSidecar,
STORAGE_APP_NAME
@@ -20,9 +21,11 @@ import {
} from '#lib/enums/index.ts';
import { HuggingFaceService } from '#lib/services/huggingface.service.ts';
import { ModelsService } from '#lib/services/models.service.ts';
-// direct imports between stores, not via the barrel, to avoid circular deps
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
import type { ModelPropsManager } from '#lib/stores/models/props.svelte.ts';
+import { serversStore } from '#lib/stores/servers.svelte.ts';
+// direct imports between stores, not via the barrel, to avoid circular deps
+import { serversModelsStore } from '#lib/stores/serversModels.svelte.ts';
// explicit type imports: the app.d.ts globals resolve to `any`, so import the real types
import type {
ApiModelsSseDownloadFailedData,
@@ -32,6 +35,7 @@ import type {
} from '#lib/types/index.ts';
import type { ModelSidecarBadge } from '#lib/types/models.js';
import { repoOf } from '#lib/utils/model-names.ts';
+import { serverIdFromModelId } from '#lib/utils/model-option-id.ts';
import { isAuxSidecar, isDraftSidecar, sidecarFromFileToken } from '#lib/utils/sidecars.ts';
import { browser } from '$app/env';
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
@@ -42,13 +46,20 @@ import { toast } from 'svelte-sonner';
* cannot reach around the host's full surface; modelsStore implements this
* structurally.
*/
+/** How long a remote operation is watched before its last answer is kept. */
+const REMOTE_STATUS_TIMEOUT_MS = 10 * 60 * 1000;
+const REMOTE_STATUS_POLL_MS = 2000;
+
export interface ModelStatusHost {
error: string | null;
+ /** Load state a model's own server reports, local or external. */
+ getModelStatus(modelId: string): ServerModelStatus | null;
readonly props: ModelPropsManager;
/** Router model rows the status feed updates. */
routerModels: ApiModelDataEntry[];
fetchRouterModels(): Promise;
isModelLoaded(modelId: string): boolean;
+ switchServer(): Promise;
toDisplayName(id: string): string;
}
@@ -196,6 +207,9 @@ export class ModelStatusManager {
return result;
});
+ /** Feed reader of the active server when it is not the built-in one. */
+ private serverAbort: AbortController | null = null;
+ private serverReaderId: string | null = null;
// /models/sse feed state, the single source of truth for status and load progress
private statusAbort: AbortController | null = null;
private statusReaderActive = false;
@@ -213,6 +227,8 @@ export class ModelStatusManager {
* removes the model along with the partial files of its download.
*/
async cancelDownload(repoWithTag: string): Promise {
+ await this.ensureLocalTarget();
+
if (!serverStore.isRouterMode) {
toast.error('Model downloads are only available in router mode');
@@ -265,12 +281,16 @@ export class ModelStatusManager {
* settled status, so no waiter is registered here.
*/
async cancelLoad(modelId: string): Promise {
- if (!serverStore.isRouterMode) return;
+ const serverId = this.serverIdFor(modelId);
+
+ // the local server takes unload requests in router mode only; a remote
+ // server cancels on its own
+ if (serverId === LOCAL_BACKEND_ID && !serverStore.localIsRouter) return;
this.subscribe();
try {
- await ModelsService.unload(modelId);
+ await ModelsService.unload(modelId, serverId);
toast.info(`Load cancelled: ${this.host.toDisplayName(modelId)}`);
} catch (error) {
toast.error(`Failed to cancel load: ${this.host.toDisplayName(modelId)}`);
@@ -306,6 +326,28 @@ export class ModelStatusManager {
return;
}
+ // a second click must not post again: the server would answer that the model
+ // already exists, so a download already on its way is left alone
+ if (this.pendingDownloads.has(model) || this.isDownloadInProgress(model)) return;
+
+ // the marker and its last reported progress survive until the request is
+ // accepted: dropping them first would drop the paused state, which reads as
+ // a lost download if the server rejects the resume
+ const snapshot = this.pausedDownloads.get(model) ?? null;
+ const wasPaused = this.pausedDownloads.has(model);
+
+ // the download reads as in flight from here, so the chip shows it before the
+ // server answers; the feed refines it with the real counts. Both marks land
+ // ahead of the first await, so a second click in the same tick still coalesces
+ this.pendingDownloads.add(model);
+ this.downloadProgress.set(model, snapshot ?? emptyDownloadProgress());
+
+ // the local server owns the download endpoint, so the target moves only once
+ // the row already reads as downloading
+ await this.ensureLocalTarget();
+
+ // read once the target has moved: the listing describes the server taking the
+ // download, not the server the row was clicked from
const registered = this.host.routerModels.find(
(m) => downloadIdKey(m.id) === downloadIdKey(model)
);
@@ -317,25 +359,9 @@ export class ModelStatusManager {
...extraSidecars
].filter((kind, i, all) => SERVER_SIDECARS.includes(kind) && all.indexOf(kind) === i);
- // a second click must not post again: the server would answer that the model
- // already exists, so a download already on its way is left alone
- if (this.pendingDownloads.has(model) || this.isDownloadInProgress(model)) return;
-
- this.pendingDownloads.add(model);
-
// the feed must be live so the resulting models_reload event refreshes the list
this.subscribe();
- // the marker and its last reported progress survive until the request is
- // accepted: dropping them first would drop the paused state, which reads as
- // a lost download if the server rejects the resume
- const snapshot = this.pausedDownloads.get(model) ?? null;
- const wasPaused = this.pausedDownloads.has(model);
-
- // the download reads as in flight from here, so the chip shows it before the
- // server answers; the feed refines it with the real counts
- this.downloadProgress.set(model, snapshot ?? emptyDownloadProgress());
-
try {
const res = await ModelsService.downloadModel(model, sidecars);
@@ -495,7 +521,10 @@ export class ModelStatusManager {
return this.loadingStates.get(modelId) ?? false;
}
- async load(modelId: string): Promise {
+ async load(modelId: string, extraArgs?: string[]): Promise {
+ const serverId = this.serverIdFor(modelId);
+ const isLocal = serverId === LOCAL_BACKEND_ID;
+
if (this.host.isModelLoaded(modelId)) return;
if (this.loadingStates.get(modelId)) return;
@@ -506,13 +535,21 @@ export class ModelStatusManager {
// the feed drives completion, so it must be live before the request
this.subscribe();
- const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED);
+ const reachedLoaded = isLocal
+ ? this.waitForStatus(modelId, ServerModelStatus.LOADED)
+ : Promise.resolve();
reachedLoaded.catch(() => {});
try {
- await ModelsService.load(modelId);
- await reachedLoaded;
+ await ModelsService.load(modelId, extraArgs, serverId);
+
+ if (isLocal) {
+ await reachedLoaded;
+ } else {
+ await this.waitForRemoteStatus(serverId, modelId, ServerModelStatus.LOADED);
+ }
+
toast.success(`Model loaded: ${this.host.toDisplayName(modelId)}`);
} catch (error) {
this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed'));
@@ -532,6 +569,8 @@ export class ModelStatusManager {
* before the request: an unconfirmed pause would leave the chip spinning.
*/
async pauseDownload(repoWithTag: string): Promise {
+ await this.ensureLocalTarget();
+
if (!serverStore.isRouterMode) {
toast.error('Model downloads are only available in router mode');
@@ -571,13 +610,37 @@ export class ModelStatusManager {
subscribe(): void {
if (this.statusReaderActive) return;
- if (!serverStore.isRouterMode) return;
+ // the feed reports the local server's models, so the local server's role
+ // decides, not the active server's
+ if (!serverStore.localIsRouter) return;
this.statusReaderActive = true;
this.statusAbort = new AbortController();
void this.runStatusReader(this.statusAbort.signal);
}
+ /**
+ * Read the status feed of the active server, when it is not the built-in one.
+ *
+ * A remote router reports its load progress over its own feed, which the
+ * tunnel carries like any other request; the operations that wait on a load
+ * keep their polling as the fallback. Pass null to close it.
+ */
+ subscribeServer(serverId: string | null): void {
+ if (this.serverReaderId === serverId) return;
+
+ this.serverAbort?.abort();
+ this.serverAbort = null;
+ this.serverReaderId = serverId;
+
+ if (!serverId) return;
+
+ const abort = new AbortController();
+
+ this.serverAbort = abort;
+ void this.runStatusReader(abort.signal, serverId);
+ }
+
/**
* A model the server lists as incomplete is a download that never finished: it
* runs in no tab and carries no progress. Offer it as paused with the sidecars
@@ -609,6 +672,9 @@ export class ModelStatusManager {
}
async unload(modelId: string): Promise {
+ const serverId = this.serverIdFor(modelId);
+ const isLocal = serverId === LOCAL_BACKEND_ID;
+
if (!this.host.isModelLoaded(modelId)) return;
if (this.loadingStates.get(modelId)) return;
@@ -618,13 +684,21 @@ export class ModelStatusManager {
this.subscribe();
- const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED);
+ const reachedUnloaded = isLocal
+ ? this.waitForStatus(modelId, ServerModelStatus.UNLOADED)
+ : Promise.resolve();
reachedUnloaded.catch(() => {});
try {
- await ModelsService.unload(modelId);
- await reachedUnloaded;
+ await ModelsService.unload(modelId, serverId);
+
+ if (isLocal) {
+ await reachedUnloaded;
+ } else {
+ await this.waitForRemoteStatus(serverId, modelId, ServerModelStatus.UNLOADED);
+ }
+
toast.info(`Model unloaded: ${this.host.toDisplayName(modelId)}`);
} catch (error) {
this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed'));
@@ -641,6 +715,7 @@ export class ModelStatusManager {
this.statusReaderActive = false;
this.statusAbort?.abort();
this.statusAbort = null;
+ this.subscribeServer(null);
this.loadProgress.clear();
this.downloadProgress.clear();
this.failedDownloads.clear();
@@ -812,6 +887,13 @@ export class ModelStatusManager {
return this.pausedDownloads.delete(repoWithTag);
}
+ private async ensureLocalTarget(): Promise {
+ if (serversStore.active.id === LOCAL_BACKEND_ID) return;
+
+ serversStore.setActive(LOCAL_BACKEND_ID);
+ await serversModelsStore.ensureLoaded(LOCAL_BACKEND_ID);
+ await this.host.switchServer();
+ }
/** True when the option is the model itself, or a sidecar its download pulls. */
private pullsSidecar(repoWithTag: string): boolean {
const { model, sidecar } = downloadTarget(repoWithTag);
@@ -849,8 +931,27 @@ export class ModelStatusManager {
void this.host.fetchRouterModels();
}
- private async runStatusReader(signal: AbortSignal): Promise {
- await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event));
+ private async runStatusReader(signal: AbortSignal, serverId?: string): Promise {
+ await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event), serverId);
+ }
+
+ /**
+ * Server a model belongs to: the one its qualified id names, else whichever
+ * enabled server lists it. Loading an external llama-server's model must not
+ * become a request to the server this UI is served from.
+ */
+ private serverIdFor(modelId: string): string {
+ const qualified = serverIdFromModelId(modelId);
+
+ if (qualified) return qualified;
+
+ for (const server of serversStore.enabled) {
+ const state = serversModelsStore.get(server.id);
+
+ if (state.models.some((option) => option.model === modelId)) return server.id;
+ }
+
+ return serversStore.active.id;
}
private setPausedDownload(repoWithTag: string, progress: ModelDownloadProgress | null): void {
@@ -883,6 +984,42 @@ export class ModelStatusManager {
for (const waiter of waiters) if (waiter.target === status) waiter.resolve();
}
+ /**
+ * Wait for an external server to report a status. Its changes never reach the
+ * local feed, so its own listing is asked again until the status lands or the
+ * deadline passes, which leaves the last answer in place.
+ */
+ private async waitForRemoteStatus(
+ serverId: string,
+ modelId: string,
+ target: ServerModelStatus
+ ): Promise {
+ const deadline = Date.now() + REMOTE_STATUS_TIMEOUT_MS;
+
+ while (Date.now() < deadline) {
+ await serversModelsStore.refresh(serverId);
+
+ const status = this.host.getModelStatus(modelId);
+
+ if (status === ServerModelStatus.FAILED) {
+ throw new Error(
+ target === ServerModelStatus.LOADED
+ ? 'the server failed to load it'
+ : 'the server failed to unload it'
+ );
+ }
+
+ const reached =
+ target === ServerModelStatus.LOADED
+ ? status === ServerModelStatus.LOADED || status === ServerModelStatus.SLEEPING
+ : status === ServerModelStatus.UNLOADED || status === null;
+
+ if (reached) return;
+
+ await new Promise((resolve) => setTimeout(resolve, REMOTE_STATUS_POLL_MS));
+ }
+ }
+
// a load and the requests that wait on it can each hold a waiter for the same model
private waitForStatus(modelId: string, target: ServerModelStatus): Promise {
return new Promise((resolve, reject) => {
diff --git a/ui/apps/chat/src/lib/stores/remote-access.svelte.ts b/ui/apps/chat/src/lib/stores/remote-access.svelte.ts
new file mode 100644
index 000000000000..16977c1f34e0
--- /dev/null
+++ b/ui/apps/chat/src/lib/stores/remote-access.svelte.ts
@@ -0,0 +1,353 @@
+/**
+ * remoteAccessStore - tunnels to remote llama.cpp servers.
+ *
+ * A remote server is a `llama-server --connect` host reached over a WebRTC
+ * data channel. Its access code is stored on the server itself, in the
+ * server list the settings carry, so this store owns no persistence: it owns
+ * the live tunnels and the transport that routes a request to the right one.
+ *
+ * Routing works on the address, not on the origin. Every request addressed at
+ * a server's synthetic tunnel origin (`tunnelOrigin`) is answered by that
+ * server's data channel, and every other request is left to the browser. A
+ * request addressed at a tunnel origin that has no live tunnel is refused here
+ * and never reaches the server serving this UI: the origin cannot resolve, so a
+ * missed call site fails loudly instead of talking to the wrong server.
+ *
+ * Tunnels are opened lazily. Only the server the app opens with is connected
+ * during startup; the rest connect when they are first needed - a model of
+ * theirs is picked, or the user hits Connect on their card.
+ *
+ * The whole transport is experimental and sits behind a setting that is off by
+ * default: while it is off no tunnel opens, and the ones already open close.
+ */
+
+import { CODE_LENGTHS, SETTINGS_KEYS, WEBRTC_TIMEOUTS } from '#lib/constants/index.ts';
+import { TunnelStatus } from '#lib/enums/index.ts';
+// direct import between stores, not via the barrel, to avoid circular deps
+import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
+import type { Server } from '#lib/types/index.ts';
+import { getServers } from '#lib/utils/api-base.ts';
+import { codeOf, isRemoteServer, parseTunnelOrigin, tunnelHostLabel } from '#lib/utils/servers.ts';
+import { ClientTunnel } from '#lib/utils/webrtc-tunnel.ts';
+import { browser } from '$app/env';
+
+/** Servers reached over a tunnel, in list order. */
+function remoteServers(): Server[] {
+ return getServers().filter(isRemoteServer);
+}
+
+class RemoteAccessStore {
+ /** Last tunnel failure by server id. */
+ errorMessage = $state>({});
+ /** Tunnel state by server id; servers with no state never attempted a tunnel. */
+ status = $state>({});
+
+ /** In-flight handshakes by server id, so a second caller joins the first. */
+ private attempts = new Map>();
+ /**
+ * Handshake count by server id, bumped when an attempt is dropped, so an
+ * answer that lands after the user gave up is not stored.
+ */
+ private generation = new Map();
+ private originalFetch: typeof window.fetch | null = null;
+ /** Pending visibility recovery, so one burst of tab switches opens one handshake. */
+ private recoverTimer: ReturnType | null = null;
+ /** Live tunnels by server id. */
+ private tunnels = new Map();
+ /** Requests held back while their server's handshake runs. */
+ private waiters = new Map void; resolve: () => void }>>();
+
+ /** Whether the experimental remote pairing is turned on; every entry point hides without it. */
+ get isEnabled(): boolean {
+ return Boolean(settingsStore.getConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING));
+ }
+
+ /**
+ * Open the tunnel of a remote server, or join the handshake already running.
+ *
+ * Resolves once the data channel is open and the pass code was accepted.
+ */
+ async connect(serverId: string): Promise {
+ if (this.isConnected(serverId)) return;
+
+ const running = this.attempts.get(serverId);
+
+ if (running) return running;
+
+ const generation = this.generationOf(serverId);
+ const attempt = this.open(serverId, generation).finally(() => {
+ if (this.attempts.get(serverId) === attempt) this.attempts.delete(serverId);
+ });
+
+ this.attempts.set(serverId, attempt);
+
+ return attempt;
+ }
+
+ /** Close a server's tunnel and forget its state; the server itself stays. */
+ disconnect(serverId: string): void {
+ // a handshake still running must land on nothing: the channel is not wanted
+ this.dropAttempt(serverId);
+ this.tunnels.get(serverId)?.disconnect();
+ this.tunnels.delete(serverId);
+ this.errorMessage[serverId] = '';
+ this.rejectWaiters(serverId, new Error('Remote access closed'));
+
+ // a server that was never connected has no state to clear
+ if (serverId in this.status) delete this.status[serverId];
+ }
+
+ /** Last tunnel failure of a server. */
+ errorOf(serverId: string): string {
+ return this.errorMessage[serverId] ?? '';
+ }
+
+ /**
+ * Replace `window.fetch` with the tunnel transport.
+ *
+ * Called from `hydrateStores`, before any request goes out. A server is
+ * recognised by its host label rather than by the page origin, so several
+ * tunnels coexist and requests to real URLs are untouched.
+ */
+ installInterceptor(): void {
+ if (!browser || this.originalFetch) return;
+
+ this.originalFetch = window.fetch.bind(window);
+ const original = this.originalFetch;
+
+ window.fetch = (input: RequestInfo | URL, init?: RequestInit) => {
+ try {
+ const href =
+ input instanceof Request ? input.url : input instanceof URL ? input.href : String(input);
+ const label = parseTunnelOrigin(href);
+
+ if (label !== null) return this.request(label, input, init);
+ } catch {
+ // unparseable input: it is not addressed at a tunnel
+ }
+
+ return original(input, init);
+ };
+ }
+
+ isConnected(serverId: string): boolean {
+ return this.statusOf(serverId) === TunnelStatus.CONNECTED;
+ }
+
+ isConnecting(serverId: string): boolean {
+ return this.statusOf(serverId) === TunnelStatus.CONNECTING;
+ }
+
+ /**
+ * Reconnect the channels a backgrounded tab lost.
+ *
+ * iOS Safari and Android PWAs tear a peer connection down when the tab goes
+ * to the background, which leaves the card on a failure the user can only
+ * clear with the retry icon. Only a channel that was already wanted is
+ * recovered: an idle server is one nobody asked to connect. The recovery
+ * stays silent, so a failure just lands back on the card.
+ */
+ recoverChannels(): void {
+ if (!browser) return;
+
+ // a tab switch is not the network being back: let the connection settle,
+ // and let a burst of switches coalesce into one attempt per server
+ if (this.recoverTimer) clearTimeout(this.recoverTimer);
+
+ this.recoverTimer = setTimeout(() => {
+ this.recoverTimer = null;
+
+ if (navigator.onLine === false) return;
+
+ for (const server of remoteServers()) {
+ if (!server.enabled || this.statusOf(server.id) !== TunnelStatus.ERROR) continue;
+
+ void this.connect(server.id).catch(() => {
+ // the card keeps the failure and offers the retry control
+ });
+ }
+ }, WEBRTC_TIMEOUTS.RECOVER_MS);
+ }
+
+ /**
+ * Answer a request addressed at a tunnel origin.
+ *
+ * @param label Host label of the origin the request was addressed at.
+ */
+ async request(label: string, input: RequestInfo | URL, init?: RequestInit): Promise {
+ const server = this.findByHostLabel(label);
+
+ // A tunnel origin that no server owns is never served: the request was
+ // built for a server that is gone, or aimed at the suffix on purpose.
+ if (!server) {
+ throw new Error(`No remote server is registered for the tunnel host "${label}"`);
+ }
+
+ const tunnel = this.tunnels.get(server.id);
+
+ if (tunnel?.isConnected) return tunnel.fetch(input, init);
+
+ if (this.status[server.id] === TunnelStatus.CONNECTING) {
+ await new Promise((resolve, reject) => {
+ const queued = this.waiters.get(server.id) ?? [];
+
+ queued.push({ reject, resolve });
+ this.waiters.set(server.id, queued);
+ });
+
+ // the handshake resolved; if the channel died before this ran, the
+ // server is simply not connected and says so
+ const opened = this.tunnels.get(server.id);
+
+ if (!opened?.isConnected) {
+ throw new Error(`"${server.name}" disconnected while the request was waiting`);
+ }
+
+ return opened.fetch(input, init);
+ }
+
+ throw new Error(
+ `"${server.name}" is not connected. Reconnect it in the servers dialog, ` +
+ 'or pick a model of another server.'
+ );
+ }
+
+ /** Tunnel state of a server. */
+ statusOf(serverId: string): TunnelStatus {
+ return this.status[serverId] ?? TunnelStatus.IDLE;
+ }
+
+ /**
+ * Drop the tunnels of servers that left the list or were disabled, so a
+ * disabled server stops holding a data channel open. Turning remote pairing
+ * off disables them all.
+ */
+ syncServers(): void {
+ const kept = new Set(
+ this.isEnabled
+ ? remoteServers()
+ .filter((server) => server.enabled)
+ .map((b) => b.id)
+ : []
+ );
+
+ for (const serverId of [...this.tunnels.keys()]) {
+ if (!kept.has(serverId)) this.disconnect(serverId);
+ }
+
+ for (const serverId of Object.keys(this.status)) {
+ if (!kept.has(serverId)) {
+ delete this.status[serverId];
+ delete this.errorMessage[serverId];
+ }
+ }
+ }
+
+ uninstallInterceptor(): void {
+ if (!this.originalFetch) return;
+
+ window.fetch = this.originalFetch;
+ this.originalFetch = null;
+ }
+
+ /**
+ * Give up on a server's handshake: a later answer must land on nothing, and a
+ * caller asking again must start over instead of joining the dropped attempt.
+ */
+ private dropAttempt(serverId: string): void {
+ this.generation.set(serverId, this.generationOf(serverId) + 1);
+ this.attempts.delete(serverId);
+ }
+
+ /** Server whose tunnel origin carries this host label. */
+ private findByHostLabel(label: string): Server | undefined {
+ return remoteServers().find((server) => tunnelHostLabel(server.id) === label);
+ }
+
+ private generationOf(serverId: string): number {
+ return this.generation.get(serverId) ?? 0;
+ }
+
+ /** Handshake a server's tunnel is not already running. */
+ private async open(serverId: string, generation: number): Promise {
+ const server = getServers().find((candidate) => candidate.id === serverId);
+
+ if (!this.isEnabled) {
+ throw new Error(
+ 'Remote server pairing is turned off. Turn it on in Settings, under General.'
+ );
+ }
+
+ if (!isRemoteServer(server) || !codeOf(server)) {
+ throw new Error('This server has no usable access code');
+ }
+
+ this.status[serverId] = TunnelStatus.CONNECTING;
+ this.errorMessage[serverId] = '';
+
+ // a dropped attempt must report nothing: the channel belongs to whoever asked
+ // for it last, and this one may land after that
+ const stillWanted = () => generation === this.generationOf(serverId);
+ const code = codeOf(server);
+ const tunnel = new ClientTunnel(
+ code.slice(0, CODE_LENGTHS.ROOM),
+ code.slice(CODE_LENGTHS.ROOM),
+ {
+ onConnected: () => {
+ if (stillWanted()) this.status[serverId] = TunnelStatus.CONNECTED;
+ },
+ onDisconnected: () => {
+ // the channel died: the server stays registered, its requests fail
+ // loudly until it is reconnected
+ if (!stillWanted()) return;
+
+ this.status[serverId] = TunnelStatus.ERROR;
+ this.errorMessage[serverId] = 'Disconnected from the host';
+ this.tunnels.delete(serverId);
+ this.rejectWaiters(serverId, new Error('Remote access disconnected'));
+ }
+ }
+ );
+
+ try {
+ await tunnel.connect();
+
+ // dropped while the handshake ran: the user is done with this channel, so
+ // the answer must not bring it back
+ if (!stillWanted()) {
+ tunnel.disconnect();
+
+ return;
+ }
+
+ this.tunnels.set(serverId, tunnel);
+ this.status[serverId] = TunnelStatus.CONNECTED;
+ this.releaseWaiters(serverId);
+ } catch (error: unknown) {
+ tunnel.disconnect();
+
+ // dropped while the handshake ran: the state the user asked for stands,
+ // and the failure belongs to an attempt nobody waits on any more
+ if (!stillWanted()) throw error;
+
+ this.tunnels.delete(serverId);
+ this.status[serverId] = TunnelStatus.ERROR;
+ this.errorMessage[serverId] = error instanceof Error ? error.message : String(error);
+ this.rejectWaiters(serverId, error instanceof Error ? error : new Error(String(error)));
+
+ throw error;
+ }
+ }
+
+ private rejectWaiters(serverId: string, error: Error): void {
+ for (const waiter of this.waiters.get(serverId) ?? []) waiter.reject(error);
+ this.waiters.delete(serverId);
+ }
+
+ private releaseWaiters(serverId: string): void {
+ for (const waiter of this.waiters.get(serverId) ?? []) waiter.resolve();
+ this.waiters.delete(serverId);
+ }
+}
+
+export const remoteAccessStore = new RemoteAccessStore();
diff --git a/ui/apps/chat/src/lib/stores/servers.svelte.ts b/ui/apps/chat/src/lib/stores/servers.svelte.ts
new file mode 100644
index 000000000000..9e68b90b16dd
--- /dev/null
+++ b/ui/apps/chat/src/lib/stores/servers.svelte.ts
@@ -0,0 +1,220 @@
+/**
+ * serversStore - llama.cpp servers the UI can talk to.
+ *
+ * The built-in server is the llama-server serving this UI. The others are
+ * user-configured: a server on a base URL, or a remote one reached over a
+ * WebRTC tunnel by the access code its `llama-server --connect` prints. The
+ * store registers the resolved list with the api-base registry, which services
+ * use to build request URLs, and tells the remote access store when a tunnel is
+ * no longer wanted. A page published with servers (`llama-chat --server`)
+ * lists them in `llama-chat.json`, and the store adds each of them once.
+ */
+
+import {
+ ACTIVE_SERVER_LOCALSTORAGE_KEY,
+ LOCAL_BACKEND_ID,
+ PRESET_SERVERS_FILE,
+ PRESET_SERVERS_LOCALSTORAGE_KEY,
+ PRESET_SERVERS_META,
+ SETTINGS_KEYS
+} from '#lib/constants/index.ts';
+import { serverStore } from '#lib/stores/llama-server.svelte.ts';
+import { remoteAccessStore } from '#lib/stores/remote-access.svelte.ts';
+import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
+import type { Server } from '#lib/types/index.ts';
+import { setServersResolver } from '#lib/utils/api-base.ts';
+import {
+ createLocalServer,
+ createRemoteServer,
+ createUrlServer,
+ isRemoteServer,
+ parsePresetServers,
+ parseServersSettings,
+ presetServersToAdd
+} from '#lib/utils/servers.ts';
+import { uuid } from '#lib/utils/uuid.ts';
+import { browser } from '$app/env';
+
+function loadActiveServerId(): string {
+ if (!browser) return LOCAL_BACKEND_ID;
+
+ try {
+ return localStorage.getItem(ACTIVE_SERVER_LOCALSTORAGE_KEY) ?? LOCAL_BACKEND_ID;
+ } catch {
+ return LOCAL_BACKEND_ID;
+ }
+}
+
+function persistActiveServerId(serverId: string): void {
+ if (!browser) return;
+
+ try {
+ localStorage.setItem(ACTIVE_SERVER_LOCALSTORAGE_KEY, serverId);
+ } catch {
+ /* ignore */
+ }
+}
+
+function loadSeenPresets(): string[] {
+ try {
+ const seen: unknown = JSON.parse(localStorage.getItem(PRESET_SERVERS_LOCALSTORAGE_KEY) ?? '[]');
+
+ return Array.isArray(seen) ? seen.filter((url) => typeof url === 'string') : [];
+ } catch {
+ return [];
+ }
+}
+
+function persistSeenPresets(urls: string[]): void {
+ try {
+ localStorage.setItem(PRESET_SERVERS_LOCALSTORAGE_KEY, JSON.stringify(urls));
+ } catch {
+ /* ignore */
+ }
+}
+
+class ServersStore {
+ activeId = $state(loadActiveServerId());
+
+ get active(): Server {
+ const active = this.enabled.find((server) => server.id === this.activeId);
+
+ return active ?? this.enabled[0] ?? this.local;
+ }
+
+ get enabled(): Server[] {
+ return this.list.filter((server) => server.enabled && !this.isMissingLocal(server));
+ }
+
+ /** Every remote server, connected or not; their code is the credential. */
+ get external(): Server[] {
+ return parseServersSettings(settingsStore.config[SETTINGS_KEYS.SERVERS]);
+ }
+
+ get list(): Server[] {
+ return [this.local, ...this.external];
+ }
+
+ get local(): Server {
+ return createLocalServer(
+ settingsStore.config.apiKey?.toString().trim() || undefined,
+ settingsStore.config[SETTINGS_KEYS.LOCAL_BACKEND_ENABLED] !== false
+ );
+ }
+
+ /** Servers a tunnel can be opened to, enabled or not. */
+ get remote(): Server[] {
+ return this.external.filter(isRemoteServer);
+ }
+
+ /**
+ * Add the servers the page was published with, listed in `llama-chat.json`
+ * next to index.html, each once: one the user removed stays removed. Only a
+ * page whose index.html carries the marker for the file asks for it, so one
+ * served by llama-server or the dev server adds nothing. With
+ * no local server to fall back on, the app picks the first enabled one.
+ *
+ * @returns the ids of the servers it added
+ */
+ async addPresetServers(): Promise {
+ if (!browser || !document.querySelector(`meta[name="${PRESET_SERVERS_META}"]`)) return [];
+
+ let raw: unknown;
+
+ try {
+ const response = await fetch(`./${PRESET_SERVERS_FILE}`, { cache: 'no-cache' });
+
+ if (!response.ok) return [];
+
+ raw = await response.json();
+ } catch {
+ // not there, or not JSON: an SPA fallback that answers with index.html
+ return [];
+ }
+
+ const urls = parsePresetServers(raw);
+
+ if (urls.length === 0) return [];
+
+ const seen = loadSeenPresets();
+ const added = presetServersToAdd(urls, this.external, seen).map((url) =>
+ createUrlServer(url, uuid())
+ );
+
+ if (added.length > 0) this.saveExternal([...this.external, ...added]);
+
+ persistSeenPresets([...new Set([...seen, ...urls])]);
+
+ return added.map((server) => server.id);
+ }
+
+ /**
+ * Pair with a `llama-server --connect` host by its access code.
+ *
+ * The tunnel is not opened here: the caller decides when the handshake is
+ * worth running (the servers dialog tests it, startup only tests the
+ * server the app opens with).
+ */
+ addRemoteServer(code: string, name?: string, id?: string, apiKey?: string): Server {
+ const server = createRemoteServer(code, id ?? uuid(), name, apiKey);
+
+ this.saveExternal([...this.external, server]);
+
+ return server;
+ }
+
+ /** Point at a llama.cpp server on its own base URL. */
+ addServer(server: Server): void {
+ this.saveExternal([...this.external, server]);
+ }
+
+ initialize(): void {
+ if (!browser) return;
+
+ setServersResolver(() => ({ activeId: this.active.id, servers: this.list }));
+ }
+
+ removeServer(serverId: string): void {
+ this.saveExternal(this.external.filter((server) => server.id !== serverId));
+
+ // the code is the credential: dropping the server drops the tunnel too,
+ // and the store forgets its state on the next sync
+ remoteAccessStore.disconnect(serverId);
+ remoteAccessStore.syncServers();
+
+ if (this.activeId === serverId) {
+ this.setActive(LOCAL_BACKEND_ID);
+ }
+ }
+
+ setActive(serverId: string): void {
+ const id = this.list.some((server) => server.id === serverId) ? serverId : LOCAL_BACKEND_ID;
+
+ this.activeId = id;
+ persistActiveServerId(id);
+ }
+
+ setLocalEnabled(enabled: boolean): void {
+ settingsStore.updateConfig(SETTINGS_KEYS.LOCAL_BACKEND_ENABLED, enabled);
+ }
+
+ updateServer(serverId: string, updates: Partial): void {
+ this.saveExternal(
+ this.external.map((server) => (server.id === serverId ? { ...server, ...updates } : server))
+ );
+
+ // a disabled or re-coded server must not keep its data channel
+ remoteAccessStore.syncServers();
+ }
+
+ /** The built-in server counts only when a local server answered this session. */
+ private isMissingLocal(server: Server): boolean {
+ return server.id === LOCAL_BACKEND_ID && serverStore.localServerMissing;
+ }
+
+ private saveExternal(servers: Server[]): void {
+ settingsStore.updateConfig(SETTINGS_KEYS.SERVERS, JSON.stringify(servers));
+ }
+}
+
+export const serversStore = new ServersStore();
diff --git a/ui/apps/chat/src/lib/stores/serversModels.svelte.ts b/ui/apps/chat/src/lib/stores/serversModels.svelte.ts
new file mode 100644
index 000000000000..a10a68bb8aff
--- /dev/null
+++ b/ui/apps/chat/src/lib/stores/serversModels.svelte.ts
@@ -0,0 +1,150 @@
+/**
+ * serversModelsStore - Per-server model catalog cache.
+ *
+ * Prefetches every enabled server's model list, so switching servers is
+ * instant and the switcher can show load state. The active server's list
+ * still lives in modelsStore, which owns selection and chat wiring; this
+ * cache is the prefetch layer the switches start from.
+ *
+ * Remote servers are listed through their tunnel, so the prefetch skips one
+ * until its tunnel is open: a handshake per server is what `connectAndLoad`
+ * is for, never the startup sweep.
+ */
+
+import { ModelsService } from '#lib/services/models.service.ts';
+import { ServersService } from '#lib/services/servers.service.ts';
+import { remoteAccessStore } from '#lib/stores/remote-access.svelte.ts';
+import { serversStore } from '#lib/stores/servers.svelte.ts';
+import type { Server } from '#lib/types/index.ts';
+import type { ApiModelsListResponse } from '#lib/types/index.ts';
+import type { ModelSidecarFile } from '#lib/types/models.js';
+import type { ModelOption } from '#lib/types/models.js';
+import { isRemoteServer } from '#lib/utils/servers.ts';
+
+export interface ServerModelsState {
+ /** Draft sidecars the listing carries, keyed by the repo they belong to. */
+ drafts?: Record;
+ error: string | null;
+ loaded: boolean;
+ loading: boolean;
+ models: ModelOption[];
+ /** Untouched list payload, kept for the local server so its router rows survive a tab switch. */
+ raw?: ApiModelsListResponse;
+}
+
+const EMPTY_STATE: ServerModelsState = {
+ error: null,
+ loaded: false,
+ loading: false,
+ models: []
+};
+
+class ServersModelsStore {
+ private states = $state>({});
+
+ clear(serverId: string): void {
+ delete this.states[serverId];
+ }
+
+ /**
+ * Open a remote server's tunnel and list it.
+ *
+ * The card's Connect action and the server switch both want this; startup
+ * prefetch does not, since a handshake per server on first paint is not what
+ * a list of servers asked for.
+ */
+ async connectAndLoad(serverId: string): Promise {
+ await this.ensureLoaded(serverId, { connect: true });
+ }
+
+ /**
+ * Load a server's models once.
+ *
+ * @param connect - Open the tunnel of a remote server when it has none. Off
+ * by default: the startup prefetch must not handshake with every server.
+ */
+ async ensureLoaded(serverId: string, options?: { connect?: boolean }): Promise {
+ const server = serversStore.enabled.find((candidate) => candidate.id === serverId);
+
+ if (!server) return;
+
+ const state = this.states[serverId];
+
+ if (state?.loaded || state?.loading) return;
+
+ if (isRemoteServer(server) && !remoteAccessStore.isConnected(serverId)) {
+ if (!options?.connect) return;
+
+ this.states[serverId] = { error: null, loaded: false, loading: true, models: [] };
+
+ try {
+ await remoteAccessStore.connect(serverId);
+ } catch (error: unknown) {
+ this.states[serverId] = {
+ error: error instanceof Error ? error.message : String(error),
+ loaded: false,
+ loading: false,
+ models: []
+ };
+
+ return;
+ }
+ }
+
+ this.states[serverId] = { error: null, loaded: false, loading: true, models: [] };
+ await this.fetch(server);
+ }
+
+ get(serverId: string): ServerModelsState {
+ return this.states[serverId] ?? EMPTY_STATE;
+ }
+
+ /** Prefetch every enabled server's model list. */
+ async loadAll(): Promise {
+ const enabled = serversStore.enabled;
+ const ids = new Set(enabled.map((server) => server.id));
+
+ for (const id of Object.keys(this.states)) {
+ if (!ids.has(id)) {
+ delete this.states[id];
+ }
+ }
+
+ await Promise.all(enabled.map((server) => this.ensureLoaded(server.id)));
+ }
+
+ /**
+ * Ask a server for its list again, keeping what is already known. Used while
+ * a remote load settles, since its status never reaches the local feed.
+ */
+ async refresh(serverId: string): Promise {
+ const server = serversStore.enabled.find((candidate) => candidate.id === serverId);
+
+ // a remote server without a tunnel has nothing to answer; keep what is
+ // already known rather than reporting a transport failure as a server one
+ if (!server || (isRemoteServer(server) && !remoteAccessStore.isConnected(serverId))) return;
+
+ await this.fetch(server);
+ }
+
+ /** Forget a server's list and ask for it again, tunnel included. */
+ async reload(serverId: string): Promise {
+ this.clear(serverId);
+ await this.connectAndLoad(serverId);
+ }
+
+ private async fetch(server: Server): Promise {
+ const result = await ServersService.listModels(server);
+
+ this.states[server.id] = {
+ drafts: result.raw ? ModelsService.sidecarFilesByRepo(result.raw) : undefined,
+ error: result.error ?? null,
+ loaded: result.ok,
+ loading: false,
+ models: result.models,
+ raw: result.raw
+ };
+ }
+}
+
+export const serversModelsStore = new ServersModelsStore();
diff --git a/ui/apps/chat/src/lib/stores/settings/index.svelte.ts b/ui/apps/chat/src/lib/stores/settings/index.svelte.ts
index 4cdc095f8971..ed9cce3b780a 100644
--- a/ui/apps/chat/src/lib/stores/settings/index.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/settings/index.svelte.ts
@@ -8,7 +8,7 @@
*/
import { SETTING_CONFIG_DEFAULT, SETTINGS_KEYS } from '#lib/constants/index.ts';
-import { ColorMode } from '#lib/enums/index.ts';
+import { ColorMode, ServerKind } from '#lib/enums/index.ts';
import { ParameterSyncService } from '#lib/services/parameter-sync.service.ts';
import { SettingsService } from '#lib/services/settings.service.ts';
import { deviceStore } from '#lib/stores/device.svelte.ts';
@@ -90,6 +90,28 @@ class SettingsStore {
? { ...this.config }
: Object.fromEntries(Object.entries(this.config).filter(([key]) => key !== 'apiKey'));
+ // Servers each carry a credential: an API key, or the access code that
+ // opens a tunnel to someone's machine. Keys go unless the user opts in; a
+ // remote server without its code is meaningless, so it goes whole.
+ if (SETTINGS_KEYS.SERVERS in configToExport && !includeSensitiveData) {
+ try {
+ const servers = JSON.parse(configToExport[SETTINGS_KEYS.SERVERS] as string) as Array<
+ Record
+ >;
+ const safeServers = servers
+ .filter((server) => server.kind !== ServerKind.REMOTE)
+ .map((server) => {
+ delete server.apiKey;
+
+ return server;
+ });
+
+ configToExport[SETTINGS_KEYS.SERVERS] = JSON.stringify(safeServers);
+ } catch {
+ delete (configToExport as Record)[SETTINGS_KEYS.SERVERS];
+ }
+ }
+
// Handle MCP servers: exclude custom headers unless user opts in
if ('mcpServers' in configToExport && !includeSensitiveData) {
try {
diff --git a/ui/apps/chat/src/lib/stores/tools.svelte.ts b/ui/apps/chat/src/lib/stores/tools.svelte.ts
index 0e926cd1673e..3c0f6ebd065a 100644
--- a/ui/apps/chat/src/lib/stores/tools.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/tools.svelte.ts
@@ -26,9 +26,11 @@ import {
type ToolProvider
} from '#lib/extensions/index.ts';
import { ToolsService } from '#lib/services/tools.service.ts';
-// direct imports between stores, not via the barrel, to avoid circular deps
+import { serverStore } from '#lib/stores/llama-server.svelte.ts';
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
import { projectsStore } from '#lib/stores/projects.svelte.ts';
+// direct imports between stores, not via the barrel, to avoid circular deps
+import { serversStore } from '#lib/stores/servers.svelte.ts';
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
import type {
OpenAIToolDefinition,
@@ -251,6 +253,16 @@ class ToolsStore {
}
async fetchServerTools(): Promise {
+ // the /tools endpoint only exists on the local llama.cpp server, so the
+ // list follows the built-in server rather than the selected model
+ if (!serverStore.hasLocalServer || !serversStore.local.enabled) {
+ this._serverToolInfos = [];
+ this._serverTools = [];
+ this.cwdAwareTools = new SvelteSet();
+
+ return;
+ }
+
if (this._loading) return;
this._loading = true;
diff --git a/ui/apps/chat/src/lib/stores/ui.svelte.ts b/ui/apps/chat/src/lib/stores/ui.svelte.ts
index ac33944a0e44..fb7c3d3e6a9d 100644
--- a/ui/apps/chat/src/lib/stores/ui.svelte.ts
+++ b/ui/apps/chat/src/lib/stores/ui.svelte.ts
@@ -121,6 +121,8 @@ class UiStore {
projectDialog = $state(null);
/** Open state of the project picker, a phone's stand-in for the project submenus. */
projectPicker = $state(null);
+ /** Open state of the servers dialog, the door to pairing on a phone. */
+ serversOpen = $state(false);
/** Section ids in the order the user dragged them into, kept across reloads. */
sidebarSectionOrder = $state(loadSidebarSectionOrder());
/** Expanded desktop sidebar width in px; null keeps the stylesheet default. */
@@ -263,6 +265,17 @@ class UiStore {
this.projectPicker = options;
}
+ /** Open the servers, where a server is paired or pointed at: a page on a phone, a dialog elsewhere. */
+ openServers(): void {
+ if (deviceStore.isMobile) {
+ void goto(ROUTES.SERVERS);
+
+ return;
+ }
+
+ this.serversOpen = true;
+ }
+
/**
* Puts sections in the order the user gave them. One never moved, such as a
* new group or project, keeps its default place: right before the next section in
@@ -298,6 +311,7 @@ class UiStore {
this.modelsReturnRoute = page.url.hash || ROUTES.START;
}
+
/** Ask the composer to take focus, e.g. after a dialog closes onto the chat. */
requestComposerFocus(): void {
this.composerFocusRequested = true;
diff --git a/ui/apps/chat/src/lib/types/api.d.ts b/ui/apps/chat/src/lib/types/api.d.ts
index 371913fc4c26..c47aed3bd79b 100644
--- a/ui/apps/chat/src/lib/types/api.d.ts
+++ b/ui/apps/chat/src/lib/types/api.d.ts
@@ -307,6 +307,11 @@ export interface ApiLlamaCppServerProps {
webui_settings?: Record;
ui_settings?: Record;
cors_proxy_enabled?: boolean;
+ /**
+ * Access code a host started with `--connect` offers, so another device can
+ * pair with it. Absent on a server that does not run a host.
+ */
+ connect_code?: string;
}
export interface ApiChatCompletionRequest {
@@ -382,7 +387,9 @@ export interface ApiChatCompletionStreamChunk {
metadata?: { model?: string };
delta: {
content?: string;
+ reasoning?: string;
reasoning_content?: string;
+ reasoning_text?: string;
model?: string;
tool_calls?: ApiChatCompletionToolCallDelta[];
};
@@ -396,6 +403,17 @@ export interface ApiChatCompletionStreamChunk {
cache_n?: number;
};
prompt_progress?: ChatMessagePromptProgress;
+ /** Token counts, sent by OpenAI-compatible servers on the final chunk. */
+ usage?: ApiChatCompletionUsage;
+}
+
+export interface ApiChatCompletionUsage {
+ cached_tokens?: number;
+ completion_tokens?: number;
+ prompt_cache_hit_tokens?: number;
+ prompt_tokens?: number;
+ prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number };
+ total_tokens?: number;
}
export interface ApiChatCompletionResponse {
diff --git a/ui/apps/chat/src/lib/types/index.ts b/ui/apps/chat/src/lib/types/index.ts
index ada84091cc94..bddea98d76da 100644
--- a/ui/apps/chat/src/lib/types/index.ts
+++ b/ui/apps/chat/src/lib/types/index.ts
@@ -26,6 +26,7 @@ export type {
ApiChatCompletionToolCallDelta,
ApiChatCompletionToolCall,
ApiChatCompletionStreamChunk,
+ ApiChatCompletionUsage,
ApiChatCompletionResponse,
ApiSlotData,
ApiProcessingState,
@@ -36,6 +37,9 @@ export type {
ApiStreamSession
} from './api';
+// Server types
+export type { Server } from './servers';
+
// HuggingFace types
export type {
HfCatalogBuild,
diff --git a/ui/apps/chat/src/lib/types/models.d.ts b/ui/apps/chat/src/lib/types/models.d.ts
index 9d4a7c88882b..49ba8e4e27e8 100644
--- a/ui/apps/chat/src/lib/types/models.d.ts
+++ b/ui/apps/chat/src/lib/types/models.d.ts
@@ -16,9 +16,11 @@ export interface ModelOption {
id: string;
name: string;
model: string;
+ /** Server that serves this model; set on the aggregated option list. */
+ serverId?: string;
description?: string;
capabilities: string[];
- /** Context size reported by the provider's model listing, when it reports one. */
+ /** Context size reported by the server's model listing, when it reports one. */
contextLength?: number;
/** Draft sidecars downloaded alongside the model, e.g. `mtp` or `dflash`. */
draftSidecars?: ModelSidecarBadge[];
@@ -27,6 +29,8 @@ export interface ModelOption {
speaks?: boolean;
details?: ApiModelDetails['details'];
meta?: ApiModelDataEntry['meta'];
+ /** Load state the server's own listing reports, when it reports one. */
+ status?: ApiModelDataEntry['status'];
parsedId?: ParsedModelId;
aliases?: string[];
tags?: string[];
@@ -58,6 +62,21 @@ export interface ModelDownloadEntry {
repoWithTag: string;
}
+/**
+ * A draft sidecar file a model listing reports as its own entry. The router lists a
+ * downloaded sidecar as a model, so this is what pairs it back with its model.
+ */
+export interface ModelSidecarFile {
+ id: string;
+ kind: ModelSidecar;
+ /** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */
+ model: string;
+ /** Parameter count the sidecar reports, e.g. `35B-A3B`. */
+ params: string | null;
+ /** Quantization of the sidecar file, e.g. `Q4_0`. */
+ quant: string | null;
+}
+
export interface ParsedModelId {
raw: string;
orgName: string | null;
@@ -86,15 +105,3 @@ export interface ModalityCapabilities {
hasAudio: boolean;
hasVideo: boolean;
}
-
-/** Sidecar file a listing reports as its own model entry, paired back to its model. */
-export interface ModelSidecarFile {
- id: string;
- kind: ModelSidecar;
- /** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */
- model: string;
- /** Parameter count the sidecar reports, e.g. `35B-A3B`. */
- params: string | null;
- /** Quantization of the sidecar file, e.g. `Q4_0`. */
- quant: string | null;
-}
diff --git a/ui/apps/chat/src/lib/types/servers.d.ts b/ui/apps/chat/src/lib/types/servers.d.ts
new file mode 100644
index 000000000000..7eceddcf6c9f
--- /dev/null
+++ b/ui/apps/chat/src/lib/types/servers.d.ts
@@ -0,0 +1,23 @@
+import type { ServerKind } from '#lib/enums/index.ts';
+
+/**
+ * A llama.cpp server the UI talks to: the one serving this app, one the user
+ * pointed at by URL, or one reached over a WebRTC tunnel by its access code.
+ * All of them speak the llama.cpp API, so nothing is negotiated at runtime.
+ */
+export interface Server {
+ /** Sent as `Authorization: Bearer ` when set. */
+ apiKey?: string;
+ /** Endpoint origin, no trailing slash; empty for the built-in and remote servers. */
+ baseUrl: string;
+ /** Remote servers only: 40-character access code of the `llama-server --connect` host. */
+ code?: string;
+ enabled: boolean;
+ /** Extra request headers sent verbatim, e.g. an authenticating proxy. */
+ headers?: Record;
+ id: string;
+ /** How the server is reached; absent in stored data means `url`. */
+ kind: ServerKind;
+ modelCount?: number;
+ name: string;
+}
diff --git a/ui/apps/chat/src/lib/types/settings.d.ts b/ui/apps/chat/src/lib/types/settings.d.ts
index cca82cd997bd..d559b3f69227 100644
--- a/ui/apps/chat/src/lib/types/settings.d.ts
+++ b/ui/apps/chat/src/lib/types/settings.d.ts
@@ -29,6 +29,8 @@ export interface SettingsEntry {
/** Options rendered for RADIO fields. Each entry maps a `value` (the radio's selected value) to the underlying config `key` whose boolean state mirrors it. */
radioOptions?: Array<{ value: string; label: string; key: string; isExperimental?: boolean }>;
isExperimental?: boolean;
+ /** Caveat shown under the help text, e.g. a known security limitation. */
+ note?: string;
isPositiveInteger?: boolean;
/** When true, the field is rendered as a password input (e.g. API key). */
isPrivate?: boolean;
@@ -59,6 +61,8 @@ export interface SettingsFieldConfig {
label: string;
type: SettingsFieldType;
isExperimental?: boolean;
+ /** Caveat shown under the help text, e.g. a known security limitation. */
+ note?: string;
isPositiveInteger?: boolean;
isPrivate?: boolean;
placeholder?: string;
diff --git a/ui/apps/chat/src/lib/utils/api-base.ts b/ui/apps/chat/src/lib/utils/api-base.ts
new file mode 100644
index 000000000000..0a6265b94513
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/api-base.ts
@@ -0,0 +1,110 @@
+/**
+ * API base resolution for servers.
+ *
+ * The UI can talk to more than one server endpoint. Services build request
+ * URLs through {@link apiUrl} so a request always targets the right server.
+ * The servers store registers a resolver here; this module never imports the
+ * store, which keeps URL resolution free of store dependencies.
+ */
+
+import { withBase } from './base-path';
+import { isLocalServer, serverChatUrl, serverModelsUrl, serverOrigin } from './servers';
+import { API_ABSOLUTE_URL_PROTOCOLS, API_CHAT, API_MODELS } from '#lib/constants/index.ts';
+import type { Server } from '#lib/types/index.ts';
+
+/** Server list and active selection as exposed to URL resolution. */
+export interface ServersSnapshot {
+ activeId: string;
+ servers: Server[];
+}
+
+type ServersResolver = () => ServersSnapshot;
+
+let resolveServers: ServersResolver | null = null;
+
+/** Registered once by the servers store. */
+export function setServersResolver(resolver: ServersResolver | null): void {
+ resolveServers = resolver;
+}
+
+/**
+ * Look up a server by id, defaulting to the active one.
+ */
+export function getServer(serverId?: string): Server | undefined {
+ const snapshot = resolveServers?.();
+
+ if (!snapshot) return undefined;
+
+ const id = serverId ?? snapshot.activeId;
+
+ return snapshot.servers.find((server) => server.id === id);
+}
+
+/** Every registered server, in list order. */
+export function getServers(): Server[] {
+ return resolveServers?.().servers ?? [];
+}
+
+/**
+ * API root for a server: its own base URL, the synthetic tunnel origin of a
+ * remote server, or an empty string for the local one.
+ */
+export function getServerBaseUrl(serverId?: string): string {
+ const server = getServer(serverId);
+
+ return server ? serverOrigin(server) : '';
+}
+
+/**
+ * Request target for a server's chat completions endpoint. Returns a relative
+ * path for the local server and an absolute URL for every other one — its own
+ * base URL, or the tunnel origin of a remote server — so callers can hand the
+ * result straight to `fetch` (or to `apiFetch`, which resolves relative paths
+ * against the base).
+ */
+export function apiChatUrl(serverId?: string): string {
+ const server = getServer(serverId);
+
+ if (server && !isLocalServer(server)) {
+ return serverChatUrl(server);
+ }
+
+ return apiUrl(API_CHAT.COMPLETIONS, serverId);
+}
+
+/**
+ * Request target for a server's models listing. Returns a plain path for the
+ * local server (so `apiFetch` applies the base) and an absolute URL otherwise.
+ */
+export function apiModelsUrl(serverId?: string): string {
+ const server = getServer(serverId);
+
+ if (server && !isLocalServer(server)) {
+ return serverModelsUrl(server);
+ }
+
+ return API_MODELS.LIST;
+}
+
+/**
+ * Absolute URL for an API path on a server.
+ *
+ * Absolute URLs pass through untouched. Paths on the local server keep the
+ * existing base-path-relative form, so serving under a subpath still works.
+ * Paths on external servers resolve against the server's API root.
+ */
+export function apiUrl(path: string, serverId?: string): string {
+ if (API_ABSOLUTE_URL_PROTOCOLS.some((protocol) => path.startsWith(protocol))) {
+ return path;
+ }
+
+ const baseUrl = getServerBaseUrl(serverId);
+
+ if (!baseUrl) {
+ return withBase(path);
+ }
+
+ const root = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
+
+ return new URL(path.replace(/^\.?\//, ''), root).toString();
+}
diff --git a/ui/apps/chat/src/lib/utils/api-fetch.ts b/ui/apps/chat/src/lib/utils/api-fetch.ts
index 85534558a80f..0f90d910e16a 100644
--- a/ui/apps/chat/src/lib/utils/api-fetch.ts
+++ b/ui/apps/chat/src/lib/utils/api-fetch.ts
@@ -1,10 +1,6 @@
+import { apiUrl } from './api-base';
import { getAuthHeaders, getJsonHeaders } from './api-headers';
-import { withBase } from './base-path';
-import {
- API_ABSOLUTE_URL_PROTOCOLS,
- ERROR_MESSAGES,
- HTTP_CODE_TO_STRING
-} from '#lib/constants/index.ts';
+import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '#lib/constants/index.ts';
/**
* API Fetch Utilities
@@ -36,6 +32,8 @@ export interface ApiFetchOptions extends Omit {
* Default: false (uses JSON headers with Content-Type: application/json)
*/
authOnly?: boolean;
+ /** Server to target; defaults to the active one. */
+ serverId?: string;
/**
* Additional headers to merge with default headers.
*/
@@ -63,11 +61,10 @@ export interface ApiFetchOptions extends Omit {
* ```
*/
export async function apiFetch(path: string, options: ApiFetchOptions = {}): Promise {
- const { authOnly = false, headers: customHeaders, ...fetchOptions } = options;
- const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders();
+ const { authOnly = false, headers: customHeaders, serverId, ...fetchOptions } = options;
+ const baseHeaders = authOnly ? getAuthHeaders(serverId) : getJsonHeaders(serverId);
const headers = { ...baseHeaders, ...customHeaders };
- // absolute URLs with an allowed protocol pass through untouched; relative paths get the base prefix
- const url = API_ABSOLUTE_URL_PROTOCOLS.some((p) => path.startsWith(p)) ? path : withBase(path);
+ const url = apiUrl(path, serverId);
let response;
@@ -110,7 +107,7 @@ export async function apiFetchWithParams(
params: Record,
options: ApiFetchOptions = {}
): Promise {
- const url = new URL(basePath, window.location.href);
+ const url = new URL(apiUrl(basePath, options.serverId), window.location.href);
for (const [key, value] of Object.entries(params)) {
if (value !== undefined && value !== null) {
diff --git a/ui/apps/chat/src/lib/utils/api-headers.ts b/ui/apps/chat/src/lib/utils/api-headers.ts
index fa7f1b91748a..d7903f9e7242 100644
--- a/ui/apps/chat/src/lib/utils/api-headers.ts
+++ b/ui/apps/chat/src/lib/utils/api-headers.ts
@@ -1,26 +1,45 @@
+import { getServer } from './api-base';
import { redactValue } from './redact';
import { CORS_PROXY, HEADERS } from '#lib/constants/index.ts';
import { MimeTypeApplication } from '#lib/enums/index.ts';
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
+import type { Server } from '#lib/types/index.ts';
/**
- * Get authorization headers for API requests
- * Includes Bearer token if API key is configured
+ * Get authorization headers for API requests to a server.
*/
-export function getAuthHeaders(): Record {
- const currentConfig = settingsStore.config;
- const apiKey = currentConfig.apiKey?.toString().trim();
+export function getAuthHeaders(serverId?: string): Record {
+ const server = getServer(serverId);
+
+ if (server) return getAuthHeadersForServer(server);
+
+ // no servers resolver yet (early startup, or a non-browser call): keep the
+ // pre-servers behaviour and authenticate against the serving origin
+ const apiKey = settingsStore.config.apiKey?.toString().trim();
return apiKey ? { [HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKey}` } : {};
}
+/**
+ * Get authorization headers for a server object, including one that is not
+ * registered yet (used on the add-server form).
+ */
+export function getAuthHeadersForServer(server: Server): Record {
+ const headers: Record = { ...(server.headers ?? {}) };
+ const apiKey = server.apiKey?.trim();
+
+ if (apiKey) headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`;
+
+ return headers;
+}
+
/**
* Get standard JSON headers with optional authorization
*/
-export function getJsonHeaders(): Record {
+export function getJsonHeaders(serverId?: string): Record {
return {
[HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON,
- ...getAuthHeaders()
+ ...getAuthHeaders(serverId)
};
}
diff --git a/ui/apps/chat/src/lib/utils/api-key-validation.ts b/ui/apps/chat/src/lib/utils/api-key-validation.ts
index 2783b32414ee..f6c938557424 100644
--- a/ui/apps/chat/src/lib/utils/api-key-validation.ts
+++ b/ui/apps/chat/src/lib/utils/api-key-validation.ts
@@ -1,34 +1,36 @@
-import { withBase } from './base-path';
import { error } from '@sveltejs/kit';
-import { HEADERS } from '#lib/constants/index.ts';
+import { HEADERS, LOCAL_BACKEND_ID } from '#lib/constants/index.ts';
import { MimeTypeApplication } from '#lib/enums/index.ts';
import { settingsStore } from '#lib/stores/settings/index.svelte.ts';
+import { apiUrl } from '#lib/utils/api-base.ts';
import { browser } from '$app/env';
/**
- * Validates API key by making a request to the server props endpoint
- * Throws SvelteKit errors for authentication failures or server issues
+ * Validates the stored API key against the server serving this UI.
+ *
+ * Pinned to the built-in server, the way the model status feed is: with a
+ * remote server active, probing *that* server with this key would raise the
+ * API key splash over a credential that belongs to a different machine. The
+ * servers a user added carry their own key, and their failures surface on
+ * their own card instead.
+ *
+ * Throws SvelteKit errors for authentication failures or server issues.
*/
export async function validateApiKey(fetch: typeof globalThis.fetch): Promise {
if (!browser) {
return;
}
- const apiKey = settingsStore.config.apiKey;
+ const headers: Record = { [HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON };
+ // Probe /props even without a stored key: on a server started with --api-key
+ // the unauthenticated request returns 401 and surfaces the API key splash,
+ // which is the onboarding path for entering the key.
+ const apiKey = settingsStore.config.apiKey?.toString().trim();
- try {
- const headers: Record = {
- [HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON
- };
-
- // Probe /props even without a stored key: on a server started with
- // --api-key the unauthenticated request returns 401 and surfaces the
- // API key splash, which is the onboarding path for entering the key.
- if (apiKey) {
- headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`;
- }
+ if (apiKey) headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`;
- const response = await fetch(withBase('/props'), { headers });
+ try {
+ const response = await fetch(apiUrl('/props', LOCAL_BACKEND_ID), { headers });
if (!response.ok) {
if (response.status === 401 || response.status === 403) {
@@ -40,12 +42,10 @@ export async function validateApiKey(fetch: typeof globalThis.fetch): Promise();
+ const rank = new Map();
+
+ recentIds.forEach((id, index) => rank.set(id, index));
+
+ const rankOf = (id: string) => rank.get(id) ?? Number.MAX_SAFE_INTEGER;
+ // recently used models lead their section, the rest keep the server's order
+ const byRecency = (items: ModelItem[]) =>
+ rank.size === 0 ? items : [...items].sort((a, b) => rankOf(a.option.id) - rankOf(b.option.id));
+
+ for (const option of options) {
+ const serverId = option.serverId ?? '';
+
+ if (!byServer.has(serverId)) byServer.set(serverId, []);
+
+ byServer.get(serverId)!.push({ option });
+ }
+
+ return servers.map((server) => {
+ const items = byRecency(byServer.get(server.serverId) ?? []);
+
+ return { ...server, items: items.slice(0, limit), matched: items.length };
+ });
}
diff --git a/ui/apps/chat/src/lib/utils/model-option-id.ts b/ui/apps/chat/src/lib/utils/model-option-id.ts
new file mode 100644
index 000000000000..05ad3123aa31
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/model-option-id.ts
@@ -0,0 +1,28 @@
+/**
+ * Server-qualified model option ids.
+ *
+ * The selector lists models from every enabled server, so a bare model id can
+ * collide. Option ids are qualified as `::`; selection
+ * strips the prefix to find the row in the active server's list.
+ */
+
+const MODEL_OPTION_ID_SEPARATOR = '::';
+
+/** Prefix a raw model id with the server that serves it. */
+export function qualifyModelId(serverId: string, modelId: string): string {
+ return `${serverId}${MODEL_OPTION_ID_SEPARATOR}${modelId}`;
+}
+
+/** Server id of a qualified model option id, or null when unqualified. */
+export function serverIdFromModelId(qualifiedId: string): string | null {
+ const index = qualifiedId.indexOf(MODEL_OPTION_ID_SEPARATOR);
+
+ return index === -1 ? null : qualifiedId.slice(0, index);
+}
+
+/** Strip the server prefix from a qualified model option id. */
+export function rawModelId(qualifiedId: string): string {
+ const index = qualifiedId.indexOf(MODEL_OPTION_ID_SEPARATOR);
+
+ return index === -1 ? qualifiedId : qualifiedId.slice(index + MODEL_OPTION_ID_SEPARATOR.length);
+}
diff --git a/ui/apps/chat/src/lib/utils/qr-code.ts b/ui/apps/chat/src/lib/utils/qr-code.ts
new file mode 100644
index 000000000000..94d002490bed
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/qr-code.ts
@@ -0,0 +1,39 @@
+/**
+ * QR code of an access code, for carrying it from the screen it is shown on to
+ * the camera of another device.
+ *
+ * The payload is the code itself, in the shape the scanner accepts: the scanner
+ * hands whatever it decodes to `normalizeShareCode`, so a code pasted by hand
+ * and a code scanned off a screen go through the same door.
+ */
+
+import { normalizeShareCode } from './servers';
+import { renderSVG } from 'uqr';
+
+/** Module size of the rendered code, in SVG units. The image is sized by CSS. */
+const PIXEL_SIZE = 4;
+/** Quiet zone, in modules on each side; the spec asks for four, a camera needs them. */
+const BORDER = 4;
+
+/**
+ * Data URL of an SVG QR code carrying the access code, or null when the value
+ * is not a usable code: a QR a phone cannot pair with is worse than no QR.
+ *
+ * Black on white, pinned: a scanner reads dark modules on a light field, and
+ * the app's dark theme must not invert that.
+ */
+export function qrCodeDataUrl(code: string): string | null {
+ const normalized = normalizeShareCode(code);
+
+ if (!normalized) return null;
+
+ const svg = renderSVG(normalized, {
+ blackColor: '#000000',
+ border: BORDER,
+ ecc: 'M',
+ pixelSize: PIXEL_SIZE,
+ whiteColor: '#ffffff'
+ });
+
+ return `data:image/svg+xml;utf8,${encodeURIComponent(svg)}`;
+}
diff --git a/ui/apps/chat/src/lib/utils/qr-scan.ts b/ui/apps/chat/src/lib/utils/qr-scan.ts
new file mode 100644
index 000000000000..69fbfc9c5909
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/qr-scan.ts
@@ -0,0 +1,58 @@
+/**
+ * Access code scanning from the camera, without a dependency.
+ *
+ * Browsers that ship a native `BarcodeDetector` can read the QR code
+ * `llama-server --connect` prints; everywhere else the field is filled by
+ * hand, which is the same thing the user does when they copy the code from a
+ * terminal. The code alphabet excludes the characters that a camera frame is
+ * most likely to misread, so a miss is a length or charset failure and the
+ * field simply stays invalid.
+ */
+
+/** Minimal shape of the native detector; it is not in every lib.dom. */
+interface BarcodeDetectorLike {
+ detect(source: CanvasImageSource): Promise>;
+}
+
+type BarcodeDetectorConstructor = new (options?: { formats?: string[] }) => BarcodeDetectorLike;
+
+function detectorConstructor(): BarcodeDetectorConstructor | null {
+ if (typeof window === 'undefined') return null;
+
+ const candidate = (window as unknown as { BarcodeDetector?: BarcodeDetectorConstructor })
+ .BarcodeDetector;
+
+ return typeof candidate === 'function' ? candidate : null;
+}
+
+/** Whether this browser can read a QR code out of a camera frame. */
+export function supportsQrScan(): boolean {
+ return detectorConstructor() !== null;
+}
+
+/**
+ * Read a QR code out of one video frame.
+ *
+ * @returns The decoded value, or null when the frame carries none.
+ */
+export async function scanFrameForCode(video: HTMLVideoElement): Promise {
+ const Detector = detectorConstructor();
+
+ // HAVE_CURRENT_DATA and up: nothing to read out of an empty frame
+ if (!Detector || video.readyState < 2) return null;
+
+ try {
+ const barcodes = await new Detector({ formats: ['qr_code'] }).detect(video);
+
+ for (const barcode of barcodes) {
+ const value = barcode.rawValue ?? barcode.value;
+
+ if (value) return value;
+ }
+
+ return null;
+ } catch {
+ // a frame the detector cannot read is not an error worth surfacing
+ return null;
+ }
+}
diff --git a/ui/apps/chat/src/lib/utils/server-props.ts b/ui/apps/chat/src/lib/utils/server-props.ts
new file mode 100644
index 000000000000..51a6d109a784
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/server-props.ts
@@ -0,0 +1,26 @@
+/**
+ * Whether the endpoint behind a server really is a llama.cpp server.
+ *
+ * The UI drives llama.cpp endpoints only: it loads and unloads models, reads
+ * the router's feed and its chat template. An endpoint that speaks just the
+ * OpenAI API answers /v1/models happily and then breaks somewhere else, so the
+ * check is made where the UI can still say no - before the server is saved.
+ */
+
+function isJsonObject(value: unknown): value is Record {
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Whether a /props payload is a llama.cpp one.
+ *
+ * Both a model and a router answer with their generation defaults; a router
+ * builds its props by hand, so the role and the build string stand in for them.
+ */
+export function isLlamaCppProps(value: unknown): boolean {
+ if (!isJsonObject(value)) return false;
+
+ if (isJsonObject(value.default_generation_settings)) return true;
+
+ return typeof value.role === 'string' && typeof value.build_info === 'string';
+}
diff --git a/ui/apps/chat/src/lib/utils/server-status.ts b/ui/apps/chat/src/lib/utils/server-status.ts
new file mode 100644
index 000000000000..414ef05011d1
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/server-status.ts
@@ -0,0 +1,102 @@
+/**
+ * What a server card reports: the pill word, the sentence under it and the
+ * state the card's controls hang off.
+ *
+ * The words carry the weight here. A failed channel and an unreachable server
+ * need different repairs, and the transport's own sentence - a refused
+ * passcode, a tracker nobody answered - is the one the user can act on, so it
+ * is reported as it came rather than folded into a generic label.
+ */
+
+import { ServerCardState, TunnelStatus } from '#lib/enums/index.ts';
+
+/** Everything the card knows about a server when it has to name its state. */
+export interface ServerCardFacts {
+ enabled: boolean;
+ /** Models the listing reported. */
+ modelCount: number;
+ /** The listing's failure, empty while it answered or was never asked. */
+ modelsError: string;
+ modelsLoaded: boolean;
+ modelsLoading: boolean;
+ /** Whether the server is reached over a tunnel. */
+ remote: boolean;
+ tunnel: TunnelStatus;
+ /** The tunnel's failure, empty while it never failed. */
+ tunnelError: string;
+}
+
+export interface ServerCardStatus {
+ /** Sentence under the pill; empty when the state needs none. */
+ detail: string;
+ /** Pill word; empty for a state the card does not report. */
+ label: string;
+ state: ServerCardState;
+}
+
+/** State a card reports for a server, with the words that state is shown in. */
+export function serverCardStatusOf(facts: ServerCardFacts): ServerCardStatus {
+ const {
+ enabled,
+ modelCount,
+ modelsError,
+ modelsLoaded,
+ modelsLoading,
+ remote,
+ tunnel,
+ tunnelError
+ } = facts;
+ const connecting = remote && tunnel === TunnelStatus.CONNECTING;
+ const idle =
+ remote && tunnel === TunnelStatus.IDLE && !modelsLoading && !modelsLoaded && !modelsError;
+ const checking = enabled && !connecting && !idle && !modelsLoaded && !modelsError && !tunnelError;
+ const online =
+ enabled && modelsLoaded && !modelsError && (!remote || tunnel === TunnelStatus.CONNECTED);
+ const offline = enabled && (tunnel === TunnelStatus.ERROR || (!idle && Boolean(modelsError)));
+
+ if (connecting) {
+ return {
+ detail: 'Opening the tunnel to the host.',
+ label: 'Connecting',
+ state: ServerCardState.CONNECTING
+ };
+ }
+
+ if (checking) {
+ return {
+ detail: remote
+ ? 'Asking the host for the models it serves.'
+ : 'Asking the server for the models it serves.',
+ label: 'Checking',
+ state: ServerCardState.CHECKING
+ };
+ }
+
+ if (online) {
+ return {
+ detail: '',
+ label: modelCount === 1 ? '1 model' : `${modelCount} models`,
+ state: ServerCardState.ONLINE
+ };
+ }
+
+ if (offline) {
+ // the channel failing is the one failure the user can act on: the code may
+ // have expired, or the host may not be listening
+ if (remote && tunnel === TunnelStatus.ERROR) {
+ return { detail: tunnelError, label: 'Channel down', state: ServerCardState.CHANNEL_DOWN };
+ }
+
+ return { detail: modelsError, label: 'Unreachable', state: ServerCardState.UNREACHABLE };
+ }
+
+ if (idle) {
+ return {
+ detail: 'Paste the code the host prints, or press Connect.',
+ label: 'Not connected',
+ state: ServerCardState.IDLE
+ };
+ }
+
+ return { detail: '', label: '', state: ServerCardState.NONE };
+}
diff --git a/ui/apps/chat/src/lib/utils/servers.ts b/ui/apps/chat/src/lib/utils/servers.ts
new file mode 100644
index 000000000000..b5dd6567f154
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/servers.ts
@@ -0,0 +1,393 @@
+/**
+ * Server list parsing, defaults and endpoint URLs.
+ *
+ * External servers are persisted in settings as a JSON list. Malformed
+ * entries are dropped instead of throwing so a corrupted settings value can
+ * never break URL resolution.
+ *
+ * A remote server has no URL of its own: it is reached through a WebRTC
+ * tunnel and addressed by a synthetic origin that carries its id
+ * (`tunnelOrigin`). The host suffix is reserved, so the address can never be
+ * mistaken for a real server, and a request that escapes the tunnel transport
+ * fails in the browser instead of reaching the server serving this UI.
+ */
+
+import {
+ CODE_CHARS,
+ CODE_LENGTHS,
+ DEFAULT_SERVER_CHAT_PATH,
+ DEFAULT_SERVER_MODELS_PATH,
+ FAVICON_SERVICE_URL,
+ LOCAL_BACKEND_ID,
+ SERVER_ID_PREFIX,
+ TUNNEL_HOST_SUFFIX
+} from '#lib/constants/index.ts';
+import { ServerKind } from '#lib/enums/index.ts';
+import type { Server } from '#lib/types/index.ts';
+
+/** Icon size requested from the favicon service, shown at 16px. */
+const FAVICON_SIZE = 64;
+/** Longest host label a server id can be shortened to (DNS label limit). */
+const HOST_LABEL_MAX = 63;
+
+/** Absolute chat completions URL for a server. */
+export function serverChatUrl(server: Server): string {
+ return joinServerUrl(serverOrigin(server), DEFAULT_SERVER_CHAT_PATH);
+}
+
+/** Absolute models listing URL for a server. */
+export function serverModelsUrl(server: Server): string {
+ return joinServerUrl(serverOrigin(server), DEFAULT_SERVER_MODELS_PATH);
+}
+
+/**
+ * Origin a server's requests are addressed at: its own base URL, or the
+ * synthetic tunnel origin of a remote server. Empty for the built-in
+ * server, whose requests stay on the origin serving this UI.
+ */
+export function serverOrigin(server: Server): string {
+ if (isRemoteServer(server)) return tunnelOrigin(server.id);
+
+ return server.baseUrl.trim().replace(/\/+$/, '');
+}
+
+/**
+ * Favicon of a server's root domain, used when no bundled mark matches. Hosts
+ * rarely serve a favicon themselves, so the subdomain is dropped
+ * (`api.z.ai` -> `z.ai`) and the icon is requested through a favicon service.
+ * Returns null when the URL carries no usable host, which is always the case
+ * for a remote server: a tunnel serves no favicon.
+ */
+export function serverFaviconUrl(baseUrl: string): string | null {
+ if (isTunnelHost(hostOf(baseUrl))) return null;
+
+ try {
+ const labels = new URL(baseUrl).hostname.split('.');
+ const root = labels.length > 2 ? labels.slice(-2).join('.') : labels.join('.');
+
+ return root ? `${FAVICON_SERVICE_URL}${root}&sz=${FAVICON_SIZE}` : null;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * The access code of a remote server, trimmed of the spacing a user tends to
+ * paste with. Empty when the value is not a usable code.
+ */
+export function codeOf(server?: Server | null): string {
+ return normalizeShareCode(server?.code ?? '');
+}
+
+/** The built-in server pointing at the server that serves this UI. */
+export function createLocalServer(apiKey?: string, enabled = true): Server {
+ return {
+ apiKey,
+ baseUrl: '',
+ enabled,
+ id: LOCAL_BACKEND_ID,
+ kind: ServerKind.LOCAL,
+ name: 'Local'
+ };
+}
+
+/** A remote llama.cpp server, reached through the tunnel of its access code. */
+export function createRemoteServer(
+ code: string,
+ id: string,
+ name?: string,
+ apiKey?: string
+): Server {
+ return {
+ apiKey,
+ baseUrl: '',
+ code: normalizeShareCode(code),
+ enabled: true,
+ id,
+ kind: ServerKind.REMOTE,
+ name: name?.trim() || 'Remote server'
+ };
+}
+
+/** A llama.cpp server answering on its own base URL. */
+export function createUrlServer(
+ baseUrl: string,
+ id: string,
+ name?: string,
+ apiKey?: string,
+ headers?: Record
+): Server {
+ return {
+ apiKey,
+ baseUrl: baseUrl.trim().replace(/\/+$/, ''),
+ enabled: true,
+ headers: headers && Object.keys(headers).length > 0 ? headers : undefined,
+ id,
+ kind: ServerKind.URL,
+ name: name?.trim() || hostOf(baseUrl) || baseUrl.trim()
+ };
+}
+
+/** Whether the built-in server is the one serving this UI. */
+export function isLocalServer(server?: Server | null): boolean {
+ return !server || server.kind === ServerKind.LOCAL;
+}
+
+/** Whether a server is reached through a WebRTC tunnel. */
+export function isRemoteServer(server?: Server | null): boolean {
+ return server?.kind === ServerKind.REMOTE;
+}
+
+/**
+ * Whether a server can be reached at all: the built-in one always, a remote
+ * one with a valid access code, and a URL one with a URL that parses and does
+ * not squat on the tunnel host suffix.
+ */
+export function isReachableServer(server: Server): boolean {
+ if (isLocalServer(server)) return true;
+
+ if (isRemoteServer(server)) return Boolean(codeOf(server));
+
+ const baseUrl = server.baseUrl.trim();
+
+ if (!baseUrl || isTunnelHost(hostOf(baseUrl))) return false;
+
+ return isHttpUrl(baseUrl);
+}
+
+/** Host of a URL, empty when it does not parse. */
+export function hostOf(url: string): string {
+ try {
+ return new URL(url).hostname;
+ } catch {
+ return '';
+ }
+}
+
+/** Whether a value parses as an http(s) URL. */
+export function isHttpUrl(value: string): boolean {
+ try {
+ const { protocol } = new URL(value);
+
+ return protocol === 'http:' || protocol === 'https:';
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * Strip the spacing from a pasted access code and check it against the alphabet
+ * llama-server generates its code from. Empty when the value is not a code.
+ */
+export function normalizeShareCode(code: string): string {
+ const code_ = code.replace(/\s/g, '');
+
+ if (code_.length !== CODE_LENGTHS.SHARE) return '';
+
+ for (const char of code_) {
+ if (!CODE_CHARS.includes(char)) return '';
+ }
+
+ return code_;
+}
+
+/**
+ * Host label a server's tunnel origin carries. Exported so a host coming back
+ * from the transport can be matched against a server without un-sanitizing it.
+ */
+export function tunnelHostLabel(serverId: string): string {
+ return hostLabel(serverId);
+}
+
+/**
+ * Host label of a tunnel origin, or null when the request is not addressed at
+ * one. The label is a server id in its hostname-safe form, which is how
+ * `tunnelOrigin` wrote it.
+ */
+export function parseTunnelOrigin(url: RequestInfo | URL): string | null {
+ const href = url instanceof URL ? url.href : typeof url === 'string' ? url : url.url;
+ const host = hostOf(href);
+
+ if (!isTunnelHost(host)) return null;
+
+ return host.slice(0, host.length - TUNNEL_HOST_SUFFIX.length - 1);
+}
+
+function joinServerUrl(origin: string, path: string): string {
+ const suffix = path.startsWith('/') ? path : `/${path}`;
+
+ return `${origin}${suffix}`;
+}
+
+/** Hostname-safe form of a server id; both sides of the comparison use it. */
+function hostLabel(serverId: string): string {
+ return (
+ serverId
+ .trim()
+ .toLowerCase()
+ .replace(/[^a-z0-9-]/g, '-')
+ .replace(/^-+|-+$/g, '')
+ .slice(0, HOST_LABEL_MAX) || 'server'
+ );
+}
+
+function isTunnelHost(host: string): boolean {
+ return host === TUNNEL_HOST_SUFFIX || host.endsWith(`.${TUNNEL_HOST_SUFFIX}`);
+}
+
+function parseServerEntry(entry: unknown, index: number): Server | null {
+ if (!entry || typeof entry !== 'object') return null;
+
+ const raw = entry as Record;
+ const id =
+ typeof raw.id === 'string' && raw.id.trim()
+ ? raw.id.trim()
+ : `${SERVER_ID_PREFIX}-${index + 1}`;
+ const name = typeof raw.name === 'string' && raw.name.trim() ? raw.name.trim() : '';
+ const enabled = raw.enabled !== false;
+ const headers = parseServerHeaders(raw.headers);
+ const code = typeof raw.code === 'string' ? normalizeShareCode(raw.code) : '';
+ const baseUrl = typeof raw.baseUrl === 'string' ? raw.baseUrl.trim().replace(/\/+$/, '') : '';
+ // a server's key is its credential: dropping it here would silently turn
+ // every request into an unauthenticated one
+ const apiKey =
+ typeof raw.apiKey === 'string' && raw.apiKey.trim() ? raw.apiKey.trim() : undefined;
+
+ // a remote server is its code; without a usable one it is nothing
+ if (raw.kind === ServerKind.REMOTE) {
+ if (!code) return null;
+
+ return {
+ apiKey,
+ baseUrl: '',
+ code,
+ enabled,
+ headers,
+ id,
+ kind: ServerKind.REMOTE,
+ name: name || 'Remote server'
+ };
+ }
+
+ // the local server is built in and never persisted
+ if (!baseUrl || id === LOCAL_BACKEND_ID) return null;
+
+ // a URL server must not squat on the host the tunnel transport routes
+ if (isTunnelHost(hostOf(baseUrl)) || !isHttpUrl(baseUrl)) return null;
+
+ return {
+ apiKey,
+ baseUrl,
+ enabled,
+ headers,
+ id,
+ kind: ServerKind.URL,
+ name: name || hostOf(baseUrl)
+ };
+}
+
+function parseServerHeaders(raw: unknown): Record | undefined {
+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined;
+
+ const entries = Object.entries(raw as Record)
+ .filter(([, value]) => typeof value === 'string' && value.trim() !== '')
+ .map(([key, value]) => [key.trim(), (value as string).trim()] as const);
+
+ return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+
+/**
+ * The server URLs a `llama-chat.json` lists, without a trailing slash; entries
+ * that are not http(s) URLs, or squat on the tunnel host, are dropped. An entry
+ * is `{ "url": "..." }` or the URL itself.
+ */
+export function parsePresetServers(raw: unknown): string[] {
+ const list = (raw as { servers?: unknown } | null)?.servers;
+
+ if (!Array.isArray(list)) return [];
+
+ const urls = list.flatMap((entry) => {
+ const value = typeof entry === 'string' ? entry : (entry as { url?: unknown } | null)?.url;
+
+ if (typeof value !== 'string') return [];
+
+ const url = value.trim().replace(/\/+$/, '');
+
+ return isHttpUrl(url) && !isTunnelHost(hostOf(url)) ? [url] : [];
+ });
+
+ return [...new Set(urls)];
+}
+
+/**
+ * The preset URLs to add: those no stored server points at already, and that
+ * were not added before, so a preset server the user removed stays removed.
+ */
+export function presetServersToAdd(urls: string[], servers: Server[], seen: string[]): string[] {
+ const stored = new Set(servers.map((server) => server.baseUrl));
+
+ return urls.filter((url) => !stored.has(url) && !seen.includes(url));
+}
+
+/**
+ * Parse the persisted servers JSON into server entries.
+ */
+export function parseServersSettings(rawServers: unknown): Server[] {
+ if (!rawServers) return [];
+
+ let parsed: unknown;
+
+ if (typeof rawServers === 'string') {
+ const trimmed = rawServers.trim();
+
+ if (!trimmed) return [];
+
+ try {
+ parsed = JSON.parse(trimmed);
+ } catch (error) {
+ console.warn('[servers] Failed to parse servers JSON, ignoring value:', error);
+
+ return [];
+ }
+ } else {
+ parsed = rawServers;
+ }
+
+ if (!Array.isArray(parsed)) return [];
+
+ return parsed.flatMap((entry, index) => {
+ const server = parseServerEntry(entry, index);
+
+ return server ? [server] : [];
+ });
+}
+
+/**
+ * Read a model's context size out of one `/v1/models` entry. A server leaves it
+ * unknown when it has not loaded the model, and a model-mode listing reports the
+ * trained size under `meta.n_ctx_train` instead.
+ */
+export function readModelContextLength(value: unknown): number | undefined {
+ if (!value || typeof value !== 'object') return undefined;
+
+ const entry = value as Record;
+
+ if (typeof entry.context_length === 'number' && entry.context_length > 0) {
+ return entry.context_length;
+ }
+
+ const meta = entry.meta;
+
+ if (meta && typeof meta === 'object') {
+ const trained = (meta as Record).n_ctx_train;
+
+ if (typeof trained === 'number' && trained > 0) return trained;
+ }
+
+ return undefined;
+}
+
+/** Origin a remote server's requests are addressed at. */
+export function tunnelOrigin(serverId: string): string {
+ return `https://${hostLabel(serverId)}.${TUNNEL_HOST_SUFFIX}`;
+}
diff --git a/ui/apps/chat/src/lib/utils/timings.ts b/ui/apps/chat/src/lib/utils/timings.ts
new file mode 100644
index 000000000000..6ecce85d02bc
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/timings.ts
@@ -0,0 +1,66 @@
+/**
+ * Client side timing fallback for servers that do not report their own.
+ *
+ * llama.cpp streams per-token timings; OpenAI-compatible servers
+ * do not. Token counts come from the usage block of the final chunk (or the
+ * count of streamed deltas as a fallback), times are measured locally: the wait
+ * for the first token is attributed to prompt processing, the rest to
+ * generation. Wall clock, so network and queueing are part of the numbers.
+ */
+
+import type { ApiChatCompletionUsage } from '#lib/types/api.js';
+import type { ChatMessageTimings } from '#lib/types/chat.js';
+
+export interface StreamClock {
+ startedAt: number;
+ firstTokenAt: number | null;
+ lastTokenAt: number | null;
+}
+
+/**
+ * Prompt/output/cache token counts. `promptTokens` excludes the cache read
+ * tokens, which are returned separately as `cacheTokens`, so the two always
+ * add up to the prompt size.
+ */
+export function usageTokenCounts(usage: ApiChatCompletionUsage | undefined): {
+ cacheTokens: number;
+ completionTokens: number;
+ promptTokens: number;
+} {
+ // a total that includes the cache reads, which are reported separately
+ const cacheTokens =
+ usage?.prompt_tokens_details?.cached_tokens ??
+ usage?.prompt_cache_hit_tokens ??
+ usage?.cached_tokens ??
+ 0;
+ const promptTotal = usage?.prompt_tokens ?? 0;
+
+ return {
+ cacheTokens,
+ completionTokens: usage?.completion_tokens ?? 0,
+ promptTokens: Math.max(0, promptTotal - cacheTokens)
+ };
+}
+
+export function buildTimingsFromUsage(
+ usage: ApiChatCompletionUsage | undefined,
+ clock: StreamClock,
+ fallbackTokens = 0
+): ChatMessageTimings | null {
+ const { cacheTokens, completionTokens, promptTokens } = usageTokenCounts(usage);
+ const predictedN = completionTokens || fallbackTokens;
+
+ if (promptTokens === 0 && predictedN === 0) return null;
+
+ const { firstTokenAt, startedAt } = clock;
+ const lastTokenAt = clock.lastTokenAt ?? firstTokenAt;
+
+ return {
+ cache_n: cacheTokens,
+ // clamp so a one-token reply still reports a positive duration
+ predicted_ms: firstTokenAt && lastTokenAt ? Math.max(1, lastTokenAt - firstTokenAt) : undefined,
+ predicted_n: predictedN,
+ prompt_ms: firstTokenAt ? Math.max(1, firstTokenAt - startedAt) : undefined,
+ prompt_n: promptTokens
+ };
+}
diff --git a/ui/apps/chat/src/lib/utils/webrtc-tunnel.ts b/ui/apps/chat/src/lib/utils/webrtc-tunnel.ts
new file mode 100644
index 000000000000..252848ca77a4
--- /dev/null
+++ b/ui/apps/chat/src/lib/utils/webrtc-tunnel.ts
@@ -0,0 +1,490 @@
+/**
+ * WebRTC tunnel for remote llama.cpp access.
+ *
+ * Signaling uses the WebTorrent tracker WebSocket protocol (no external deps).
+ * The room code is used as the info_hash rendezvous key; the pass code
+ * authenticates the client on the data channel after the WebRTC handshake.
+ *
+ * The client sends an offer, authenticates via pass code, then every request
+ * addressed at the server's tunnel origin is forwarded through the data
+ * channel to the host running llama-connect next to its local server.
+ *
+ * One instance serves exactly one server, and `remoteAccessStore` owns them:
+ * a server is addressed by its tunnel origin, so the request path the host
+ * receives is already clean and needs no base-path handling.
+ */
+
+import {
+ CODE_CHARS,
+ CODE_LENGTHS,
+ ID_LENGTHS,
+ STUN_URLS,
+ TRACKER_URLS,
+ WEBRTC_TIMEOUTS
+} from '#lib/constants/index.ts';
+
+const STUN_CONFIG: RTCConfiguration = {
+ iceServers: STUN_URLS.map((urls) => ({ urls }))
+};
+
+function randomStr(len: number): string {
+ const bytes = new Uint8Array(len);
+
+ crypto.getRandomValues(bytes);
+ let result = '';
+
+ for (const b of bytes) result += CODE_CHARS[b % CODE_CHARS.length];
+
+ return result;
+}
+
+// info_hash must be exactly 20 chars for WebTorrent trackers
+function roomToInfoHash(roomCode: string): string {
+ return roomCode.padEnd(CODE_LENGTHS.INFO_HASH, '0').slice(0, CODE_LENGTHS.INFO_HASH);
+}
+
+// Non-trickle ICE: the offer carries whatever candidates were gathered when
+// the deadline hits, since a single slow candidate must not sink the offer.
+function waitForIceComplete(pc: RTCPeerConnection): Promise {
+ return new Promise((resolve) => {
+ if (pc.iceGatheringState === 'complete') {
+ resolve();
+
+ return;
+ }
+
+ const timer = setTimeout(resolve, WEBRTC_TIMEOUTS.ICE_GATHER_MS);
+
+ pc.addEventListener('icegatheringstatechange', () => {
+ if (pc.iceGatheringState === 'complete') {
+ clearTimeout(timer);
+ resolve();
+ }
+ });
+ });
+}
+
+// ---------------------------------------------------------------------------
+// Tracker signaling (WebTorrent WS tracker protocol)
+// ---------------------------------------------------------------------------
+
+type TrackerMsg = Record;
+
+class Tracker {
+ onAnswer?: (offerId: string, answer: RTCSessionDescriptionInit) => void;
+ onClose?: () => void;
+ private readonly infoHash: string;
+
+ private readonly peerId: string;
+ private ws: WebSocket | null = null;
+
+ announce(
+ opts: {
+ numwant?: number;
+ offers?: Array<{ offer_id: string; offer: RTCSessionDescriptionInit }>;
+ } = {}
+ ): void {
+ const msg: TrackerMsg = {
+ action: 'announce',
+ info_hash: this.infoHash,
+ numwant: opts.numwant ?? 0,
+ peer_id: this.peerId
+ };
+
+ if (opts.offers) msg.offers = opts.offers;
+
+ this.send(msg);
+ }
+
+ close(): void {
+ this.ws?.close();
+ this.ws = null;
+ }
+
+ connect(url: string): Promise {
+ return new Promise((resolve, reject) => {
+ const ws = new WebSocket(url);
+
+ this.ws = ws;
+ const timer = setTimeout(
+ () => reject(new Error('tracker connect timeout')),
+ WEBRTC_TIMEOUTS.TRACKER_CONNECT_MS
+ );
+
+ ws.onopen = () => {
+ clearTimeout(timer);
+ resolve();
+ };
+ ws.onerror = () => {
+ clearTimeout(timer);
+ reject(new Error('tracker WebSocket error'));
+ };
+ ws.onclose = () => {
+ this.onClose?.();
+ };
+ ws.onmessage = (event) => {
+ try {
+ const msg = JSON.parse(event.data as string) as TrackerMsg;
+
+ if (msg.answer && msg.offer_id) {
+ this.onAnswer?.(msg.offer_id as string, msg.answer as RTCSessionDescriptionInit);
+ }
+ } catch {
+ // ignore malformed tracker messages
+ }
+ };
+ });
+ }
+
+ constructor(infoHash: string, peerId: string) {
+ this.infoHash = infoHash;
+ this.peerId = peerId;
+ }
+
+ private send(msg: TrackerMsg): void {
+ if (this.ws?.readyState === WebSocket.OPEN) {
+ this.ws.send(JSON.stringify(msg));
+ }
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Tunnel message types (JSON, sent over RTCDataChannel)
+// ---------------------------------------------------------------------------
+
+interface ReqMsg {
+ type: 'req';
+ id: string;
+ method: string;
+ path: string;
+ headers: Record;
+ body: string | null; // base64 or null
+}
+interface ResStartMsg {
+ type: 'res_start';
+ id: string;
+ status: number;
+ headers: Record;
+}
+interface ResChunkMsg {
+ type: 'res_chunk';
+ id: string;
+ data: string; // base64
+}
+interface ResEndMsg {
+ type: 'res_end';
+ id: string;
+}
+interface ResErrMsg {
+ type: 'res_err';
+ id: string;
+ message: string;
+}
+interface CancelMsg {
+ type: 'cancel';
+ id: string;
+}
+interface AuthMsg {
+ type: 'auth';
+ pass: string;
+}
+interface AuthOkMsg {
+ type: 'auth_ok';
+}
+interface AuthFailMsg {
+ type: 'auth_fail';
+}
+
+type TunnelMsg =
+ | ReqMsg
+ | ResStartMsg
+ | ResChunkMsg
+ | ResEndMsg
+ | ResErrMsg
+ | CancelMsg
+ | AuthMsg
+ | AuthOkMsg
+ | AuthFailMsg;
+
+function uint8ToBase64(bytes: Uint8Array): string {
+ let binary = '';
+
+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+
+ return btoa(binary);
+}
+
+function base64ToUint8(b64: string): Uint8Array {
+ return Uint8Array.from(atob(b64), (c) => c.charCodeAt(0));
+}
+// ---------------------------------------------------------------------------
+// ClientTunnel
+// ---------------------------------------------------------------------------
+
+export type ClientCallbacks = {
+ onConnected?: () => void;
+ onDisconnected?: () => void;
+};
+
+type PendingReq = {
+ onStart: (status: number, headers: Record) => void;
+ onChunk: (data: string) => void;
+ onEnd: () => void;
+ onError: (message: string) => void;
+};
+
+export class ClientTunnel {
+ private readonly callbacks: ClientCallbacks;
+ private channel: RTCDataChannel | null = null;
+ private readonly infoHash: string;
+ private readonly passCode: string;
+
+ private pc: RTCPeerConnection | null = null;
+ private readonly peerId: string;
+ private pending = new Map();
+ private trackers: Tracker[] = [];
+
+ get isConnected(): boolean {
+ return this.channel?.readyState === 'open';
+ }
+
+ async connect(): Promise {
+ const offerId = randomStr(ID_LENGTHS.OFFER);
+ const pc = new RTCPeerConnection(STUN_CONFIG);
+
+ this.pc = pc;
+
+ const channel = pc.createDataChannel('tunnel', { ordered: true });
+
+ this.channel = channel;
+
+ const offer = await pc.createOffer();
+
+ await pc.setLocalDescription(offer);
+ await waitForIceComplete(pc);
+
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ reject(new Error('connection timed out waiting for host'));
+ }, WEBRTC_TIMEOUTS.CONNECT_MS);
+
+ channel.onopen = () => {
+ channel.send(JSON.stringify({ pass: this.passCode, type: 'auth' } satisfies AuthMsg));
+ };
+
+ channel.onmessage = (event) => {
+ try {
+ const msg = JSON.parse(event.data as string) as TunnelMsg;
+
+ if (msg.type === 'auth_ok') {
+ clearTimeout(timer);
+ this.callbacks.onConnected?.();
+ resolve();
+ } else if (msg.type === 'auth_fail') {
+ clearTimeout(timer);
+ reject(new Error('authentication failed: invalid passcode'));
+ } else {
+ this.routeResponseMsg(msg);
+ }
+ } catch {
+ // ignore
+ }
+ };
+
+ channel.onclose = () => {
+ this.callbacks.onDisconnected?.();
+ this.rejectAllPending('connection closed');
+ };
+
+ channel.onerror = () => {
+ clearTimeout(timer);
+ reject(new Error('data channel error'));
+ };
+
+ // The host announces on every tracker, so the same offer goes out to
+ // all of them at once and the first answer back wins.
+ let answered = false;
+ let unreachable = 0;
+
+ for (const url of TRACKER_URLS) {
+ const tracker = new Tracker(this.infoHash, this.peerId);
+
+ this.trackers.push(tracker);
+
+ tracker.onAnswer = async (answeredOfferId, answer) => {
+ if (answeredOfferId !== offerId || answered) return;
+
+ answered = true;
+
+ try {
+ await pc.setRemoteDescription(new RTCSessionDescription(answer));
+ } catch (e) {
+ clearTimeout(timer);
+ reject(e);
+ }
+ };
+
+ tracker
+ .connect(url)
+ .then(() => {
+ tracker.announce({
+ numwant: 1,
+ offers: [{ offer: pc.localDescription!, offer_id: offerId }]
+ });
+ })
+ .catch(() => {
+ unreachable++;
+
+ if (unreachable === TRACKER_URLS.length) {
+ clearTimeout(timer);
+ reject(new Error('no tracker reachable'));
+ }
+ });
+ }
+ });
+ }
+
+ constructor(roomCode: string, passCode: string, callbacks: ClientCallbacks = {}) {
+ this.passCode = passCode;
+ this.infoHash = roomToInfoHash(roomCode);
+ this.peerId = randomStr(ID_LENGTHS.PEER);
+ this.callbacks = callbacks;
+ }
+
+ disconnect(): void {
+ this.rejectAllPending('disconnected');
+ this.cleanupConnection();
+ }
+
+ async fetch(input: RequestInfo | URL, init?: RequestInit): Promise {
+ if (!this.channel || this.channel.readyState !== 'open') {
+ throw new Error('tunnel not connected');
+ }
+
+ const request = new Request(input, init);
+ const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
+ const id = randomStr(ID_LENGTHS.REQUEST);
+
+ if (signal?.aborted) {
+ return Promise.reject(new DOMException('Aborted', 'AbortError'));
+ }
+
+ // Requests reach the tunnel addressed at the server's tunnel origin, so
+ // the path is already the one the host's server serves: nothing to strip.
+ const reqUrl = new URL(request.url);
+ const path = reqUrl.pathname + reqUrl.search;
+ const headers: Record = {};
+
+ request.headers.forEach((v, k) => {
+ headers[k] = v;
+ });
+
+ let bodyB64: string | null = null;
+
+ const bodyBytes = await request.arrayBuffer();
+
+ if (bodyBytes.byteLength > 0) {
+ bodyB64 = uint8ToBase64(new Uint8Array(bodyBytes));
+ }
+
+ return new Promise((resolve, reject) => {
+ let streamController!: ReadableStreamDefaultController;
+
+ const stream = new ReadableStream({
+ start(ctrl) {
+ streamController = ctrl;
+ }
+ });
+ const abortHandler = () => {
+ this.pending.delete(id);
+ try {
+ streamController.error(new DOMException('Aborted', 'AbortError'));
+ } catch {
+ // stream may already be closed
+ }
+ reject(new DOMException('Aborted', 'AbortError'));
+
+ // Tell the host to stop the in-flight fetch
+ if (this.channel?.readyState === 'open') {
+ this.channel.send(JSON.stringify({ id, type: 'cancel' } satisfies CancelMsg));
+ }
+ };
+
+ signal?.addEventListener('abort', abortHandler, { once: true });
+
+ this.pending.set(id, {
+ onChunk: (data) => {
+ streamController.enqueue(base64ToUint8(data));
+ },
+ onEnd: () => {
+ signal?.removeEventListener('abort', abortHandler);
+ streamController.close();
+ },
+ onError: (message) => {
+ signal?.removeEventListener('abort', abortHandler);
+ try {
+ streamController.error(new Error(message));
+ } catch {
+ // stream may already be closed
+ }
+ reject(new Error(message));
+ this.pending.delete(id);
+ },
+ onStart: (status, resHeaders) => {
+ resolve(new Response(stream, { headers: resHeaders, status }));
+ }
+ });
+
+ this.channel!.send(
+ JSON.stringify({
+ body: bodyB64,
+ headers,
+ id,
+ method: request.method,
+ path,
+ type: 'req'
+ } satisfies ReqMsg)
+ );
+ });
+ }
+
+ private cleanupConnection(): void {
+ this.channel?.close();
+ this.pc?.close();
+ for (const tracker of this.trackers) tracker.close();
+ this.channel = null;
+ this.pc = null;
+ this.trackers = [];
+ }
+
+ private rejectAllPending(reason: string): void {
+ for (const req of this.pending.values()) req.onError(reason);
+ this.pending.clear();
+ }
+
+ private routeResponseMsg(msg: TunnelMsg): void {
+ if (
+ msg.type !== 'res_start' &&
+ msg.type !== 'res_chunk' &&
+ msg.type !== 'res_end' &&
+ msg.type !== 'res_err'
+ )
+ return;
+
+ const req = this.pending.get(msg.id);
+
+ if (!req) return;
+
+ if (msg.type === 'res_start') {
+ req.onStart(msg.status, msg.headers);
+ } else if (msg.type === 'res_chunk') {
+ req.onChunk(msg.data);
+ } else if (msg.type === 'res_end') {
+ req.onEnd();
+ this.pending.delete(msg.id);
+ } else if (msg.type === 'res_err') {
+ req.onError(msg.message);
+ this.pending.delete(msg.id);
+ }
+ }
+}
diff --git a/ui/apps/chat/src/routes/+layout.svelte b/ui/apps/chat/src/routes/+layout.svelte
index 8b7cb80db335..f3f58fb65816 100644
--- a/ui/apps/chat/src/routes/+layout.svelte
+++ b/ui/apps/chat/src/routes/+layout.svelte
@@ -23,6 +23,9 @@
conversationsStore,
deviceStore,
modelsStore,
+ remoteAccessStore,
+ serversModelsStore,
+ serversStore,
serverStore,
settingsStore,
tabsStore,
@@ -30,8 +33,9 @@
versionStore
} from '#lib/stores/index.ts';
import { initStores } from '#lib/stores/init.ts';
- import { withBase } from '#lib/utils/base-path.ts';
+ import { apiUrl } from '#lib/utils/api-base.ts';
import { applySurfaceTheme, hapticTap, resolveTabView } from '#lib/utils/index.ts';
+ import { isLocalServer, isRemoteServer } from '#lib/utils/servers.ts';
import { browser } from '$app/env';
import { afterNavigate, goto } from '$app/navigation';
import { page } from '$app/state';
@@ -204,7 +208,7 @@
[HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKey.trim()}`
};
- fetch(withBase('/props'), { headers })
+ fetch(apiUrl('/props'), { headers })
.then((response) => {
if (response.status === 401 || response.status === 403) {
window.location.reload();
@@ -219,7 +223,7 @@
onMount(() => {
updateFavicon();
- // snapshot of every backend running stream on first load, populates the sidebar spinners
+ // snapshot of every server running stream on first load, populates the sidebar spinners
// so the user sees each conv that has a live inference, even ones not opened yet
void chatStore.syncRemoteRunningStreams();
});
@@ -230,6 +234,10 @@
if (document.visibilityState !== 'visible') return;
void chatStore.syncRemoteRunningStreams();
+
+ // mobile browsers drop a peer connection when the tab is backgrounded, so a
+ // tunnel that was open is asked for again on the way back
+ remoteAccessStore.recoverChannels();
}
$effect(() => {
@@ -238,6 +246,11 @@
updateFavicon();
});
+ // turning the experimental remote pairing off closes the tunnels it opened
+ $effect(() => {
+ if (!remoteAccessStore.isEnabled) untrack(() => remoteAccessStore.syncServers());
+ });
+
// Initialize server properties on app load (run once)
$effect(() => {
// Only fetch if we don't already have props
@@ -265,37 +278,34 @@
node.textContent = (settingsStore.config.customCss as string | undefined) ?? '';
}
- // Fetch router models when in router mode (for status and modalities)
- // Wait for models to be loaded first, run only once
- let routerModelsFetched = false;
-
+ // Live model status and load progress via the /models/sse feed (router mode).
+ // The feed is kept for the session: switching to an external server and back
+ // must not tear it down and reconnect on every tab switch.
$effect(() => {
- const isRouter = serverStore.isRouterMode;
- const modelsCount = modelsStore.models.length;
+ if (!browser) return;
- // Only fetch router models once when we have models loaded and in router mode
- if (isRouter && modelsCount > 0 && !routerModelsFetched) {
- routerModelsFetched = true;
+ if (!serverStore.localIsRouter) return;
- untrack(() => {
- modelsStore.fetchRouterModels();
- });
- }
+ untrack(() => {
+ modelsStore.status.subscribe();
+ });
});
- // Live model status and load progress via the /models/sse feed (router mode)
+ // The active server's own feed, when it is not the built-in one: a llama.cpp
+ // server serves /models/sse at its origin, so a router behind a tunnel animates
+ // its loads the way a local one does. It follows that server being reachable,
+ // and the operations that wait on a load keep their polling as the fallback.
$effect(() => {
if (!browser) return;
- if (!serverStore.isRouterMode) return;
-
- untrack(() => {
- modelsStore.status.subscribe();
- });
+ const server = serversStore.active;
+ const reachable =
+ !isLocalServer(server) &&
+ (isRemoteServer(server)
+ ? remoteAccessStore.isConnected(server.id)
+ : serversModelsStore.get(server.id).loaded);
- return () => {
- modelsStore.status.unsubscribe();
- };
+ untrack(() => modelsStore.status.subscribeServer(reachable ? server.id : null));
});
// Monitor API key changes and redirect to error page if removed or changed when required
diff --git a/ui/apps/chat/src/routes/[extensionPage]/+page.ts b/ui/apps/chat/src/routes/[extensionPage]/+page.ts
index af674fcc8f60..98440b7e2176 100644
--- a/ui/apps/chat/src/routes/[extensionPage]/+page.ts
+++ b/ui/apps/chat/src/routes/[extensionPage]/+page.ts
@@ -1,12 +1,14 @@
import type { PageLoad } from './$types';
import { error } from '@sveltejs/kit';
import { extensions } from '#lib/extensions/index.ts';
-import { loadAppRoute } from '#lib/stores/init.ts';
+import { loadAppRoute, startExtensions } from '#lib/stores/init.ts';
export const load: PageLoad = async ({ fetch, params }) => {
await loadAppRoute(fetch);
+ // the rest of the startup runs after this load, so the extensions are started here
+ await startExtensions();
- // the extensions are active by now: a path none of them has a page for is the app's 404
+ // a path none of them has a page for is the app's 404
if (!extensions.contributions('pages').some((page) => page.path === params.extensionPage)) {
error(404, 'Not Found');
}
diff --git a/ui/apps/chat/src/routes/models/+page.svelte b/ui/apps/chat/src/routes/models/+page.svelte
index e8295430845b..0a4e3643a3ad 100644
--- a/ui/apps/chat/src/routes/models/+page.svelte
+++ b/ui/apps/chat/src/routes/models/+page.svelte
@@ -1,10 +1,15 @@
+
+
+ Servers - {APP_NAME}
+
+
+
+
+
+
Servers
+
+
+ {remoteAccessStore.isEnabled
+ ? 'Pair a remote server by its access code, or point at one by its URL.'
+ : 'Point at a llama-server by its URL.'}
+
+
+
+
+
diff --git a/ui/apps/chat/src/routes/servers/+page.ts b/ui/apps/chat/src/routes/servers/+page.ts
new file mode 100644
index 000000000000..3c3deac5178a
--- /dev/null
+++ b/ui/apps/chat/src/routes/servers/+page.ts
@@ -0,0 +1,4 @@
+import type { PageLoad } from './$types';
+import { loadAppRoute } from '#lib/stores/init.ts';
+
+export const load: PageLoad = ({ fetch }) => loadAppRoute(fetch);
diff --git a/ui/apps/chat/static/backend-presets/deepseek.svg b/ui/apps/chat/static/backend-presets/deepseek.svg
new file mode 100644
index 000000000000..185c108b458c
--- /dev/null
+++ b/ui/apps/chat/static/backend-presets/deepseek.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/apps/chat/static/backend-presets/google.svg b/ui/apps/chat/static/backend-presets/google.svg
new file mode 100644
index 000000000000..4545c8331f9e
--- /dev/null
+++ b/ui/apps/chat/static/backend-presets/google.svg
@@ -0,0 +1,10 @@
+
diff --git a/ui/apps/chat/static/backend-presets/huggingface.svg b/ui/apps/chat/static/backend-presets/huggingface.svg
new file mode 100644
index 000000000000..6b3b4f687eb3
--- /dev/null
+++ b/ui/apps/chat/static/backend-presets/huggingface.svg
@@ -0,0 +1,37 @@
+
diff --git a/ui/apps/chat/static/backend-presets/kimi.png b/ui/apps/chat/static/backend-presets/kimi.png
new file mode 100644
index 000000000000..213b9b83d6cd
Binary files /dev/null and b/ui/apps/chat/static/backend-presets/kimi.png differ
diff --git a/ui/apps/chat/static/backend-presets/openrouter-dark.svg b/ui/apps/chat/static/backend-presets/openrouter-dark.svg
new file mode 100644
index 000000000000..5df05ac31558
--- /dev/null
+++ b/ui/apps/chat/static/backend-presets/openrouter-dark.svg
@@ -0,0 +1,5 @@
+
+
diff --git a/ui/apps/chat/static/backend-presets/openrouter-light.svg b/ui/apps/chat/static/backend-presets/openrouter-light.svg
new file mode 100644
index 000000000000..8fbbb8717851
--- /dev/null
+++ b/ui/apps/chat/static/backend-presets/openrouter-light.svg
@@ -0,0 +1,5 @@
+
+
diff --git a/ui/apps/chat/static/backend-presets/qwen.png b/ui/apps/chat/static/backend-presets/qwen.png
new file mode 100644
index 000000000000..87a98a952624
Binary files /dev/null and b/ui/apps/chat/static/backend-presets/qwen.png differ
diff --git a/ui/apps/chat/static/backend-presets/zai.png b/ui/apps/chat/static/backend-presets/zai.png
new file mode 100644
index 000000000000..70c33481a4ce
Binary files /dev/null and b/ui/apps/chat/static/backend-presets/zai.png differ
diff --git a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
index 058c7e02ed98..d35aa0cf3272 100644
--- a/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
+++ b/ui/apps/chat/tests/client/chat-form-voice.svelte.test.ts
@@ -12,7 +12,7 @@ import {
VOICE_MODE_MESSAGE
} from '#lib/constants/index.ts';
import { ServerRole, VoiceInputMode } from '#lib/enums/index.ts';
-import { voiceTranscriptsStore } from '#lib/stores/index.ts';
+import { modelsStore, voiceTranscriptsStore } from '#lib/stores/index.ts';
import { parseFilesToMessageExtras } from '#lib/utils/browser-only.ts';
import { tick } from 'svelte';
import { afterEach, describe, expect, it } from 'vitest';
@@ -46,8 +46,7 @@ function seed(
/** A router whose chat model (Qwen3) can't hear while another (Gemma) can, none of them loaded. */
function seedRouter(transcriptionModel = '') {
const [chat, hearing] = routerModels;
-
- restore = seedStores({
+ const restoreStores = seedStores({
config: {
autoMicOnEmpty: true,
[SETTINGS_KEYS.VOICE_INPUT_MODE]: VoiceInputMode.AUTO,
@@ -63,6 +62,14 @@ function seedRouter(transcriptionModel = '') {
selectedModelId: chat.id,
serverRole: ServerRole.ROUTER
});
+ // the microphone looks among the active server's models for one that hears
+ const activeModels = modelsStore.activeModels;
+
+ modelsStore.activeModels = routerModels;
+ restore = () => {
+ modelsStore.activeModels = activeModels;
+ restoreStores();
+ };
}
async function renderForm(options: { isLoading?: boolean; voiceMode?: boolean } = {}) {
diff --git a/ui/apps/chat/tests/client/components/DialogServersWrapper.svelte b/ui/apps/chat/tests/client/components/DialogServersWrapper.svelte
new file mode 100644
index 000000000000..4f1d3ae5b2b4
--- /dev/null
+++ b/ui/apps/chat/tests/client/components/DialogServersWrapper.svelte
@@ -0,0 +1,15 @@
+
+
+
+
+
+
diff --git a/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte b/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte
index 494ccd81ae6b..92f55e9127e7 100644
--- a/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte
+++ b/ui/apps/chat/tests/client/components/ModelsManagerRowWrapper.svelte
@@ -12,7 +12,7 @@
let { onSelect, option, selected = false }: Props = $props();
-
+
false}
diff --git a/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte b/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte
index 26af1bf324af..7d12eb29ab99 100644
--- a/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte
+++ b/ui/apps/chat/tests/client/components/ModelsManagerWrapper.svelte
@@ -3,7 +3,7 @@
import ModelsManager from '#lib/components/app/models/ModelsManager/ModelsManager.svelte';
-
+
diff --git a/ui/apps/chat/tests/client/components/ServerCardWrapper.svelte b/ui/apps/chat/tests/client/components/ServerCardWrapper.svelte
new file mode 100644
index 000000000000..200dd144995d
--- /dev/null
+++ b/ui/apps/chat/tests/client/components/ServerCardWrapper.svelte
@@ -0,0 +1,18 @@
+
+
+
+
+
+
diff --git a/ui/apps/chat/tests/client/components/TestWrapper.svelte b/ui/apps/chat/tests/client/components/TestWrapper.svelte
index 3f050cdec8f5..f09265d7f87c 100644
--- a/ui/apps/chat/tests/client/components/TestWrapper.svelte
+++ b/ui/apps/chat/tests/client/components/TestWrapper.svelte
@@ -4,8 +4,8 @@
diff --git a/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts b/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts
index 32da9e692af3..646623708648 100644
--- a/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts
+++ b/ui/apps/chat/tests/client/models-manager-rows.svelte.test.ts
@@ -4,8 +4,10 @@
import ModelsManagerRowWrapper from './components/ModelsManagerRowWrapper.svelte';
import ModelsManagerRepoRow from '#lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte';
import type { ModelQuantGroup } from '#lib/components/app/models/ModelsManager/utils.ts';
+import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts';
import { ModelGroupKind, ServerRole } from '#lib/enums/index.ts';
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
+import { serversStore } from '#lib/stores/servers.svelte.ts';
import type { ModelOption } from '#lib/types/models.js';
import { beforeEach, describe, expect, it } from 'vitest';
import { userEvent } from 'vitest/browser';
@@ -15,7 +17,8 @@ const option: ModelOption = {
capabilities: [],
id: 'org/Qwen3-8B:Q4_K_M',
model: 'org/Qwen3-8B:Q4_K_M',
- name: 'Qwen3-8B'
+ name: 'Qwen3-8B',
+ serverId: LOCAL_BACKEND_ID
};
const entry: ModelQuantGroup = {
base: option,
@@ -34,7 +37,9 @@ describe('manager model row', () => {
beforeEach(() => {
selected = [];
- // the row only offers a load control on a router server
+ // the row resolves the server that serves it, and only offers a load control on
+ // a router server
+ serversStore.initialize();
serverStore.role = ServerRole.ROUTER;
});
diff --git a/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts b/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts
index 1bcc556a7ba3..40f7124311dd 100644
--- a/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts
+++ b/ui/apps/chat/tests/client/models-manager-utils.svelte.test.ts
@@ -15,7 +15,7 @@ import type { ApiModelDataEntry, ModelOption } from '#lib/types/index.ts';
import { beforeEach, describe, expect, it } from 'vitest';
function option(model: string, id = model): ModelOption {
- return { capabilities: [], id, model, name: model };
+ return { capabilities: [], id, model, name: model, serverId: LOCAL_BACKEND_ID };
}
function routerEntry(id: string, status: ServerModelStatus): ApiModelDataEntry {
@@ -43,8 +43,8 @@ describe('groupModelQuants', () => {
]);
expect(groups.map((group) => [group.key, group.kind, group.quants.length])).toEqual([
- ['org/Qwen3-8B', ModelGroupKind.QUANTS, 2],
- ['org/Llama-3-8B', ModelGroupKind.QUANTS, 1]
+ [`${LOCAL_BACKEND_ID}::org/Qwen3-8B`, ModelGroupKind.QUANTS, 2],
+ [`${LOCAL_BACKEND_ID}::org/Llama-3-8B`, ModelGroupKind.QUANTS, 1]
]);
});
@@ -61,8 +61,8 @@ describe('groupModelQuants', () => {
]);
expect(groups.map((group) => [group.key, group.kind])).toEqual([
- ['org/Qwen3-8B::first', ModelGroupKind.VARIANTS],
- ['org/Qwen3-8B::second', ModelGroupKind.VARIANTS]
+ [`${LOCAL_BACKEND_ID}::org/Qwen3-8B::first`, ModelGroupKind.VARIANTS],
+ [`${LOCAL_BACKEND_ID}::org/Qwen3-8B::second`, ModelGroupKind.VARIANTS]
]);
});
});
@@ -102,9 +102,9 @@ describe('downloadGroups', () => {
// the sub-lines carry the sidecars, so the inline badges stay empty
expect(groups[0].base).toMatchObject({
- backendId: LOCAL_BACKEND_ID,
draftSidecars: [],
- model: 'org/Qwen3-8B:Q4_K_M'
+ model: 'org/Qwen3-8B:Q4_K_M',
+ serverId: LOCAL_BACKEND_ID
});
expect(groups[0].sidecarBadges).toEqual([
{ kind: ModelDraftSidecar.MTP, quant: 'Q8_0', repo: 'org/Qwen3-8B' }
diff --git a/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts b/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts
index a6785edc231a..fb9dfacb79fd 100644
--- a/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts
+++ b/ui/apps/chat/tests/client/models-selector-hidden.svelte.test.ts
@@ -3,6 +3,7 @@
// can be hidden and selected at the same time.
import ModelsSelectorHookHarness from './components/ModelsSelectorHookHarness.svelte';
+import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts';
import { ServerModelStatus, ServerRole } from '#lib/enums/index.ts';
import { serverStore } from '#lib/stores/llama-server.svelte.ts';
import { modelsStore } from '#lib/stores/models/index.svelte.ts';
@@ -11,7 +12,7 @@ import { beforeEach, describe, expect, it } from 'vitest';
import { render } from 'vitest-browser-svelte';
function option(model: string): ModelOption {
- return { capabilities: [], id: model, model, name: model };
+ return { capabilities: [], id: model, model, name: model, serverId: LOCAL_BACKEND_ID };
}
function routerEntry(id: string, status: ServerModelStatus): ApiModelDataEntry {
diff --git a/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts b/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts
index 2c1d61284d07..24b70a0a6789 100644
--- a/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts
+++ b/ui/apps/chat/tests/client/models-selector-mobile.svelte.test.ts
@@ -34,13 +34,14 @@ describe('model picker drawer on a phone', () => {
vi.spyOn(ModelsService, 'list').mockResolvedValue({
data: [{ id: option.model }, { id: other.model }]
} as never);
- modelsStore.models = [option, other];
+ // the models read through the servers data layer on this layer of the stack
+ modelsStore.activeModels = [option, other];
await modelsStore.selectModelById(option.id);
await page.viewport(PHONE.width, PHONE.height);
});
afterEach(async () => {
- modelsStore.models = [];
+ modelsStore.activeModels = [];
vi.restoreAllMocks();
await page.viewport(DESKTOP.width, DESKTOP.height);
});
diff --git a/ui/apps/chat/tests/client/qr-code.svelte.test.ts b/ui/apps/chat/tests/client/qr-code.svelte.test.ts
new file mode 100644
index 000000000000..91756b180e8d
--- /dev/null
+++ b/ui/apps/chat/tests/client/qr-code.svelte.test.ts
@@ -0,0 +1,56 @@
+// The two ends of the code handoff, exercised against each other: the QR the
+// host screen draws is read back by the same native detector the phone's camera
+// scanner uses. A change to the payload or to the rendering shows up here, and
+// only here - everything else about the code is a unit test.
+
+import { CODE_CHARS, CODE_LENGTHS } from '#lib/constants/index.ts';
+import { qrCodeDataUrl } from '#lib/utils/qr-code.ts';
+import { supportsQrScan } from '#lib/utils/qr-scan.ts';
+import { expect, it } from 'vitest';
+
+/** A code of the right length, drawn from the alphabet llama-server generates from. */
+const CODE = Array.from(
+ { length: CODE_LENGTHS.SHARE },
+ (_, index) => CODE_CHARS[index % CODE_CHARS.length]
+).join('');
+/** Browsers that ship the detector the scanner uses; the rest skip the round trip. */
+const canReadCodes = supportsQrScan();
+
+interface BarcodeDetectorLike {
+ detect(source: CanvasImageSource): Promise>;
+}
+
+/** The payload the scanner reports for a rendered code, as a phone would see it. */
+async function readBack(url: string): Promise {
+ const image = new Image();
+
+ image.src = url;
+ await image.decode();
+
+ const canvas = document.createElement('canvas');
+
+ canvas.width = 256;
+ canvas.height = 256;
+
+ const context = canvas.getContext('2d')!;
+
+ context.fillStyle = '#ffffff';
+ context.fillRect(0, 0, canvas.width, canvas.height);
+ context.drawImage(image, 0, 0, canvas.width, canvas.height);
+
+ const Detector = (
+ window as unknown as {
+ BarcodeDetector: new (options: { formats: string[] }) => BarcodeDetectorLike;
+ }
+ ).BarcodeDetector;
+ const [found] = await new Detector({ formats: ['qr_code'] }).detect(canvas);
+
+ return found?.rawValue ?? null;
+}
+
+it.skipIf(!canReadCodes)('reads back the code the host screen shows', async () => {
+ const url = qrCodeDataUrl(CODE);
+
+ expect(url).not.toBeNull();
+ expect(await readBack(url!)).toBe(CODE);
+});
diff --git a/ui/apps/chat/tests/client/remote-servers.svelte.test.ts b/ui/apps/chat/tests/client/remote-servers.svelte.test.ts
new file mode 100644
index 000000000000..523cf290d0dc
--- /dev/null
+++ b/ui/apps/chat/tests/client/remote-servers.svelte.test.ts
@@ -0,0 +1,416 @@
+// Guards the transport rule the whole remote-server feature leans on: a request
+// addressed at a server's tunnel origin is answered by that server's tunnel or
+// refused out loud - it never quietly reaches the server serving this UI. The
+// card is checked here too: what it says about a channel is what the user acts on.
+
+import ServerCardWrapper from './components/ServerCardWrapper.svelte';
+import {
+ API_MODELS,
+ CODE_CHARS,
+ CODE_LENGTHS,
+ LOCAL_BACKEND_ID,
+ SETTINGS_KEYS,
+ WEBRTC_TIMEOUTS
+} from '#lib/constants/index.ts';
+import { TunnelStatus } from '#lib/enums/index.ts';
+import {
+ modelsStore,
+ remoteAccessStore,
+ serversStore,
+ serverStore,
+ settingsStore
+} from '#lib/stores/index.ts';
+import type { ApiLlamaCppServerProps } from '#lib/types/index.ts';
+import { createRemoteServer, tunnelOrigin } from '#lib/utils/servers.ts';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { render } from 'vitest-browser-svelte';
+
+const { FakeTunnel } = vi.hoisted(() => {
+ /**
+ * Stands in for the transport: a handshake the test settles by hand, so an
+ * attempt the user gave up on can be examined without a live tracker.
+ */
+ class FakeTunnel {
+ disconnected = false;
+ static instances: FakeTunnel[] = [];
+ onConnected: (() => void) | undefined;
+ settle: (() => void) | undefined;
+
+ get isConnected(): boolean {
+ return !this.disconnected;
+ }
+
+ connect(): Promise {
+ return new Promise((resolve) => {
+ this.settle = resolve;
+ });
+ }
+
+ constructor(_room: string, _pass: string, callbacks: { onConnected?: () => void } = {}) {
+ this.onConnected = callbacks.onConnected;
+ FakeTunnel.instances.push(this);
+ }
+
+ disconnect(): void {
+ this.disconnected = true;
+ }
+ }
+
+ return { FakeTunnel };
+});
+
+vi.mock('#lib/utils/webrtc-tunnel.ts', () => ({ ClientTunnel: FakeTunnel }));
+
+/** A code of the right length, drawn from the alphabet llama-server generates from. */
+const CODE = Array.from(
+ { length: CODE_LENGTHS.SHARE },
+ (_, index) => CODE_CHARS[index % CODE_CHARS.length]
+).join('');
+const originalFetch = window.fetch;
+
+let passthrough: string[] = [];
+
+/**
+ * Register one remote server, swap `window.fetch` for a recorder, and install
+ * the tunnel transport over it. Returns the recorded requests.
+ */
+function withTransport(url: string): Promise {
+ settingsStore.updateConfig(
+ SETTINGS_KEYS.SERVERS,
+ JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }])
+ );
+ serversStore.initialize();
+
+ passthrough = [];
+
+ window.fetch = ((input: RequestInfo | URL) => {
+ passthrough.push(String(input instanceof Request ? input.url : input));
+
+ return Promise.resolve(new Response('ok', { status: 200 }));
+ }) as typeof window.fetch;
+
+ remoteAccessStore.installInterceptor();
+
+ return fetch(url);
+}
+
+// pairing is an experimental opt-in; these tests exercise it switched on
+beforeEach(() => {
+ settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, true);
+});
+
+afterEach(() => {
+ remoteAccessStore.uninstallInterceptor();
+ window.fetch = originalFetch;
+ settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, false);
+});
+
+it('starts a paired server idle', () => {
+ settingsStore.updateConfig(
+ SETTINGS_KEYS.SERVERS,
+ JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }])
+ );
+ serversStore.initialize();
+
+ expect(remoteAccessStore.statusOf('r1')).toBe(TunnelStatus.IDLE);
+ expect(remoteAccessStore.isConnected('r1')).toBe(false);
+ expect(remoteAccessStore.isConnecting('r1')).toBe(false);
+});
+
+it('refuses a server it has no tunnel for', async () => {
+ const request = withTransport(`${tunnelOrigin('r1')}/props`);
+
+ await expect(request).rejects.toThrow(/is not connected/);
+
+ // the point of the whole exercise: nothing leaked to the serving origin
+ expect(passthrough).toEqual([]);
+});
+
+it('refuses a tunnel origin no server owns', async () => {
+ const request = withTransport('https://ghost.remote.llamacpp.invalid/v1/models');
+
+ await expect(request).rejects.toThrow(/No remote server is registered/);
+ expect(passthrough).toEqual([]);
+});
+
+it('leaves a real URL to the browser', async () => {
+ await withTransport('https://example.com/v1/models');
+
+ expect(passthrough).toEqual(['https://example.com/v1/models']);
+});
+
+it('leaves the page origin to the browser', async () => {
+ await withTransport('/props');
+
+ expect(passthrough).toEqual(['/props']);
+});
+
+/** The remote server the card tests below are about. */
+const REMOTE = createRemoteServer(CODE, 'r1', 'Home');
+
+/** A tunnel status the card reads back for `r1`, without a transport behind it. */
+function withTunnel(tunnel: TunnelStatus, error = '') {
+ remoteAccessStore.status['r1'] = tunnel;
+ remoteAccessStore.errorMessage['r1'] = error;
+}
+
+describe('server card', () => {
+ beforeEach(() => {
+ // the local card only reports the host of the server the app opens with
+ serversStore.setActive(LOCAL_BACKEND_ID);
+ });
+
+ afterEach(() => {
+ delete remoteAccessStore.status['r1'];
+ delete remoteAccessStore.errorMessage['r1'];
+ serverStore.props = null;
+ });
+
+ it('renders the host this machine runs, and pairs with its code', async () => {
+ serverStore.props = { connect_code: CODE } as ApiLlamaCppServerProps;
+ const paired: string[] = [];
+ const screen = render(ServerCardWrapper, {
+ isLocal: true,
+ onPair: (code: string) => paired.push(code),
+ server: serversStore.local
+ });
+
+ await expect.element(screen.getByText('Remote access: on')).toBeVisible();
+ await expect
+ .element(screen.getByText('This server can be reached through its access code.'))
+ .toBeVisible();
+
+ await screen.getByRole('button', { name: 'Pair' }).click();
+
+ expect(paired).toEqual([CODE]);
+ });
+
+ it('renders no host side for a machine without one', async () => {
+ serverStore.props = {} as ApiLlamaCppServerProps;
+
+ const screen = render(ServerCardWrapper, { isLocal: true, server: serversStore.local });
+
+ await expect.element(screen.getByText('Remote access: on')).not.toBeInTheDocument();
+ });
+
+ it('offers the host code as a QR the other device can scan', async () => {
+ serverStore.props = { connect_code: CODE } as ApiLlamaCppServerProps;
+
+ const screen = render(ServerCardWrapper, { isLocal: true, server: serversStore.local });
+
+ await expect.element(screen.getByText('Remote access: on')).toBeVisible();
+
+ // the QR is behind its own control: the card stays compact until it is asked for
+ expect(document.querySelector('img[alt="Access code QR code"]')).toBeNull();
+
+ await screen.getByRole('button', { name: 'Show the access code as a QR code' }).click();
+
+ const qr = screen.getByAltText('Access code QR code');
+
+ await expect.element(qr).toBeVisible();
+ expect((qr.element() as HTMLImageElement).src.startsWith('data:image/svg+xml')).toBe(true);
+ });
+
+ it('offers a way out of a handshake that stalls', async () => {
+ withTunnel(TunnelStatus.CONNECTING);
+
+ const screen = render(ServerCardWrapper, { server: REMOTE });
+
+ await expect.element(screen.getByText('Connecting')).toBeVisible();
+ await expect
+ .element(screen.getByRole('button', { name: 'Disconnect this server' }))
+ .toBeVisible();
+ });
+
+ it('reports the reason a channel dropped, not just that it did', async () => {
+ withTunnel(TunnelStatus.ERROR, 'authentication failed: invalid passcode');
+
+ const screen = render(ServerCardWrapper, { server: REMOTE });
+
+ await expect.element(screen.getByText('Channel down')).toBeVisible();
+ await expect.element(screen.getByText('authentication failed: invalid passcode')).toBeVisible();
+ await expect
+ .element(screen.getByRole('button', { name: 'Reconnect to the host' }))
+ .toBeVisible();
+ });
+
+ it('keeps an idle server a choice, with the code in reach', async () => {
+ const writeText = vi.spyOn(navigator.clipboard, 'writeText').mockResolvedValue();
+ const screen = render(ServerCardWrapper, { server: REMOTE });
+
+ await expect.element(screen.getByText('Not connected')).toBeVisible();
+ await expect
+ .element(screen.getByText('Paste the code the host prints, or press Connect.'))
+ .toBeVisible();
+
+ await screen.getByRole('button', { name: 'Copy the access code' }).click();
+
+ expect(writeText).toHaveBeenCalledWith(CODE);
+
+ await expect.element(screen.getByText('Not connected')).toBeVisible();
+ });
+
+ it('reveals the code a server stores', async () => {
+ const screen = render(ServerCardWrapper, { server: REMOTE });
+
+ await screen.getByTitle('Show the access code').click();
+
+ await expect.element(screen.getByText(CODE)).toBeVisible();
+ });
+});
+
+describe('the status feed of a tunneled router', () => {
+ const calls: string[] = [];
+
+ beforeEach(() => {
+ calls.length = 0;
+ settingsStore.updateConfig(
+ SETTINGS_KEYS.SERVERS,
+ JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }])
+ );
+ serversStore.initialize();
+ vi.spyOn(globalThis, 'fetch').mockImplementation(async (input: RequestInfo | URL) => {
+ calls.push(String(input instanceof Request ? input.url : input));
+
+ return new Response(': ping\n\n', { status: 200 });
+ });
+ });
+
+ afterEach(() => {
+ modelsStore.status.subscribeServer(null);
+ vi.restoreAllMocks();
+ });
+
+ it('reads the feed at the origin of the server it was given', async () => {
+ modelsStore.status.subscribeServer('r1');
+
+ await vi.waitFor(() => expect(calls.length).toBeGreaterThan(0));
+
+ // the tunnel carries the feed like any other request: the server's own
+ // origin, never the one serving this UI
+ expect(calls[0]).toBe(`${tunnelOrigin('r1')}${API_MODELS.SSE}`);
+ });
+
+ it('opens one reader per server', async () => {
+ modelsStore.status.subscribeServer('r1');
+ modelsStore.status.subscribeServer('r1');
+
+ await vi.waitFor(() => expect(calls.length).toBe(1));
+
+ expect(calls.length).toBe(1);
+ });
+
+ it('opens nothing for no server', async () => {
+ modelsStore.status.subscribeServer(null);
+
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ expect(calls).toEqual([]);
+ });
+});
+
+describe('channels lost to a backgrounded tab', () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ settingsStore.updateConfig(
+ SETTINGS_KEYS.SERVERS,
+ JSON.stringify([
+ { code: CODE, id: 'r1', kind: 'remote', name: 'Home' },
+ { code: CODE, id: 'r2', kind: 'remote', name: 'Office' }
+ ])
+ );
+ serversStore.initialize();
+ // one channel was open before the tab went away, the other was never asked for
+ withTunnel(TunnelStatus.ERROR, 'Disconnected from the host');
+ remoteAccessStore.status['r2'] = TunnelStatus.IDLE;
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ delete remoteAccessStore.status['r1'];
+ delete remoteAccessStore.errorMessage['r1'];
+ delete remoteAccessStore.status['r2'];
+ });
+
+ it('asks again, and only for a channel that was wanted', async () => {
+ const connect = vi.spyOn(remoteAccessStore, 'connect').mockResolvedValue();
+
+ remoteAccessStore.recoverChannels();
+
+ // the network is not back the moment the tab is: nothing happens at once
+ expect(connect).not.toHaveBeenCalled();
+
+ await vi.advanceTimersByTimeAsync(WEBRTC_TIMEOUTS.RECOVER_MS);
+
+ expect(connect).toHaveBeenCalledWith('r1');
+ expect(connect).not.toHaveBeenCalledWith('r2');
+ });
+
+ it('opens one handshake per burst of tab switches', async () => {
+ const connect = vi.spyOn(remoteAccessStore, 'connect').mockResolvedValue();
+
+ remoteAccessStore.recoverChannels();
+ await vi.advanceTimersByTimeAsync(WEBRTC_TIMEOUTS.RECOVER_MS / 2);
+ remoteAccessStore.recoverChannels();
+ await vi.advanceTimersByTimeAsync(WEBRTC_TIMEOUTS.RECOVER_MS);
+
+ expect(connect).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('a handshake the user gives up on', () => {
+ beforeEach(() => {
+ FakeTunnel.instances = [];
+ settingsStore.updateConfig(
+ SETTINGS_KEYS.SERVERS,
+ JSON.stringify([{ code: CODE, id: 'r1', kind: 'remote', name: 'Home' }])
+ );
+ serversStore.initialize();
+ });
+
+ afterEach(() => {
+ remoteAccessStore.disconnect('r1');
+ });
+
+ it('leaves the server idle when the answer lands late', async () => {
+ const attempt = remoteAccessStore.connect('r1');
+ const [tunnel] = FakeTunnel.instances;
+
+ remoteAccessStore.disconnect('r1');
+
+ // the host answers after the user stopped waiting
+ tunnel.onConnected?.();
+ tunnel.settle?.();
+
+ await attempt;
+
+ expect(remoteAccessStore.statusOf('r1')).toBe(TunnelStatus.IDLE);
+ expect(remoteAccessStore.isConnected('r1')).toBe(false);
+ expect(tunnel.disconnected).toBe(true);
+ });
+
+ it('starts a fresh handshake when asked again', async () => {
+ const dropped = remoteAccessStore.connect('r1');
+
+ remoteAccessStore.disconnect('r1');
+
+ const kept = remoteAccessStore.connect('r1');
+ const [first, second] = FakeTunnel.instances;
+
+ expect(second).not.toBe(first);
+
+ second.onConnected?.();
+ second.settle?.();
+ await kept;
+
+ expect(remoteAccessStore.isConnected('r1')).toBe(true);
+
+ // the dropped attempt landing now must not close the one that is up
+ first.settle?.();
+ await dropped;
+
+ expect(remoteAccessStore.isConnected('r1')).toBe(true);
+ expect(first.disconnected).toBe(true);
+ });
+});
diff --git a/ui/apps/chat/tests/client/servers-dialog.svelte.test.ts b/ui/apps/chat/tests/client/servers-dialog.svelte.test.ts
new file mode 100644
index 000000000000..56837e8b6425
--- /dev/null
+++ b/ui/apps/chat/tests/client/servers-dialog.svelte.test.ts
@@ -0,0 +1,166 @@
+// Guards the phone's door to servers. The models dialog is desktop-only, so a
+// phone reaches its server list from the sidebar; without that entry there is no
+// way to pair a server, and a phone with no paired server has nothing to pick a
+// model from.
+
+import DialogServersWrapper from './components/DialogServersWrapper.svelte';
+import { ChatScreenServerError, SidebarNavigationActions } from '#lib/components/app/index.ts';
+import {
+ CODE_CHARS,
+ CODE_LENGTHS,
+ SETTINGS_KEYS,
+ SIDEBAR_FOOTER_ITEMS
+} from '#lib/constants/index.ts';
+import { SidebarAction } from '#lib/enums/index.ts';
+import { serversStore, serverStore, settingsStore, uiStore } from '#lib/stores/index.ts';
+import { getAuthHeadersForServer } from '#lib/utils/api-headers.ts';
+import { createRemoteServer, createUrlServer } from '#lib/utils/servers.ts';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { page } from 'vitest/browser';
+import { render } from 'vitest-browser-svelte';
+
+const PHONE = { height: 844, width: 390 };
+const DESKTOP = { height: 900, width: 1280 };
+/** A code of the right length, drawn from the alphabet llama-server generates from. */
+const CODE = Array.from(
+ { length: CODE_LENGTHS.SHARE },
+ (_, index) => CODE_CHARS[index % CODE_CHARS.length]
+).join('');
+
+describe('servers on a phone', () => {
+ const opened: boolean[] = [];
+
+ beforeEach(async () => {
+ opened.length = 0;
+ settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, true);
+ await page.viewport(PHONE.width, PHONE.height);
+ });
+
+ afterEach(async () => {
+ settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, false);
+ await page.viewport(DESKTOP.width, DESKTOP.height);
+ });
+
+ it('offers the servers list in the sidebar', async () => {
+ // the sidebar's footer carries the entry, and a phone lists it with the top actions
+ const screen = render(SidebarNavigationActions, {
+ class: '',
+ items: SIDEBAR_FOOTER_ITEMS,
+ onAction: (action) => opened.push(action === SidebarAction.SERVERS)
+ });
+ const item = screen.getByRole('button', { name: 'Servers' });
+
+ await expect.element(item).toBeVisible();
+
+ await item.click();
+
+ expect(opened).toEqual([true]);
+ });
+
+ it('opens the list as a dialog of its own', async () => {
+ const screen = render(DialogServersWrapper);
+
+ await expect.element(screen.getByText('Servers')).toBeVisible();
+ await expect.element(screen.getByRole('button', { name: 'Close' })).toBeVisible();
+ });
+
+ it('reaches the pairing form through the add control', async () => {
+ const screen = render(DialogServersWrapper);
+
+ await screen.getByRole('button', { name: 'Add new server' }).click();
+
+ await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible();
+
+ await screen.getByRole('tab', { name: 'Remote server' }).click();
+
+ await expect.element(screen.getByText('Not paired yet')).toBeVisible();
+ });
+
+ it('offers only a URL while pairing is switched off', async () => {
+ settingsStore.updateConfig(SETTINGS_KEYS.ENABLE_REMOTE_PAIRING, false);
+
+ const screen = render(DialogServersWrapper);
+
+ await screen.getByRole('button', { name: 'Add new server' }).click();
+
+ await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible();
+ expect(screen.getByRole('tab', { name: 'Remote server' }).query()).toBeNull();
+ });
+
+ it('opens the form as a bottom drawer, and as a dialog on a desktop', async () => {
+ const screen = render(DialogServersWrapper);
+
+ await screen.getByRole('button', { name: 'Add new server' }).click();
+ await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible();
+
+ const drawer = document.querySelector('[data-slot="drawer-content"]');
+
+ expect(drawer?.getAttribute('data-vaul-drawer-direction')).toBe('bottom');
+
+ await page.viewport(DESKTOP.width, DESKTOP.height);
+
+ expect(document.querySelector('[data-slot="drawer-content"]')).toBeNull();
+ await expect.element(screen.getByRole('heading', { name: 'Add New Server' })).toBeVisible();
+ });
+
+ it('offers the API key of a paired server too', async () => {
+ const screen = render(DialogServersWrapper);
+
+ await screen.getByRole('button', { name: 'Add new server' }).click();
+ await screen.getByRole('tab', { name: 'Remote server' }).click();
+
+ // a llama-server started with --connect can still guard its API with a key,
+ // and the tunnel carries the header through
+ await expect.element(screen.getByLabelText(/API key/)).toBeVisible();
+ });
+});
+
+describe('server credentials', () => {
+ afterEach(() => {
+ settingsStore.updateConfig(SETTINGS_KEYS.SERVERS, '[]');
+ });
+
+ it('keeps the API key of every server it stores', () => {
+ serversStore.addServer(createUrlServer('https://example.com', 'u1', 'Office', 'sk-url'));
+ serversStore.addRemoteServer(CODE, 'Home', 'r1', 'sk-tunnel');
+
+ // the list a request is built from is the parsed one: a key that does not
+ // survive the round trip is a key that never reaches the server
+ const stored = serversStore.external;
+ const url = stored.find((server) => server.id === 'u1');
+ const remote = stored.find((server) => server.id === 'r1');
+
+ expect(url?.apiKey).toBe('sk-url');
+ expect(remote?.apiKey).toBe('sk-tunnel');
+ expect(getAuthHeadersForServer(remote!).Authorization).toBe('Bearer sk-tunnel');
+ });
+
+ it('keeps the code of a remote server', () => {
+ serversStore.addRemoteServer(CODE, 'Home', 'r1');
+
+ expect(serversStore.external.find((server) => server.id === 'r1')?.code).toBe(CODE);
+ expect(createRemoteServer(CODE, 'r2', 'Office').code).toBe(CODE);
+ });
+});
+
+describe('a chat with no server at all', () => {
+ beforeEach(() => {
+ settingsStore.updateConfig(SETTINGS_KEYS.SERVERS, '[]');
+ serverStore.error = 'Connection refused';
+ });
+
+ afterEach(() => {
+ serverStore.error = null;
+ uiStore.serversOpen = false;
+ });
+
+ it('offers pairing as the way out', async () => {
+ const screen = render(ChatScreenServerError);
+
+ await expect.element(screen.getByText('Server unavailable')).toBeVisible();
+
+ await screen.getByRole('button', { name: 'Pair a remote server' }).click();
+
+ expect(uiStore.serversOpen).toBe(true);
+ });
+});
diff --git a/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte b/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte
index 3aa5c6045df4..c960d6b8401c 100644
--- a/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte
+++ b/ui/apps/chat/tests/stories/models/ModelsSelector.stories.svelte
@@ -72,7 +72,8 @@
items: [availableModels[4]],
orgName: 'intel'
}
- ]
+ ],
+ servers: []
};
function handleSelect(modelId: string) {
diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts
index fc573064ccbf..6729dbae4c39 100644
--- a/ui/apps/chat/tests/unit/cli.test.ts
+++ b/ui/apps/chat/tests/unit/cli.test.ts
@@ -1,4 +1,13 @@
-import { buildStatic, createChatServer, DEFAULTS, parseCliArgs } from '../../cli/server.js';
+import {
+ buildStatic,
+ createChatServer,
+ DEFAULTS,
+ parseCliArgs,
+ parseServerUrl,
+ PRESET_FILE,
+ PRESET_META,
+ withPresetMeta
+} from '../../cli/server.js';
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
@@ -14,6 +23,9 @@ function listen(server: http.Server): Promise {
});
}
+/** A built index.html in short: the CLI adds its marker for llama-chat.json before ``. */
+const INDEX = 'Llama Chat';
+
function close(server: http.Server): Promise {
return new Promise((resolve) => server.close(() => resolve()));
}
@@ -25,6 +37,7 @@ describe('llama-chat CLI arguments', () => {
expect(options.host).toBe(DEFAULTS.host);
expect(options.port).toBe(DEFAULTS.port);
expect(options.open).toBe(true);
+ expect(options.servers).toEqual([]);
});
it('reads the documented flags', () => {
@@ -42,8 +55,30 @@ describe('llama-chat CLI arguments', () => {
expect(() => parseCliArgs(['--nope'])).toThrow();
expect(() => parseCliArgs(['--port', 'abc'])).toThrow(/invalid --port/);
expect(() => parseCliArgs(['--port', '70000'])).toThrow(/invalid --port/);
- // it forwards nothing, so there is no llama-server to name
- expect(() => parseCliArgs(['--server', 'http://127.0.0.1:9931'])).toThrow();
+ });
+
+ it('takes --server more than once, for serving and for build alike', () => {
+ expect(
+ parseCliArgs(['--server', 'localhost:9931', '--server', 'https://llm.example.com/api/'])
+ .servers
+ ).toEqual(['http://localhost:9931', 'https://llm.example.com/api']);
+ expect(parseCliArgs(['build', '--server', '127.0.0.1:9931']).servers).toEqual([
+ 'http://127.0.0.1:9931'
+ ]);
+ });
+
+ it('puts the marker for llama-chat.json in the head of index.html once', () => {
+ const once = withPresetMeta(INDEX);
+
+ expect(once).toContain(`${PRESET_META}\n\t`);
+ expect(withPresetMeta(once)).toBe(once);
+ });
+
+ it('refuses a server address the app could not use', () => {
+ expect(() => parseServerUrl('ftp://localhost')).toThrow(/invalid --server/);
+ expect(() => parseServerUrl('http://localhost:9931/?key=1')).toThrow(/invalid --server/);
+ expect(() => parseServerUrl('http://user:pass@localhost:9931')).toThrow(/invalid --server/);
+ expect(() => parseServerUrl('http://')).toThrow(/invalid --server/);
});
it('reads build with its own flags, and keeps serving flags out of it', () => {
@@ -85,7 +120,7 @@ describe('llama-chat build', () => {
dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-'));
work = await mkdtemp(join(tmpdir(), 'llama-chat-out-'));
await mkdir(join(dist, '_app'), { recursive: true });
- await writeFile(join(dist, 'index.html'), '');
+ await writeFile(join(dist, 'index.html'), INDEX);
await writeFile(join(dist, '_app', 'app.js'), 'export {};');
await writeFile(join(dist, '_app', 'app.js.map'), '{}');
});
@@ -103,6 +138,23 @@ describe('llama-chat build', () => {
join('_app', 'app.js'),
'index.html'
]);
+ // without servers there is no llama-chat.json, so nothing points the app at one
+ expect(await readFile(join(out, 'index.html'), 'utf8')).toBe(INDEX);
+ });
+
+ it('writes the servers to llama-chat.json when given', async () => {
+ const out = await buildStatic({
+ distDir: dist,
+ out: join(work, 'with-servers'),
+ servers: ['http://127.0.0.1:9931']
+ });
+
+ expect(JSON.parse(await readFile(join(out, PRESET_FILE), 'utf8'))).toEqual({
+ servers: [{ url: 'http://127.0.0.1:9931' }]
+ });
+ expect(await readFile(join(out, 'index.html'), 'utf8')).toContain(PRESET_META);
+ // the copy in dist keeps no marker
+ expect(await readFile(join(dist, 'index.html'), 'utf8')).toBe(INDEX);
});
it('refuses a folder that is not empty unless forced', async () => {
@@ -114,7 +166,7 @@ describe('llama-chat build', () => {
await expect(buildStatic({ distDir: dist, out })).rejects.toThrow(/not empty/);
await buildStatic({ distDir: dist, force: true, out });
expect(await readFile(join(out, 'keep.txt'), 'utf8')).toBe('mine');
- expect(await readFile(join(out, 'index.html'), 'utf8')).toBe('');
+ expect(await readFile(join(out, 'index.html'), 'utf8')).toBe(INDEX);
});
it('refuses to write into the app files themselves', async () => {
@@ -135,7 +187,7 @@ describe('llama-chat server', () => {
beforeAll(async () => {
dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-'));
await mkdir(join(dist, '_app', 'immutable'), { recursive: true });
- await writeFile(join(dist, 'index.html'), 'Llama Chat');
+ await writeFile(join(dist, 'index.html'), INDEX);
await writeFile(join(dist, 'sw.js'), 'self;');
await writeFile(join(dist, 'service-worker.js'), 'self;');
await writeFile(join(dist, '_app', 'immutable', 'app.abc123.js'), 'export {};');
@@ -157,7 +209,7 @@ describe('llama-chat server', () => {
expect(res.headers.get('cache-control')).toBe('no-cache');
expect(res.headers.get('cross-origin-embedder-policy')).toBe('require-corp');
expect(res.headers.get('cross-origin-opener-policy')).toBe('same-origin');
- expect(await res.text()).toContain('Llama Chat');
+ expect(await res.text()).toContain(PRESET_META);
const again = await fetch(`${chatUrl}/`, {
headers: { 'If-None-Match': res.headers.get('etag') ?? '' }
@@ -181,6 +233,31 @@ describe('llama-chat server', () => {
expect(worker.headers.get('cache-control')).toBe('no-cache');
});
+ it('serves llama-chat.json, revalidated, with no servers when none are given', async () => {
+ const empty = await fetch(`${chatUrl}/${PRESET_FILE}`);
+
+ expect(empty.status).toBe(200);
+ expect(await empty.json()).toEqual({ servers: [] });
+
+ const withServers = createChatServer({
+ distDir: dist,
+ servers: ['http://127.0.0.1:9931', 'https://llm.example.com']
+ });
+ const url = await listen(withServers);
+
+ try {
+ const res = await fetch(`${url}/${PRESET_FILE}`);
+
+ expect(res.status).toBe(200);
+ expect(res.headers.get('cache-control')).toBe('no-cache');
+ expect(await res.json()).toEqual({
+ servers: [{ url: 'http://127.0.0.1:9931' }, { url: 'https://llm.example.com' }]
+ });
+ } finally {
+ await close(withServers);
+ }
+ });
+
it('forwards nothing: API paths are not found and other methods are refused', async () => {
const props = await fetch(`${chatUrl}/props`);
diff --git a/ui/apps/chat/tests/unit/extension-page-route.test.ts b/ui/apps/chat/tests/unit/extension-page-route.test.ts
index 7cb28d3344e1..595387e2285e 100644
--- a/ui/apps/chat/tests/unit/extension-page-route.test.ts
+++ b/ui/apps/chat/tests/unit/extension-page-route.test.ts
@@ -4,11 +4,14 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('$app/env', () => ({ browser: true }));
-vi.mock('#lib/stores/init.ts', () => ({ loadAppRoute: vi.fn(async () => {}) }));
+vi.mock('#lib/stores/init.ts', () => ({
+ loadAppRoute: vi.fn(async () => {}),
+ startExtensions: vi.fn(async () => {})
+}));
import { load } from '../../src/routes/[extensionPage]/+page';
import { extensions } from '#lib/extensions/index.ts';
-import { loadAppRoute } from '#lib/stores/init.ts';
+import { loadAppRoute, startExtensions } from '#lib/stores/init.ts';
import type { Component } from 'svelte';
const Page = (() => {}) as unknown as Component;
@@ -51,8 +54,9 @@ describe('extension page route', () => {
await expect(loadPath('fake-page')).resolves.toBeUndefined();
await expect(loadPath('other-page')).rejects.toMatchObject({ status: 404 });
- // the stores start, and the API key is checked, before the path is looked up
+ // the stores hydrate, the API key is checked and the extensions start before the path is looked up
expect(loadAppRoute).toHaveBeenCalledWith(fetch);
+ expect(startExtensions).toHaveBeenCalled();
await extensions.setEnabled('fake-pages', false);
diff --git a/ui/apps/chat/tests/unit/model-list.test.ts b/ui/apps/chat/tests/unit/model-list.test.ts
index 6a7c18fec146..793faa2ae235 100644
--- a/ui/apps/chat/tests/unit/model-list.test.ts
+++ b/ui/apps/chat/tests/unit/model-list.test.ts
@@ -33,7 +33,8 @@ function groups(): GroupedModelOptions {
available: [
{ items: [row('org/a'), row('org/b'), row('org/c')], orgName: 'org' },
{ items: [row('other/d')], orgName: 'other' }
- ]
+ ],
+ servers: []
};
}
diff --git a/ui/apps/chat/tests/unit/qr-code.test.ts b/ui/apps/chat/tests/unit/qr-code.test.ts
new file mode 100644
index 000000000000..c9b1c91058bb
--- /dev/null
+++ b/ui/apps/chat/tests/unit/qr-code.test.ts
@@ -0,0 +1,66 @@
+// Guards the other end of the code scanner: what the host screen shows must be
+// the payload the camera accepts, and nothing else. A QR that carries a URL, or
+// a half-typed code, scans into a pairing that cannot work.
+
+import { CODE_CHARS, CODE_LENGTHS } from '#lib/constants/index.ts';
+import { qrCodeDataUrl } from '#lib/utils/qr-code.ts';
+import { describe, expect, it } from 'vitest';
+
+/** A code of the right length, drawn from the alphabet llama-server generates from. */
+const CODE = Array.from(
+ { length: CODE_LENGTHS.SHARE },
+ (_, index) => CODE_CHARS[index % CODE_CHARS.length]
+).join('');
+/** The same alphabet, shifted: a different code of the same shape. */
+const OTHER = Array.from(
+ { length: CODE_LENGTHS.SHARE },
+ (_, index) => CODE_CHARS[(index + 7) % CODE_CHARS.length]
+).join('');
+/** Module size the renderer draws with, in SVG units. */
+const PIXEL_SIZE = 4;
+
+function svgOf(url: string | null): string {
+ expect(url).not.toBeNull();
+
+ return decodeURIComponent(url!.split(',')[1]);
+}
+
+describe('access code QR', () => {
+ it('renders the code as an SVG data URL', () => {
+ const svg = svgOf(qrCodeDataUrl(CODE));
+
+ expect(svg.startsWith('