From 80792f7fa795fab25f51d9fd169874348ed407f1 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 28 Sep 2026 15:36:23 +0200 Subject: [PATCH 01/34] ui : add the providers data layer A provider is a server entry with a base url, an optional key, a protocol and the paths its API lives at; the local llama.cpp server is the built-in one. Backends persist in settings and the active one is restored on load. Requests resolve against the active backend, model ids become backend-qualified, and every backend's model list is fetched and cached in the background. The manager's helpers learn to read a model's drafts, context and the provider that serves it. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash --- ui/apps/chat/src/app.d.ts | 2 + .../ModelsManagerModelsTable.svelte | 2 +- .../app/models/ModelsManager/utils.ts | 45 +++- .../src/lib/constants/backend.constants.ts | 66 ++++- .../lib/constants/settings-keys.constants.ts | 2 + .../src/lib/constants/storage.constants.ts | 9 + .../src/lib/constants/stream.constants.ts | 4 + ui/apps/chat/src/lib/enums/model.enums.ts | 3 +- .../chat/src/lib/services/backends.service.ts | 193 ++++++++++++++ ui/apps/chat/src/lib/services/index.ts | 8 + .../chat/src/lib/stores/backends.svelte.ts | 117 +++++++++ .../src/lib/stores/backendsModels.svelte.ts | 103 ++++++++ ui/apps/chat/src/lib/types/api.d.ts | 13 + ui/apps/chat/src/lib/types/backend.d.ts | 98 +++++++ ui/apps/chat/src/lib/types/index.ts | 10 + ui/apps/chat/src/lib/types/models.d.ts | 31 ++- ui/apps/chat/src/lib/utils/api-base.ts | 99 +++++++ ui/apps/chat/src/lib/utils/api-fetch.ts | 19 +- ui/apps/chat/src/lib/utils/api-headers.ts | 30 ++- .../chat/src/lib/utils/api-key-validation.ts | 12 +- ui/apps/chat/src/lib/utils/backend.ts | 248 ++++++++++++++++++ ui/apps/chat/src/lib/utils/cors-proxy.ts | 4 +- ui/apps/chat/src/lib/utils/index.ts | 30 ++- ui/apps/chat/src/lib/utils/model-option-id.ts | 28 ++ ui/apps/chat/src/lib/utils/timings.ts | 66 +++++ 25 files changed, 1196 insertions(+), 46 deletions(-) create mode 100644 ui/apps/chat/src/lib/services/backends.service.ts create mode 100644 ui/apps/chat/src/lib/stores/backends.svelte.ts create mode 100644 ui/apps/chat/src/lib/stores/backendsModels.svelte.ts create mode 100644 ui/apps/chat/src/lib/types/backend.d.ts create mode 100644 ui/apps/chat/src/lib/utils/api-base.ts create mode 100644 ui/apps/chat/src/lib/utils/backend.ts create mode 100644 ui/apps/chat/src/lib/utils/model-option-id.ts create mode 100644 ui/apps/chat/src/lib/utils/timings.ts diff --git a/ui/apps/chat/src/app.d.ts b/ui/apps/chat/src/app.d.ts index 006a37a8cfaa..7b9e60c457f0 100644 --- a/ui/apps/chat/src/app.d.ts +++ b/ui/apps/chat/src/app.d.ts @@ -10,6 +10,7 @@ import type { ApiChatCompletionStreamChunk, ApiChatCompletionToolCall, ApiChatCompletionToolCallDelta, + ApiChatCompletionUsage, ApiChatMessageContentPart, ApiChatMessageData, ApiContextSizeError, @@ -72,6 +73,7 @@ declare global { ApiChatCompletionResponse, ApiChatCompletionStreamChunk, ApiChatCompletionToolCall, + ApiChatCompletionUsage, ApiChatCompletionToolCallDelta, ApiChatMessageData, ApiChatMessageContentPart, diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte index e24f4e91b51b..1f9bd6046b77 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte @@ -95,7 +95,7 @@ /** Repos whose quants are folded away; the rest show them. */ const collapsedQuants = new SvelteSet(); /** Sections that list their models straight, without folding them into families. */ - const FLAT_SECTIONS = new Set([ + const FLAT_SECTIONS = new Set([ ModelsTableGroupKind.DOWNLOADING, ModelsTableGroupKind.FAVORITES, ModelsTableGroupKind.LOADED diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts b/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts index 41aec8530455..a74565e1f19e 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts @@ -8,6 +8,7 @@ import type { ModelOption, ModelSidecarBadge } from '#lib/types/models.js'; +import { getBackend } from '#lib/utils/api-base.ts'; import { detectThinkingSupport, detectToolUseSupport, repoOf } from '#lib/utils/index.ts'; /** One repo of the table, with the rows it ships as. */ @@ -87,10 +88,14 @@ export function findModelOption( /** One collapsible block of the manager's table. */ export interface ModelsTableGroup { + /** Backend the block belongs to, when it is tied to one. */ + backendId?: string | null; /** One entry per repo, its quants hanging off it. */ items: ModelQuantGroup[]; + isLocal?: boolean; key: string; - kind: ModelsTableGroupKind; + /** Manager sections use the kind constants, provider blocks their own kinds. */ + kind: ModelsTableGroupKind | 'compat' | 'provider'; label: string; } @@ -111,7 +116,8 @@ export function configuredContext(option: ModelOption): number | null { /** * Capability a model reports: true or false once its listing or its chat template - * answers, null while the Hub record that carries the template is not read yet. + * answers, null while the Hub record that carries the template is not read yet. A + * listing that declares nothing is not a listing that lacks the capability. */ export function modelCapability(option: ModelOption, capability: ModelCapability): boolean | null { if (option.capabilities.includes(capability)) return true; @@ -128,6 +134,14 @@ export function modelCapability(option: ModelOption, capability: ModelCapability : detectThinkingSupport(template); } +export function servedByLabel(option: ModelOption): string { + const backend = getBackend(option.backendId); + + if (!backend || backend.id === LOCAL_BACKEND_ID) return 'This server'; + + return backend.name; +} + /** * Context a model reports: the provider listing first, then the cached Hub record. * Null while neither of them has answered. @@ -140,12 +154,20 @@ export function modelContextLength(option: ModelOption): number | null { ); } +export function isLocalOption(option: ModelOption): boolean { + return (option.backendId ?? LOCAL_BACKEND_ID) === LOCAL_BACKEND_ID; +} + /** Fold the quants of one repo into a single entry, so the table shows one row per model. */ -export function groupModelQuants(models: ModelOption[]): ModelQuantGroup[] { +export function groupModelQuants(models: ModelOption[], mergeProviders = false): ModelQuantGroup[] { const groups = new Map(); for (const option of models) { - const key = repoOf(option.model); + const repo = repoOf(option.model); + // groups stay within one backend, so the same repo served by two providers + // is not read as two quants of one model. The OAI-compat block asks for the + // opposite: one repo, one row per provider that serves it. + const key = mergeProviders ? repo : `${option.backendId ?? ''}::${repo}`; const group = groups.get(key); if (group) { @@ -161,8 +183,13 @@ export function groupModelQuants(models: ModelOption[]): ModelQuantGroup[] { const kind = groupKind(group.quants); const modelIds = group.quants.map((option) => option.model); - // the very same id twice is not a quant set; keep those rows apart - if (modelIds.length > 1 && modelIds.every((model) => model === modelIds[0])) { + // the very same id twice is not a quant set; keep those rows apart, unless + // the group exists to list the providers that serve it + if ( + kind !== ModelGroupKind.PROVIDERS && + modelIds.length > 1 && + modelIds.every((model) => model === modelIds[0]) + ) { return group.quants.map((option) => ({ ...group, base: option, @@ -176,8 +203,12 @@ export function groupModelQuants(models: ModelOption[]): ModelQuantGroup[] { }); } -/** What a group folds: quants or variants of one repo. */ +/** What a group folds: providers, quants, or variants of one repo. */ function groupKind(quants: ModelOption[]): ModelGroupKind { + const backends = new Set(quants.map((option) => option.backendId ?? '')); + + if (backends.size > 1) return ModelGroupKind.PROVIDERS; + const isQuant = quants.every( (option) => (option.parsedId ?? ModelsService.parseModelId(option.model)).quantization ); diff --git a/ui/apps/chat/src/lib/constants/backend.constants.ts b/ui/apps/chat/src/lib/constants/backend.constants.ts index 57f5c4213220..5ab3ab756822 100644 --- a/ui/apps/chat/src/lib/constants/backend.constants.ts +++ b/ui/apps/chat/src/lib/constants/backend.constants.ts @@ -1,2 +1,66 @@ -/** Marks the built-in local llama.cpp backend. */ +import type { BackendCapabilities, BackendCompat, BackendProtocol } from '#lib/types/index.ts'; + +/** Prefix for generated ids of user-added backends. */ +export const BACKEND_ID_PREFIX = 'backend'; + +/** Protocols a configured backend can speak, in display order. */ +export const BACKEND_PROTOCOLS: readonly BackendProtocol[] = ['llama.cpp', 'openai']; + +/** Chat completions path used when a backend does not override it. */ +export const DEFAULT_BACKEND_CHAT_PATH = '/v1/chat/completions'; + +/** Models listing path used when a backend does not override it. */ +export const DEFAULT_BACKEND_MODELS_PATH = '/v1/models'; + +/** Id of the built-in backend that points at the server serving this UI. */ export const LOCAL_BACKEND_ID = 'local'; + +/** Capabilities of a full llama.cpp server. */ +const LLAMA_CPP_CAPABILITIES: BackendCapabilities = { + corsProxy: true, + loadUnload: true, + props: true, + resumableStreams: true, + router: true, + slots: true, + statusFeed: true, + tools: true +}; +/** Capabilities of a plain OpenAI-compatible endpoint. */ +const COMPATIBLE_CAPABILITIES: BackendCapabilities = { + corsProxy: false, + loadUnload: false, + props: false, + resumableStreams: false, + router: false, + slots: false, + statusFeed: false, + tools: false +}; + +/** Capabilities per backend protocol. */ +export const BACKEND_CAPABILITIES: Record = { + 'llama.cpp': LLAMA_CPP_CAPABILITIES, + openai: COMPATIBLE_CAPABILITIES +}; + +/** Default wire quirks per protocol. */ +export const BACKEND_COMPAT: Record = { + // llama-server reports its own timings, so it needs no usage chunk + 'llama.cpp': { maxTokensField: 'max_tokens', supportsUsageInStreaming: false }, + openai: { maxTokensField: 'max_tokens', supportsUsageInStreaming: true } +}; + +/** + * Fields that may carry a model's context size in an OpenAI-compatible model + * listing. Providers pick their own name, and most report nothing at all. + */ +export const MODEL_CONTEXT_LENGTH_FIELDS = [ + 'context_length', + 'context_window', + 'max_context_length', + 'max_position_embeddings' +] as const; + +/** Favicon extract keyed by domain, for backends with no bundled mark. */ +export const FAVICON_SERVICE_URL = 'https://www.google.com/s2/favicons?domain='; diff --git a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts index dcaed3da880a..cbd7dffce1cf 100644 --- a/ui/apps/chat/src/lib/constants/settings-keys.constants.ts +++ b/ui/apps/chat/src/lib/constants/settings-keys.constants.ts @@ -11,6 +11,7 @@ export const SETTINGS_KEYS = { API_KEY: 'apiKey', AUTO_MIC_ON_EMPTY: 'autoMicOnEmpty', BACKEND_SAMPLING: 'backend_sampling', + BACKENDS: 'backends', COMPACTION_PRESERVED_STEPS: 'compactionPreservedSteps', COMPACTION_PROMPT: 'compactionPrompt', CONVERSATION_TABS: 'conversationTabs', @@ -34,6 +35,7 @@ export const SETTINGS_KEYS = { FULL_HEIGHT_CODE_BLOCKS: 'fullHeightCodeBlocks', GROUP_MODELS_BY_FAMILY: 'groupModelsByFamily', LIVE_MARKDOWN_PREVIEW: 'liveMarkdownPreview', + LOCAL_BACKEND_ENABLED: 'localBackendEnabled', MAX_IMAGE_RESOLUTION: 'maxImageMPixels', MAX_TOKENS: 'max_tokens', MCP_REQUEST_TIMEOUT_SECONDS: 'mcpRequestTimeoutSeconds', diff --git a/ui/apps/chat/src/lib/constants/storage.constants.ts b/ui/apps/chat/src/lib/constants/storage.constants.ts index fa8d6a09dfb9..0290b3511f63 100644 --- a/ui/apps/chat/src/lib/constants/storage.constants.ts +++ b/ui/apps/chat/src/lib/constants/storage.constants.ts @@ -16,6 +16,9 @@ export const DB_APP_NAME_DEPRECATED = 'LlamacppWebui'; export const ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.alwaysAllowedTools`; +/** Id of the backend the selector and new requests target, restored on page load. */ +export const ACTIVE_BACKEND_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.activeBackend`; + export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`; export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`; @@ -32,6 +35,12 @@ export const FAVORITE_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.favoriteMod /** Open state the user set for a model list section or one of its families, by id. */ export const MODEL_GROUP_OPEN_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.modelGroupOpen`; +/** Per-model load and inference overrides, keyed by backend-qualified model id. */ +export const MODEL_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.modelOverrides`; + +/** Model the user picked last, kept across reloads. Stores `{ id, model }`. */ +export const SELECTED_MODEL_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.selectedModel`; + /** Recently used model ids, most recent first, backend-qualified. */ export const RECENT_MODELS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.recentModels`; diff --git a/ui/apps/chat/src/lib/constants/stream.constants.ts b/ui/apps/chat/src/lib/constants/stream.constants.ts index 64f67243c266..d68ffbe4bf75 100644 --- a/ui/apps/chat/src/lib/constants/stream.constants.ts +++ b/ui/apps/chat/src/lib/constants/stream.constants.ts @@ -2,6 +2,10 @@ // while the tab was hidden. covers brief background pauses without thrashing live streams export const STREAM_VISIBILITY_KICK_MS = 3000; +// minimum gap between synthesized live timing updates for backends that do not +// stream their own, keeps the per-chunk state updates cheap +export const STREAM_LIVE_TIMINGS_INTERVAL_MS = 500; + // separator joining a conversation id and its per-model stream identity // suffix (conv::model) used by the server side replay buffer export const CONVERSATION_ID_SEPARATOR = '::'; diff --git a/ui/apps/chat/src/lib/enums/model.enums.ts b/ui/apps/chat/src/lib/enums/model.enums.ts index e3a4d7b4f4f8..c7cdeece81d4 100644 --- a/ui/apps/chat/src/lib/enums/model.enums.ts +++ b/ui/apps/chat/src/lib/enums/model.enums.ts @@ -40,8 +40,9 @@ export enum ModelDownloadConfirmAction { DELETE = 'delete' } -/** What a table row group folds: the quants of one repo, or its variants. */ +/** What a table row group folds: the providers of one repo, its quants, or its variants. */ export enum ModelGroupKind { + PROVIDERS = 'providers', QUANTS = 'quants', VARIANTS = 'variants' } diff --git a/ui/apps/chat/src/lib/services/backends.service.ts b/ui/apps/chat/src/lib/services/backends.service.ts new file mode 100644 index 000000000000..524bbe840310 --- /dev/null +++ b/ui/apps/chat/src/lib/services/backends.service.ts @@ -0,0 +1,193 @@ +/** + * BackendsService - Stateless backend connectivity checks and model listing + * + * Probes a backend's models endpoint to validate its URL and credentials, and + * normalizes the response into the UI model shape. No reactive state; + * consumed by the backends settings UI and the per-backend model cache. + */ + +import { API_MODELS, LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; +import { ModelsService } from '#lib/services/models.service.ts'; +import type { + ApiModelsListResponse, + Backend, + BackendProtocol, + ModelOption +} from '#lib/types/index.ts'; +import { isAbortError } from '#lib/utils/abort.ts'; +import { apiUrl } from '#lib/utils/api-base.ts'; +import { getAuthHeadersForBackend } from '#lib/utils/api-headers.ts'; +import { backendModelsUrl, readModelContextLength } from '#lib/utils/backend.ts'; + +/** Models returned by a backend, plus the failure detail when the call fails. */ +export interface BackendModelsResult { + error?: string; + models: ModelOption[]; + ok: boolean; + status: number | null; + /** Untouched list payload of the local backend, kept so the router rows and their load statuses can be rebuilt without asking again. */ + raw?: ApiModelsListResponse; +} + +/** What probing a backend's endpoint said about it. */ +export interface BackendProbe { + /** The endpoint refused the request for want of a key, so it wants one. */ + authRequired: boolean; + protocol: BackendProtocol; +} + +/** Outcome of a backend connectivity check. */ +export interface BackendTestResult { + error?: string; + modelCount?: number; + ok: boolean; + status: number | null; +} + +export class BackendsService { + /** + * List the models a backend exposes on its models endpoint. + * + * @param backend - Backend to query. Does not need to be registered yet. + * @param signal - Optional abort signal for a cancelled request. + */ + static async detectProtocol(backend: Backend): Promise { + const base = backend.baseUrl.trim().replace(/\/+$/, ''); + + if (!base) return { authRequired: false, protocol: 'openai' }; + + try { + // llama-server answers /props with its build and generation defaults; a + // plain OpenAI-compatible endpoint answers 404 there, or not at all + const response = await fetch(`${base}/props`, { + headers: getAuthHeadersForBackend(backend), + signal: AbortSignal.timeout(5000) + }); + + // a llama-server behind a key refuses before it says anything else, while + // an OpenAI-compatible endpoint has no /props to guard in the first place + if (response.status === 401) { + return { authRequired: true, protocol: 'llama.cpp' }; + } + + if (!response.ok) return { authRequired: false, protocol: 'openai' }; + + const body = (await response.json()) as Record; + const isLlamaCpp = + 'default_generation_settings' in body || 'build_info' in body || body.role === 'router'; + + return { + authRequired: false, + protocol: isLlamaCpp ? 'llama.cpp' : 'openai' + }; + } catch { + return { authRequired: false, protocol: 'openai' }; + } + } + + static async listModels(backend: Backend, signal?: AbortSignal): Promise { + // the local backend has no base URL; its models endpoint is base relative + const url = backend.baseUrl.trim() + ? backendModelsUrl(backend) + : apiUrl(API_MODELS.LIST, LOCAL_BACKEND_ID); + + if (!backend.baseUrl.trim() && backend.id !== LOCAL_BACKEND_ID) { + return { error: 'Backend URL is required', models: [], ok: false, status: null }; + } + + try { + const response = await fetch(url, { + headers: getAuthHeadersForBackend(backend), + signal + }); + + if (!response.ok) { + return { + error: await describeFailure(response), + models: [], + ok: false, + status: response.status + }; + } + + const body = (await response.json()) as { data?: unknown }; + const entries = Array.isArray(body?.data) ? body.data : []; + const models = entries.flatMap((entry) => normalizeBackendModel(entry)); + // the local rows carry load status, external ones carry the context size + const raw = body as ApiModelsListResponse; + + return { models, ok: true, raw, status: response.status }; + } catch (error) { + if (isAbortError(error)) { + return { models: [], ok: false, status: null }; + } + + return { + error: error instanceof Error ? error.message : String(error), + models: [], + ok: false, + status: null + }; + } + } + + /** + * Check that a backend answers on its models endpoint. + * + * @param backend - Backend to probe. Does not need to be registered yet. + * @param signal - Optional abort signal for a cancelled test. + */ + static async test(backend: Backend, signal?: AbortSignal): Promise { + const result = await BackendsService.listModels(backend, signal); + + return { + error: result.error, + modelCount: result.models.length, + ok: result.ok, + status: result.status + }; + } +} + +/** Build a human-readable message from a non-OK response. */ +async function describeFailure(response: Response): Promise { + const status = `${response.status} ${response.statusText}`.trim(); + + try { + const body = (await response.json()) as { error?: { message?: string }; message?: string }; + const message = body?.error?.message ?? body?.message; + + if (message) return `${status}: ${message}`; + } catch { + // non-JSON error body, fall back to the status line + } + + return status; +} + +/** + * Normalize one entry of an OpenAI-compatible `/v1/models` response. External + * backends only guarantee an id, so that doubles as the display name. + */ +function normalizeBackendModel(entry: unknown): ModelOption[] { + if (!entry || typeof entry !== 'object') return []; + + const raw = entry as Record; + const id = typeof raw.id === 'string' ? raw.id.trim() : ''; + + if (!id) return []; + + // a llama-compat server lists its projector and draft sidecars as models too + if (ModelsService.isSidecarEntry(id)) return []; + + return [ + { + capabilities: [], + contextLength: readModelContextLength(raw), + id, + model: id, + name: id, + status: raw.status as ApiModelDataEntry['status'] + } + ]; +} diff --git a/ui/apps/chat/src/lib/services/index.ts b/ui/apps/chat/src/lib/services/index.ts index 04a63ad898de..e8ae0c6810fe 100644 --- a/ui/apps/chat/src/lib/services/index.ts +++ b/ui/apps/chat/src/lib/services/index.ts @@ -19,6 +19,14 @@ * */ +/** + * **BackendsService** - Backend connectivity checks + * + * Probes an external backend's models endpoint to validate its URL and + * credentials before it is saved. Stateless. + */ +export { BackendsService } from './backends.service'; + /** * **ChatService** - Chat Completions API communication layer * diff --git a/ui/apps/chat/src/lib/stores/backends.svelte.ts b/ui/apps/chat/src/lib/stores/backends.svelte.ts new file mode 100644 index 000000000000..6e73162f8637 --- /dev/null +++ b/ui/apps/chat/src/lib/stores/backends.svelte.ts @@ -0,0 +1,117 @@ +/** + * backendsStore - API endpoints the UI can talk to. + * + * The built-in local backend is the llama-server serving this UI. External + * backends are user-configured endpoints persisted in settings. The store + * registers the resolved list with the api-base registry, which services use + * to build request URLs. + */ + +import { + ACTIVE_BACKEND_LOCALSTORAGE_KEY, + LOCAL_BACKEND_ID, + SETTINGS_KEYS +} from '#lib/constants/index.ts'; +import { serverStore } from '#lib/stores/llama-server.svelte.ts'; +import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import type { Backend } from '#lib/types/index.ts'; +import { setBackendsResolver } from '#lib/utils/api-base.ts'; +import { createLocalBackend, parseBackendsSettings } from '#lib/utils/backend.ts'; +import { browser } from '$app/env'; + +function loadActiveBackendId(): string { + if (!browser) return LOCAL_BACKEND_ID; + + try { + return localStorage.getItem(ACTIVE_BACKEND_LOCALSTORAGE_KEY) ?? LOCAL_BACKEND_ID; + } catch { + return LOCAL_BACKEND_ID; + } +} + +function persistActiveBackendId(backendId: string): void { + if (!browser) return; + + try { + localStorage.setItem(ACTIVE_BACKEND_LOCALSTORAGE_KEY, backendId); + } catch { + /* ignore */ + } +} + +class BackendsStore { + activeId = $state(loadActiveBackendId()); + + get active(): Backend { + const active = this.enabled.find((backend) => backend.id === this.activeId); + + return active ?? this.enabled[0] ?? this.local; + } + + get enabled(): Backend[] { + return this.list.filter((backend) => backend.enabled && !this.isMissingLocal(backend)); + } + + get external(): Backend[] { + return parseBackendsSettings(settingsStore.config[SETTINGS_KEYS.BACKENDS]); + } + + get list(): Backend[] { + return [this.local, ...this.external]; + } + + get local(): Backend { + return createLocalBackend( + settingsStore.config.apiKey?.toString().trim() || undefined, + settingsStore.config[SETTINGS_KEYS.LOCAL_BACKEND_ENABLED] !== false + ); + } + + addBackend(backend: Backend): void { + this.saveExternal([...this.external, backend]); + } + + initialize(): void { + if (!browser) return; + + setBackendsResolver(() => ({ activeId: this.active.id, backends: this.list })); + } + + removeBackend(backendId: string): void { + this.saveExternal(this.external.filter((backend) => backend.id !== backendId)); + + if (this.activeId === backendId) { + this.setActive(LOCAL_BACKEND_ID); + } + } + + setActive(backendId: string): void { + const id = this.list.some((backend) => backend.id === backendId) ? backendId : LOCAL_BACKEND_ID; + + this.activeId = id; + persistActiveBackendId(id); + } + + setLocalEnabled(enabled: boolean): void { + settingsStore.updateConfig(SETTINGS_KEYS.LOCAL_BACKEND_ENABLED, enabled); + } + + updateBackend(backendId: string, updates: Partial): void { + this.saveExternal( + this.external.map((backend) => + backend.id === backendId ? { ...backend, ...updates } : backend + ) + ); + } + + /** The built-in backend counts only when a local server answered this session. */ + private isMissingLocal(backend: Backend): boolean { + return backend.id === LOCAL_BACKEND_ID && serverStore.localServerMissing; + } + + private saveExternal(backends: Backend[]): void { + settingsStore.updateConfig(SETTINGS_KEYS.BACKENDS, JSON.stringify(backends)); + } +} + +export const backendsStore = new BackendsStore(); diff --git a/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts b/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts new file mode 100644 index 000000000000..e13b24860057 --- /dev/null +++ b/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts @@ -0,0 +1,103 @@ +/** + * backendsModelsStore - Per-backend model catalog cache. + * + * Prefetches every enabled backend's model list, so switching backends is + * instant and the switcher can show load state. The active backend's list + * still lives in modelsStore, which owns selection and chat wiring; this + * cache is the prefetch layer the switches start from. + */ + +import { BackendsService } from '#lib/services/backends.service.ts'; +import { ModelsService } from '#lib/services/models.service.ts'; +import { backendsStore } from '#lib/stores/backends.svelte.ts'; +import type { Backend } from '#lib/types/index.ts'; +import type { ApiModelsListResponse } from '#lib/types/index.ts'; +import type { ModelSidecarFile } from '#lib/types/models.js'; +import type { ModelOption } from '#lib/types/models.js'; + +export interface BackendModelsState { + /** Draft sidecars the listing carries, keyed by the repo they belong to. */ + drafts?: Record; + error: string | null; + loaded: boolean; + loading: boolean; + models: ModelOption[]; + /** Untouched list payload, kept for the local backend so its router rows survive a tab switch. */ + raw?: ApiModelsListResponse; +} + +const EMPTY_STATE: BackendModelsState = { + error: null, + loaded: false, + loading: false, + models: [] +}; + +class BackendsModelsStore { + private states = $state>({}); + + clear(backendId: string): void { + delete this.states[backendId]; + } + + /** + * Load a backend's models once. + */ + async ensureLoaded(backendId: string): Promise { + const backend = backendsStore.enabled.find((candidate) => candidate.id === backendId); + + if (!backend) return; + + const state = this.states[backendId]; + + if (state?.loaded || state?.loading) return; + + this.states[backendId] = { error: null, loaded: false, loading: true, models: [] }; + await this.fetch(backend); + } + + get(backendId: string): BackendModelsState { + return this.states[backendId] ?? EMPTY_STATE; + } + + /** Prefetch every enabled backend's model list. */ + async loadAll(): Promise { + const enabled = backendsStore.enabled; + const ids = new Set(enabled.map((backend) => backend.id)); + + for (const id of Object.keys(this.states)) { + if (!ids.has(id)) { + delete this.states[id]; + } + } + + await Promise.all(enabled.map((backend) => this.ensureLoaded(backend.id))); + } + + /** + * Ask a backend for its list again, keeping what is already known. Used while + * a remote load settles, since its status never reaches the local feed. + */ + async refresh(backendId: string): Promise { + const backend = backendsStore.enabled.find((candidate) => candidate.id === backendId); + + if (!backend) return; + + await this.fetch(backend); + } + + private async fetch(backend: Backend): Promise { + const result = await BackendsService.listModels(backend); + + this.states[backend.id] = { + drafts: result.raw ? ModelsService.draftSidecarsByRepo(result.raw) : undefined, + error: result.error ?? null, + loaded: result.ok, + loading: false, + models: result.models, + raw: result.raw + }; + } +} + +export const backendsModelsStore = new BackendsModelsStore(); diff --git a/ui/apps/chat/src/lib/types/api.d.ts b/ui/apps/chat/src/lib/types/api.d.ts index 371913fc4c26..8cf64c1eb9cc 100644 --- a/ui/apps/chat/src/lib/types/api.d.ts +++ b/ui/apps/chat/src/lib/types/api.d.ts @@ -382,7 +382,9 @@ export interface ApiChatCompletionStreamChunk { metadata?: { model?: string }; delta: { content?: string; + reasoning?: string; reasoning_content?: string; + reasoning_text?: string; model?: string; tool_calls?: ApiChatCompletionToolCallDelta[]; }; @@ -396,6 +398,17 @@ export interface ApiChatCompletionStreamChunk { cache_n?: number; }; prompt_progress?: ChatMessagePromptProgress; + /** Token counts, sent by OpenAI-compatible servers on the final chunk. */ + usage?: ApiChatCompletionUsage; +} + +export interface ApiChatCompletionUsage { + cached_tokens?: number; + completion_tokens?: number; + prompt_cache_hit_tokens?: number; + prompt_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number; cache_write_tokens?: number }; + total_tokens?: number; } export interface ApiChatCompletionResponse { diff --git a/ui/apps/chat/src/lib/types/backend.d.ts b/ui/apps/chat/src/lib/types/backend.d.ts new file mode 100644 index 000000000000..019fce361026 --- /dev/null +++ b/ui/apps/chat/src/lib/types/backend.d.ts @@ -0,0 +1,98 @@ +/** + * Backend types. + * + * A backend is one API endpoint the UI can talk to. The built-in `local` + * backend is the llama-server serving the UI. External backends are + * user-configured endpoints that speak an OpenAI-compatible + * protocol. + */ + +/** Request/response shape a backend speaks. */ +export type BackendProtocol = 'llama.cpp' | 'openai'; + +/** + * Wire-level quirks of a backend's protocol. Capabilities gate llama.cpp + * features; compat describes how the request and stream payloads differ. + */ +export interface BackendCompat { + /** Field carrying the output token cap. */ + maxTokensField: 'max_completion_tokens' | 'max_tokens'; + /** Whether the endpoint accepts stream_options.include_usage. */ + supportsUsageInStreaming: boolean; +} + +/** + * Features a backend supports. A llama.cpp server exposes extra endpoints on + * top of the OpenAI-compatible API; plain OpenAI-compatible + * endpoints only provide chat and model listing. + */ +export interface BackendCapabilities { + /** llama-server's /cors-proxy endpoint for cross-origin MCP requests. */ + corsProxy: boolean; + /** Router-mode model load/unload. */ + loadUnload: boolean; + /** The /props endpoint with server role and generation defaults. */ + props: boolean; + /** Resumable stream sessions (/v1/stream, /v1/streams/lookup). */ + resumableStreams: boolean; + /** Multi-model router mode. */ + router: boolean; + /** The /slots introspection endpoint. */ + slots: boolean; + /** The /models/sse load and download progress feed. */ + statusFeed: boolean; + /** The /tools listing and execution endpoint. */ + tools: boolean; +} + +/** + * One configured API endpoint. + * + * TODO: a backend paired by QR code is reached over WebRTC instead of plain + * HTTP, so it needs a transport discriminator and its peer description here. + */ +export interface Backend { + /** Bearer token / API key used for this backend. */ + apiKey?: string; + /** + * API root the endpoint paths are appended to, e.g. https://api.example.com. + * Empty for the local backend, which resolves against the UI origin instead. + */ + baseUrl: string; + /** Chat completions path override, e.g. /v1/messages. */ + chatPath?: string; + /** Wire quirks overriding the protocol defaults. */ + compat?: Partial; + /** Disabled backends stay configured but are not queried. */ + enabled: boolean; + /** Extra headers merged into every request to this backend. */ + headers?: Record; + /** Stable identity. The local backend id is reserved. */ + id: string; + /** Models listing path override, e.g. /models. */ + modelsPath?: string; + name: string; + protocol: BackendProtocol; +} + +/** A ready-made backend configuration offered when adding a backend. */ +export interface BackendPreset { + /** Optional help text shown under the API key field. */ + apiKeyHelp?: string; + baseUrl: string; + /** One line describing the endpoint, shown on the preset card. */ + description?: string; + chatPath?: string; + /** Brand mark used in both themes, for logos that carry their own background. */ + iconUrl?: string; + /** Brand mark for the dark theme. Preferred over `iconUrl` when paired with `iconUrlLight`. */ + iconUrlDark?: string; + /** Brand mark for the light theme. Preferred over `iconUrl` when paired with `iconUrlDark`. */ + iconUrlLight?: string; + /** Wire quirks this preset needs on top of the protocol defaults. */ + compat?: Partial; + id: string; + modelsPath?: string; + name: string; + protocol: BackendProtocol; +} diff --git a/ui/apps/chat/src/lib/types/index.ts b/ui/apps/chat/src/lib/types/index.ts index ada84091cc94..f1d156502912 100644 --- a/ui/apps/chat/src/lib/types/index.ts +++ b/ui/apps/chat/src/lib/types/index.ts @@ -26,6 +26,7 @@ export type { ApiChatCompletionToolCallDelta, ApiChatCompletionToolCall, ApiChatCompletionStreamChunk, + ApiChatCompletionUsage, ApiChatCompletionResponse, ApiSlotData, ApiProcessingState, @@ -36,6 +37,15 @@ export type { ApiStreamSession } from './api'; +// Backend types +export type { + Backend, + BackendCapabilities, + BackendCompat, + BackendPreset, + BackendProtocol +} from './backend'; + // HuggingFace types export type { HfCatalogBuild, diff --git a/ui/apps/chat/src/lib/types/models.d.ts b/ui/apps/chat/src/lib/types/models.d.ts index 9d4a7c88882b..db696e601d8b 100644 --- a/ui/apps/chat/src/lib/types/models.d.ts +++ b/ui/apps/chat/src/lib/types/models.d.ts @@ -16,6 +16,8 @@ export interface ModelOption { id: string; name: string; model: string; + /** Backend that serves this model; set on the aggregated option list. */ + backendId?: string; description?: string; capabilities: string[]; /** Context size reported by the provider's model listing, when it reports one. */ @@ -27,6 +29,8 @@ export interface ModelOption { speaks?: boolean; details?: ApiModelDetails['details']; meta?: ApiModelDataEntry['meta']; + /** Load state the provider's own listing reports, when it reports one. */ + status?: ApiModelDataEntry['status']; parsedId?: ParsedModelId; aliases?: string[]; tags?: string[]; @@ -58,6 +62,21 @@ export interface ModelDownloadEntry { repoWithTag: string; } +/** + * A draft sidecar file a model listing reports as its own entry. The router lists a + * downloaded sidecar as a model, so this is what pairs it back with its model. + */ +export interface ModelSidecarFile { + id: string; + kind: ModelSidecar; + /** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */ + model: string; + /** Parameter count the sidecar reports, e.g. `35B-A3B`. */ + params: string | null; + /** Quantization of the sidecar file, e.g. `Q4_0`. */ + quant: string | null; +} + export interface ParsedModelId { raw: string; orgName: string | null; @@ -86,15 +105,3 @@ export interface ModalityCapabilities { hasAudio: boolean; hasVideo: boolean; } - -/** Sidecar file a listing reports as its own model entry, paired back to its model. */ -export interface ModelSidecarFile { - id: string; - kind: ModelSidecar; - /** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */ - model: string; - /** Parameter count the sidecar reports, e.g. `35B-A3B`. */ - params: string | null; - /** Quantization of the sidecar file, e.g. `Q4_0`. */ - quant: string | null; -} diff --git a/ui/apps/chat/src/lib/utils/api-base.ts b/ui/apps/chat/src/lib/utils/api-base.ts new file mode 100644 index 000000000000..1c8de35499c0 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/api-base.ts @@ -0,0 +1,99 @@ +/** + * API base resolution for backends. + * + * The UI can talk to more than one backend endpoint. Services build request + * URLs through {@link apiUrl} so a request always targets the right backend. + * The backends store registers a resolver here; this module never imports the + * store, which keeps URL resolution free of store dependencies. + */ + +import { backendChatUrl, backendModelsUrl } from './backend'; +import { withBase } from './base-path'; +import { API_ABSOLUTE_URL_PROTOCOLS, API_CHAT, API_MODELS } from '#lib/constants/index.ts'; +import type { Backend } from '#lib/types/index.ts'; + +/** Backend list and active selection as exposed to URL resolution. */ +export interface BackendsSnapshot { + activeId: string; + backends: Backend[]; +} + +type BackendsResolver = () => BackendsSnapshot; + +let resolveBackends: BackendsResolver | null = null; + +/** Registered once by the backends store. */ +export function setBackendsResolver(resolver: BackendsResolver | null): void { + resolveBackends = resolver; +} + +/** + * Look up a backend by id, defaulting to the active one. + */ +export function getBackend(backendId?: string): Backend | undefined { + const snapshot = resolveBackends?.(); + + if (!snapshot) return undefined; + + const id = backendId ?? snapshot.activeId; + + return snapshot.backends.find((backend) => backend.id === id); +} + +/** API root for a backend, or an empty string for the local backend. */ +export function getBackendBaseUrl(backendId?: string): string { + return getBackend(backendId)?.baseUrl.trim() ?? ''; +} + +/** + * Request target for a backend's chat completions endpoint. Returns a relative + * path for the local backend and an absolute URL for external ones, so callers + * can pass the result straight to `fetch` (or `apiFetch`, which resolves + * relative paths against the base). + */ +export function apiChatUrl(backendId?: string): string { + const backend = getBackend(backendId); + + if (backend?.baseUrl.trim()) { + return backendChatUrl(backend); + } + + return apiUrl(API_CHAT.COMPLETIONS, backendId); +} + +/** + * Request target for a backend's models listing. Returns a plain path for the + * local backend (so `apiFetch` applies the base) and an absolute URL otherwise. + */ +export function apiModelsUrl(backendId?: string): string { + const backend = getBackend(backendId); + + if (backend?.baseUrl.trim()) { + return backendModelsUrl(backend); + } + + return API_MODELS.LIST; +} + +/** + * Absolute URL for an API path on a backend. + * + * Absolute URLs pass through untouched. Paths on the local backend keep the + * existing base-path-relative form, so serving under a subpath still works. + * Paths on external backends resolve against the backend's API root. + */ +export function apiUrl(path: string, backendId?: string): string { + if (API_ABSOLUTE_URL_PROTOCOLS.some((protocol) => path.startsWith(protocol))) { + return path; + } + + const baseUrl = getBackendBaseUrl(backendId); + + if (!baseUrl) { + return withBase(path); + } + + const root = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`; + + return new URL(path.replace(/^\.?\//, ''), root).toString(); +} diff --git a/ui/apps/chat/src/lib/utils/api-fetch.ts b/ui/apps/chat/src/lib/utils/api-fetch.ts index 85534558a80f..4dc1c18df9a3 100644 --- a/ui/apps/chat/src/lib/utils/api-fetch.ts +++ b/ui/apps/chat/src/lib/utils/api-fetch.ts @@ -1,10 +1,6 @@ +import { apiUrl } from './api-base'; import { getAuthHeaders, getJsonHeaders } from './api-headers'; -import { withBase } from './base-path'; -import { - API_ABSOLUTE_URL_PROTOCOLS, - ERROR_MESSAGES, - HTTP_CODE_TO_STRING -} from '#lib/constants/index.ts'; +import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '#lib/constants/index.ts'; /** * API Fetch Utilities @@ -36,6 +32,8 @@ export interface ApiFetchOptions extends Omit { * Default: false (uses JSON headers with Content-Type: application/json) */ authOnly?: boolean; + /** Backend to target; defaults to the active one. */ + backendId?: string; /** * Additional headers to merge with default headers. */ @@ -63,11 +61,10 @@ export interface ApiFetchOptions extends Omit { * ``` */ export async function apiFetch(path: string, options: ApiFetchOptions = {}): Promise { - const { authOnly = false, headers: customHeaders, ...fetchOptions } = options; - const baseHeaders = authOnly ? getAuthHeaders() : getJsonHeaders(); + const { authOnly = false, backendId, headers: customHeaders, ...fetchOptions } = options; + const baseHeaders = authOnly ? getAuthHeaders(backendId) : getJsonHeaders(backendId); const headers = { ...baseHeaders, ...customHeaders }; - // absolute URLs with an allowed protocol pass through untouched; relative paths get the base prefix - const url = API_ABSOLUTE_URL_PROTOCOLS.some((p) => path.startsWith(p)) ? path : withBase(path); + const url = apiUrl(path, backendId); let response; @@ -110,7 +107,7 @@ export async function apiFetchWithParams( params: Record, options: ApiFetchOptions = {} ): Promise { - const url = new URL(basePath, window.location.href); + const url = new URL(apiUrl(basePath, options.backendId), window.location.href); for (const [key, value] of Object.entries(params)) { if (value !== undefined && value !== null) { diff --git a/ui/apps/chat/src/lib/utils/api-headers.ts b/ui/apps/chat/src/lib/utils/api-headers.ts index fa7f1b91748a..0c646eac7d1b 100644 --- a/ui/apps/chat/src/lib/utils/api-headers.ts +++ b/ui/apps/chat/src/lib/utils/api-headers.ts @@ -1,26 +1,42 @@ +import { getBackend } from './api-base'; import { redactValue } from './redact'; import { CORS_PROXY, HEADERS } from '#lib/constants/index.ts'; import { MimeTypeApplication } from '#lib/enums/index.ts'; +import { getProtocolAdapter } from '#lib/services/protocols'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import type { Backend } from '#lib/types/index.ts'; /** - * Get authorization headers for API requests - * Includes Bearer token if API key is configured + * Get authorization headers for API requests to a backend. */ -export function getAuthHeaders(): Record { - const currentConfig = settingsStore.config; - const apiKey = currentConfig.apiKey?.toString().trim(); +export function getAuthHeaders(backendId?: string): Record { + const backend = getBackend(backendId); + + if (backend) return getAuthHeadersForBackend(backend); + + // no backends resolver yet (early startup, or a non-browser call): keep the + // pre-backends behaviour and authenticate against the serving origin + const apiKey = settingsStore.config.apiKey?.toString().trim(); return apiKey ? { [HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKey}` } : {}; } +/** + * Get authorization headers for a backend object, including one that is not + * registered yet (used by the connection test on the add-backend form). + * The protocol adapter owns the credential scheme and any required headers. + */ +export function getAuthHeadersForBackend(backend: Backend): Record { + return getProtocolAdapter(backend).authHeaders(backend); +} + /** * Get standard JSON headers with optional authorization */ -export function getJsonHeaders(): Record { +export function getJsonHeaders(backendId?: string): Record { return { [HEADERS.CONTENT_TYPE]: MimeTypeApplication.JSON, - ...getAuthHeaders() + ...getAuthHeaders(backendId) }; } diff --git a/ui/apps/chat/src/lib/utils/api-key-validation.ts b/ui/apps/chat/src/lib/utils/api-key-validation.ts index 2783b32414ee..b9bcd74ede4c 100644 --- a/ui/apps/chat/src/lib/utils/api-key-validation.ts +++ b/ui/apps/chat/src/lib/utils/api-key-validation.ts @@ -1,8 +1,9 @@ -import { withBase } from './base-path'; import { error } from '@sveltejs/kit'; import { HEADERS } from '#lib/constants/index.ts'; import { MimeTypeApplication } from '#lib/enums/index.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; +import { apiUrl, getBackend } from '#lib/utils/api-base.ts'; +import { getBackendCapabilities } from '#lib/utils/backend.ts'; import { browser } from '$app/env'; /** @@ -14,6 +15,13 @@ export async function validateApiKey(fetch: typeof globalThis.fetch): Promise `z.ai`) and the icon is requested through a favicon service. + * Returns null when the URL carries no usable host. + */ +export function backendFaviconUrl(baseUrl: string): string | null { + try { + const labels = new URL(baseUrl).hostname.split('.'); + const root = labels.length > 2 ? labels.slice(-2).join('.') : labels.join('.'); + + return root ? `${FAVICON_SERVICE_URL}${root}&sz=${FAVICON_SIZE}` : null; + } catch { + return null; + } +} + +/** + * Features a backend supports, derived from its protocol. A missing backend + * (unknown model, early startup) gets the plain compatible defaults. + */ +export function getBackendCapabilities(backend?: Backend): BackendCapabilities { + return BACKEND_CAPABILITIES[backend?.protocol ?? 'openai'] ?? BACKEND_CAPABILITIES.openai; +} + +/** Wire quirks for a backend: protocol defaults overridden by the backend. */ +export function getBackendCompat(backend: Backend): BackendCompat { + return { ...(BACKEND_COMPAT[backend.protocol] ?? BACKEND_COMPAT.openai), ...backend.compat }; +} + +/** The built-in backend pointing at the server that serves this UI. */ +export function createLocalBackend(apiKey?: string, enabled = true): Backend { + return { + apiKey, + baseUrl: '', + enabled, + id: LOCAL_BACKEND_ID, + name: 'Local', + protocol: 'llama.cpp' + }; +} + +/** + * Parse the persisted backends JSON into backend entries. + */ +export function parseBackendsSettings(rawBackends: unknown): Backend[] { + if (!rawBackends) return []; + + let parsed: unknown; + + if (typeof rawBackends === 'string') { + const trimmed = rawBackends.trim(); + + if (!trimmed) return []; + + try { + parsed = JSON.parse(trimmed); + } catch (error) { + console.warn('[backends] Failed to parse backends JSON, ignoring value:', error); + + return []; + } + } else { + parsed = rawBackends; + } + + if (!Array.isArray(parsed)) return []; + + return parsed.flatMap((entry, index) => { + const backend = parseBackendEntry(entry, index); + + return backend ? [backend] : []; + }); +} + +function joinBackendUrl(baseUrl: string, path: string): string { + const base = baseUrl.replace(/\/+$/, ''); + const suffix = path.startsWith('/') ? path : `/${path}`; + + return `${base}${suffix}`; +} + +function parseBackendEntry(entry: unknown, index: number): Backend | null { + if (!entry || typeof entry !== 'object') return null; + + const raw = entry as Record; + const baseUrl = typeof raw.baseUrl === 'string' ? raw.baseUrl.trim() : ''; + + // the local backend is built in and never persisted + if (!baseUrl || raw.id === LOCAL_BACKEND_ID) return null; + + const protocol = BACKEND_PROTOCOLS.includes(raw.protocol as BackendProtocol) + ? (raw.protocol as BackendProtocol) + : 'openai'; + const id = + typeof raw.id === 'string' && raw.id.trim() + ? raw.id.trim() + : `${BACKEND_ID_PREFIX}-${index + 1}`; + const name = typeof raw.name === 'string' && raw.name.trim() ? raw.name.trim() : baseUrl; + const apiKey = + typeof raw.apiKey === 'string' && raw.apiKey.trim() ? raw.apiKey.trim() : undefined; + + return { + apiKey, + baseUrl, + chatPath: parseOptionalPath(raw.chatPath), + compat: parseBackendCompat(raw.compat, protocol), + enabled: raw.enabled !== false, + headers: parseBackendHeaders(raw.headers), + id, + modelsPath: parseOptionalPath(raw.modelsPath), + name, + protocol + }; +} + +// only keep the override keys the protocol understands, so a stale persisted +// value can never inject an unknown field into a request +function parseBackendCompat( + raw: unknown, + protocol: BackendProtocol +): Partial | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + + const entry = raw as Record; + const defaults = BACKEND_COMPAT[protocol] ?? BACKEND_COMPAT.openai; + const overrides: Partial = {}; + + if (entry.maxTokensField === 'max_tokens' || entry.maxTokensField === 'max_completion_tokens') { + overrides.maxTokensField = entry.maxTokensField; + } + + if (typeof entry.supportsUsageInStreaming === 'boolean') { + overrides.supportsUsageInStreaming = entry.supportsUsageInStreaming; + } + + // drop a no-op override so an unmodified backend stays undefined + const isDefault = + (overrides.maxTokensField === undefined || + overrides.maxTokensField === defaults.maxTokensField) && + (overrides.supportsUsageInStreaming === undefined || + overrides.supportsUsageInStreaming === defaults.supportsUsageInStreaming); + + return isDefault ? undefined : overrides; +} + +function parseBackendHeaders(raw: unknown): Record | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + + const entries = Object.entries(raw as Record) + .filter(([, value]) => typeof value === 'string' && value.trim() !== '') + .map(([key, value]) => [key.trim(), (value as string).trim()] as const); + + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + +function parseOptionalPath(raw: unknown): string | undefined { + return typeof raw === 'string' && raw.trim() ? raw.trim() : undefined; +} + +/** + * Read a model's context size out of one `/v1/models` entry. Providers use + * different field names, and OpenRouter nests the authoritative value under + * `top_provider`, so try the flat fields first and the nested one after. + */ +export function readModelContextLength(value: unknown): number | undefined { + if (!value || typeof value !== 'object') return undefined; + + const entry = value as Record; + const flat = readContextField(entry); + + if (flat !== undefined) return flat; + + const topProvider = entry.top_provider; + + if (topProvider && typeof topProvider === 'object') { + const nested = readContextField(topProvider as Record); + + if (nested !== undefined) return nested; + } + + // Hugging Face lists one entry per inference provider, and they disagree on + // the budget; take the largest so the gauge does not undersell the model. + const providers = entry.providers; + + if (Array.isArray(providers)) { + const sizes = providers + .map((provider) => + provider && typeof provider === 'object' + ? readContextField(provider as Record) + : undefined + ) + .filter((size): size is number => size !== undefined); + + if (sizes.length > 0) return Math.max(...sizes); + } + + return undefined; +} + +function readContextField(entry: Record): number | undefined { + for (const field of MODEL_CONTEXT_LENGTH_FIELDS) { + const value = entry[field]; + + if (typeof value === 'number' && value > 0) return value; + } + + return undefined; +} diff --git a/ui/apps/chat/src/lib/utils/cors-proxy.ts b/ui/apps/chat/src/lib/utils/cors-proxy.ts index c5875482bd28..7f3307c372f4 100644 --- a/ui/apps/chat/src/lib/utils/cors-proxy.ts +++ b/ui/apps/chat/src/lib/utils/cors-proxy.ts @@ -2,7 +2,7 @@ * CORS Proxy utility for routing requests through llama-server's CORS proxy. */ -import { withBase } from './base-path'; +import { apiUrl } from './api-base'; import { CORS_PROXY, CORS_PROXY_ENDPOINT } from '#lib/constants/index.ts'; import { UrlProtocol } from '#lib/enums/index.ts'; @@ -12,7 +12,7 @@ import { UrlProtocol } from '#lib/enums/index.ts'; * @returns URL pointing to the CORS proxy with target encoded */ export function buildProxiedUrl(targetUrl: string): URL { - const proxyPath = withBase(CORS_PROXY_ENDPOINT); + const proxyPath = apiUrl(CORS_PROXY_ENDPOINT); const proxyUrl = new URL(proxyPath, window.location.origin); proxyUrl.searchParams.set(CORS_PROXY.URL_PARAM, targetUrl); diff --git a/ui/apps/chat/src/lib/utils/index.ts b/ui/apps/chat/src/lib/utils/index.ts index ec699e061282..441e1dc20a95 100644 --- a/ui/apps/chat/src/lib/utils/index.ts +++ b/ui/apps/chat/src/lib/utils/index.ts @@ -8,10 +8,31 @@ */ // API utilities -export { getAuthHeaders, getJsonHeaders, sanitizeHeaders } from './api-headers'; +export { + apiChatUrl, + apiModelsUrl, + apiUrl, + getBackend, + getBackendBaseUrl, + type BackendsSnapshot +} from './api-base'; +export { + getAuthHeaders, + getAuthHeadersForBackend, + getJsonHeaders, + sanitizeHeaders +} from './api-headers'; export { ApiError, apiDelete, apiFetch, apiFetchWithParams, apiPost } from './api-fetch'; export { withBase } from './base-path'; export { validateApiKey } from './api-key-validation'; +export { + backendChatUrl, + backendFaviconUrl, + backendModelsUrl, + createLocalBackend, + getBackendCapabilities, + parseBackendsSettings +} from './backend'; // Attachment utilities export { getAttachmentDisplayItems, isMcpPrompt, isMcpResource } from './attachment-display'; @@ -122,6 +143,10 @@ export { // Model name utilities export { isValidModelName, normalizeModelName, orgOf, repoOf } from './model-names'; +// Backend-qualified model option ids +export { backendIdFromModelId, qualifyModelId, rawModelId } from './model-option-id'; +export { groupModelFamilies, modelFamilyKey, type ModelFamilyGroup } from './model-families'; + // Sidecar token utilities export { directoryOf, @@ -164,6 +189,8 @@ export { extractSseDataPayload, parseSseJsonStream, splitSseRecords } from './ss // Stream session identity (conversation-id based) export { streamIdentity } from './stream-identity'; +export { buildTimingsFromUsage, usageTokenCounts } from './timings'; + // MCP utilities export { detectMcpTransportFromUrl, @@ -390,7 +417,6 @@ export { applySurfaceTheme, remToPx } from './css'; // Audio format helper (used by agentic store and chat service) export { getAudioInputFormat } from './audio-format'; -export { groupModelFamilies, modelFamilyKey, type ModelFamilyGroup } from './model-families'; // Svelte actions export { nearViewport } from './near-viewport'; diff --git a/ui/apps/chat/src/lib/utils/model-option-id.ts b/ui/apps/chat/src/lib/utils/model-option-id.ts new file mode 100644 index 000000000000..b856cb39dc20 --- /dev/null +++ b/ui/apps/chat/src/lib/utils/model-option-id.ts @@ -0,0 +1,28 @@ +/** + * Backend-qualified model option ids. + * + * The selector lists models from every enabled backend, so a bare model id can + * collide. Option ids are qualified as `::`; selection + * strips the prefix to find the row in the active backend's list. + */ + +const MODEL_OPTION_ID_SEPARATOR = '::'; + +/** Prefix a raw model id with the backend that serves it. */ +export function qualifyModelId(backendId: string, modelId: string): string { + return `${backendId}${MODEL_OPTION_ID_SEPARATOR}${modelId}`; +} + +/** Backend id of a qualified model option id, or null when unqualified. */ +export function backendIdFromModelId(qualifiedId: string): string | null { + const index = qualifiedId.indexOf(MODEL_OPTION_ID_SEPARATOR); + + return index === -1 ? null : qualifiedId.slice(0, index); +} + +/** Strip the backend prefix from a qualified model option id. */ +export function rawModelId(qualifiedId: string): string { + const index = qualifiedId.indexOf(MODEL_OPTION_ID_SEPARATOR); + + return index === -1 ? qualifiedId : qualifiedId.slice(index + MODEL_OPTION_ID_SEPARATOR.length); +} diff --git a/ui/apps/chat/src/lib/utils/timings.ts b/ui/apps/chat/src/lib/utils/timings.ts new file mode 100644 index 000000000000..96f3a637cc9b --- /dev/null +++ b/ui/apps/chat/src/lib/utils/timings.ts @@ -0,0 +1,66 @@ +/** + * Client side timing fallback for backends that do not report their own. + * + * llama.cpp streams per-token timings; OpenAI-compatible servers + * do not. Token counts come from the usage block of the final chunk (or the + * count of streamed deltas as a fallback), times are measured locally: the wait + * for the first token is attributed to prompt processing, the rest to + * generation. Wall clock, so network and queueing are part of the numbers. + */ + +import type { ApiChatCompletionUsage } from '#lib/types/api.js'; +import type { ChatMessageTimings } from '#lib/types/chat.js'; + +export interface StreamClock { + startedAt: number; + firstTokenAt: number | null; + lastTokenAt: number | null; +} + +/** + * Prompt/output/cache token counts. `promptTokens` excludes the cache read + * tokens, which are returned separately as `cacheTokens`, so the two always + * add up to the prompt size. + */ +export function usageTokenCounts(usage: ApiChatCompletionUsage | undefined): { + cacheTokens: number; + completionTokens: number; + promptTokens: number; +} { + // a total that includes the cache reads, which are reported separately + const cacheTokens = + usage?.prompt_tokens_details?.cached_tokens ?? + usage?.prompt_cache_hit_tokens ?? + usage?.cached_tokens ?? + 0; + const promptTotal = usage?.prompt_tokens ?? 0; + + return { + cacheTokens, + completionTokens: usage?.completion_tokens ?? 0, + promptTokens: Math.max(0, promptTotal - cacheTokens) + }; +} + +export function buildTimingsFromUsage( + usage: ApiChatCompletionUsage | undefined, + clock: StreamClock, + fallbackTokens = 0 +): ChatMessageTimings | null { + const { cacheTokens, completionTokens, promptTokens } = usageTokenCounts(usage); + const predictedN = completionTokens || fallbackTokens; + + if (promptTokens === 0 && predictedN === 0) return null; + + const { firstTokenAt, startedAt } = clock; + const lastTokenAt = clock.lastTokenAt ?? firstTokenAt; + + return { + cache_n: cacheTokens, + // clamp so a one-token reply still reports a positive duration + predicted_ms: firstTokenAt && lastTokenAt ? Math.max(1, lastTokenAt - firstTokenAt) : undefined, + predicted_n: predictedN, + prompt_ms: firstTokenAt ? Math.max(1, firstTokenAt - startedAt) : undefined, + prompt_n: promptTokens + }; +} From 3a02747fdb826a0352cb233bd12bc7b4b46e6c73 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 28 Sep 2026 15:37:21 +0200 Subject: [PATCH 02/34] ui : route chat through the active provider Chat goes through protocol adapters, so an OpenAI-compatible endpoint speaks its own wire format: per-backend paths and headers, the model on the request, tools kept on the local server, and token counts synthesized for endpoints that do not stream their own timings. The server store keeps the local server's props while another provider is active, a conversation resolves the provider its model belongs to before sending, and the chat screen never blocks on the local probe when the install has none. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash --- .../ContextGaugePopup.svelte | 2 +- .../ChatMessageAssistant.svelte | 1 - .../app/chat/ChatScreen/ChatScreen.svelte | 7 +- .../chat/ChatScreen/ChatScreenGreeting.svelte | 5 +- .../app/servers/ServerErrorSplash.svelte | 4 +- .../app/servers/ServerStatus.svelte | 2 +- .../hooks/use-backend-availability.svelte.ts | 38 +++ .../src/lib/hooks/use-context-gauge.svelte.ts | 11 +- ui/apps/chat/src/lib/services/chat.service.ts | 277 +++++++++++------- .../chat/src/lib/services/models.service.ts | 118 +++++++- .../chat/src/lib/services/props.service.ts | 8 +- .../chat/src/lib/services/protocols/index.ts | 24 ++ .../chat/src/lib/services/protocols/openai.ts | 185 ++++++++++++ .../chat/src/lib/services/protocols/types.ts | 43 +++ .../chat/src/lib/services/tools.service.ts | 20 +- .../src/lib/stores/backendsModels.svelte.ts | 2 +- .../chat/src/lib/stores/chat/index.svelte.ts | 21 +- ui/apps/chat/src/lib/stores/index.ts | 4 + ui/apps/chat/src/lib/stores/init.ts | 69 ++++- .../src/lib/stores/llama-server.svelte.ts | 153 +++++++++- .../src/lib/stores/models/index.svelte.ts | 257 ++++++++++++++-- .../src/lib/stores/models/props.svelte.ts | 48 ++- .../src/lib/stores/models/status.svelte.ts | 155 ++++++++-- ui/apps/chat/src/lib/stores/tools.svelte.ts | 11 + ui/apps/chat/src/lib/utils/api-headers.ts | 2 +- ui/apps/chat/src/lib/utils/backend.ts | 10 + .../chat/src/routes/[extensionPage]/+page.ts | 6 +- .../tests/unit/extension-page-route.test.ts | 10 +- 28 files changed, 1274 insertions(+), 219 deletions(-) create mode 100644 ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts create mode 100644 ui/apps/chat/src/lib/services/protocols/index.ts create mode 100644 ui/apps/chat/src/lib/services/protocols/openai.ts create mode 100644 ui/apps/chat/src/lib/services/protocols/types.ts diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte index 802dd7e8d081..53c0820e1896 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte @@ -104,7 +104,7 @@ {/snippet} {#snippet body()} - {#if gauge.activeModelId !== null && !gauge.isActiveModelLoaded} + {#if gauge.canLoadActiveModel && gauge.activeModelId !== null && !gauge.isActiveModelLoaded} diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte index 97c74861c62e..4b97863c4229 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreen.svelte @@ -26,6 +26,7 @@ CHAT_FORM_MOVE_TO_BOTTOM_IN_MS, CHAT_FORM_MOVE_TO_BOTTOM_OUT_MS } from '#lib/constants/index.ts'; + import { useBackendAvailability } from '#lib/hooks/use-backend-availability.svelte.ts'; import { useChatScreenActiveModel } from '#lib/hooks/use-chat-screen-active-model.svelte.ts'; import { useChatScreenDragAndDrop } from '#lib/hooks/use-chat-screen-drag-and-drop.svelte.ts'; import { useChatScreenFileUpload } from '#lib/hooks/use-chat-screen-file-upload.svelte.ts'; @@ -77,8 +78,10 @@ let formWrapper: HTMLElement | undefined = $state(); let chatScreenForm: ChatScreenForm | undefined = $state(); let activeErrorDialog = $derived(chatStore.errorDialogState); - let isServerLoading = $derived(serverStore.loading); - let hasPropsError = $derived(!!serverStore.error); + // no local server in this deployment: never block on its probe + let isServerLoading = $derived(serverStore.loading && !serverStore.localServerMissing); + const availability = useBackendAvailability(); + let hasPropsError = $derived(availability.isOffline); let isCurrentConversationLoading = $derived(chatStore.isLoading || chatStore.isStreaming()); let chatFormBottomPosition = $derived.by(() => { if (!deviceStore.isMobile) return CHAT_FORM_BOTTOM_OFFSET.desktop; diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte index 22bde59ce057..720e397368ca 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte @@ -63,8 +63,9 @@

Hello there

- {serverStore.props?.modalities?.audio ? 'Record audio, type a message ' : 'Type a message'} or upload - files to get started + {serverStore.localProps?.modalities?.audio + ? 'Record audio, type a message ' + : 'Type a message'} or upload files to get started

diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte index fab3edba4090..a1ea04305dce 100644 --- a/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte +++ b/ui/apps/chat/src/lib/components/app/servers/ServerErrorSplash.svelte @@ -6,7 +6,7 @@ import { HEADERS, ICON_CLASS_DEFAULT, ROUTES, SETTINGS_KEYS } from '#lib/constants/index.ts'; import { KeyboardKey } from '#lib/enums/index.ts'; import { serverStore, settingsStore } from '#lib/stores/index.ts'; - import { withBase } from '#lib/utils/base-path.ts'; + import { apiUrl } from '#lib/utils/api-base.ts'; import { goto } from '$app/navigation'; import { fade, fly, scale } from 'svelte/transition'; @@ -67,7 +67,7 @@ settingsStore.updateConfig(SETTINGS_KEYS.API_KEY, apiKeyInput.trim()); // Test the API key by making a real request to the server - const response = await fetch(withBase('/props'), { + const response = await fetch(apiUrl('/props'), { headers: { 'Content-Type': 'application/json', [HEADERS.AUTHORIZATION]: `${HEADERS.BEARER}${apiKeyInput.trim()}` diff --git a/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte b/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte index 630079fd9a73..590fcc66067f 100644 --- a/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte +++ b/ui/apps/chat/src/lib/components/app/servers/ServerStatus.svelte @@ -15,7 +15,7 @@ let error = $derived(serverStore.error); let loading = $derived(serverStore.loading); let model = $derived(modelsStore.singleModelName); - let serverData = $derived(serverStore.props); + let serverData = $derived(serverStore.localProps); function getStatusColor() { if (loading) return 'status-lamp orb-warning'; diff --git a/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts b/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts new file mode 100644 index 000000000000..e88c1540ed79 --- /dev/null +++ b/ui/apps/chat/src/lib/hooks/use-backend-availability.svelte.ts @@ -0,0 +1,38 @@ +/** + * Backend availability for connection error states. + * + * The server error banner and the offline affordances should only appear when + * no backend can serve requests. The active backend failing is not enough: + * with an external backend configured, the UI stays usable. + */ + +import { LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; +import { backendsModelsStore, backendsStore, serverStore } from '#lib/stores/index.ts'; + +export interface BackendAvailability { + readonly hasAvailableBackend: boolean; + readonly isOffline: boolean; +} + +export function useBackendAvailability(): BackendAvailability { + const hasAvailableBackend = $derived.by(() => { + // no connection error at all: nothing to report + if (!serverStore.error) return true; + + // the local server failed, but an external backend may still be usable + return backendsStore.enabled.some( + (backend) => + backend.id !== LOCAL_BACKEND_ID && backendsModelsStore.get(backend.id).error === null + ); + }); + + return { + get hasAvailableBackend() { + return hasAvailableBackend; + }, + + get isOffline() { + return !hasAvailableBackend; + } + }; +} diff --git a/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts b/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts index 66bd9058269b..bca2078f3cc3 100644 --- a/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts +++ b/ui/apps/chat/src/lib/hooks/use-context-gauge.svelte.ts @@ -15,13 +15,16 @@ import { chatStore, contextStatsStore, conversationsStore, - modelsStore + modelsStore, + serverStore } from '#lib/stores/index.ts'; import type { DatabaseMessage } from '#lib/types/index.ts'; import { page } from '$app/state'; export interface UseContextGaugeReturn { readonly activeModelId: string | null; + /** Whether the active backend can load the model it is serving. */ + readonly canLoadActiveModel: boolean; readonly isActiveModelLoaded: boolean; readonly isActiveModelLoading: boolean; readonly contextTotal: number | null; @@ -100,6 +103,9 @@ export function useContextGauge(): UseContextGaugeReturn { contextStatsStore.averageTokensPerSecond !== null || transientDetails.length > 0 ); + // loading is a llama.cpp router feature, and the gauge tracks the active model, + // so the active backend decides whether a load is possible at all + const canLoadActiveModel = $derived(serverStore.isRouterMode); async function loadModel() { const modelId = contextStatsStore.activeModelId; @@ -120,6 +126,9 @@ export function useContextGauge(): UseContextGaugeReturn { get averageTokensPerSecond() { return contextStatsStore.averageTokensPerSecond; }, + get canLoadActiveModel() { + return canLoadActiveModel; + }, get colorLevel() { return colorLevel; }, diff --git a/ui/apps/chat/src/lib/services/chat.service.ts b/ui/apps/chat/src/lib/services/chat.service.ts index 38c7eba144b1..8a5353c3ea7b 100644 --- a/ui/apps/chat/src/lib/services/chat.service.ts +++ b/ui/apps/chat/src/lib/services/chat.service.ts @@ -19,6 +19,7 @@ import { SSE_DATA_PREFIX, SSE_DONE_MARKER, SSE_LINE_SEPARATOR, + STREAM_LIVE_TIMINGS_INTERVAL_MS, STREAM_QUERY_PARAMS, STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX, STREAM_VISIBILITY_KICK_MS @@ -31,6 +32,9 @@ import { ReasoningFormat, StreamConnectionState } from '#lib/enums/index.ts'; +import { getProtocolAdapter } from '#lib/services/protocols/index.ts'; +import { extractModelName } from '#lib/services/protocols/openai.ts'; +import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; import type { @@ -45,10 +49,12 @@ import type { } from '#lib/types/index.ts'; import type { OpenAIToolDefinition } from '#lib/types/mcp.js'; import { isAbortError } from '#lib/utils/abort.ts'; +import { apiChatUrl, apiUrl, getBackend } from '#lib/utils/api-base.ts'; import { ApiError } from '#lib/utils/api-fetch.ts'; import { getAuthHeaders, getJsonHeaders } from '#lib/utils/api-headers.ts'; import { formatAttachmentText } from '#lib/utils/formatters.ts'; import { streamIdentity } from '#lib/utils/stream-identity.ts'; +import { buildTimingsFromUsage } from '#lib/utils/timings.ts'; /** How stored messages are shaped for the model a request goes to. */ export interface ApiMessageConversionOptions { @@ -95,9 +101,12 @@ export class ChatService { * @returns {Promise} Promise that resolves to true if all slots are idle, false if any is processing */ static async areAllSlotsIdle(model?: string | null, signal?: AbortSignal): Promise { + // the /slots endpoint only exists on llama.cpp servers + if (!serverStore.capabilities.slots) return true; + try { const url = model ? `${API_SLOTS.LIST}?model=${encodeURIComponent(model)}` : API_SLOTS.LIST; - const res = await fetch(url, { signal }); + const res = await fetch(apiUrl(url), { signal }); if (!res.ok) return true; @@ -115,6 +124,8 @@ export class ChatService { static async cancelServerStream(conversationId: string, model?: string | null): Promise { if (!conversationId) return; + if (!serverStore.capabilities.resumableStreams) return; + try { const id = streamIdentity(conversationId, model); @@ -373,6 +384,10 @@ export class ChatService { * caller can pipe it through the SSE parser like a fresh stream. */ static async fetchStreamReplay(streamId: string): Promise { + if (!serverStore.capabilities.resumableStreams) { + return new Response(null, { status: 501, statusText: 'Not Implemented' }); + } + const resp = await fetch(ChatService.buildStreamUrl(streamId, 0), { headers: getAuthHeaders() }); @@ -519,6 +534,17 @@ export class ChatService { let toolCallIndexOffset = 0; let hasOpenToolCallBatch = false; + // client side clock for backends that do not stream their own timings + const startedAt = Date.now(); + + let firstTokenAt: number | null = null; + let lastTokenAt: number | null = null; + let streamedTokens = 0; + let liveTimingsAt = 0; + let usage: ApiChatCompletionUsage | undefined; + + // the protocol decides how payloads map onto canonical events + const streamReader = getProtocolAdapter(getBackend()).createStreamReader(); const finalizeOpenToolCallBatch = () => { if (!hasOpenToolCallBatch) { return; @@ -558,6 +584,32 @@ export class ChatService { onToolCallChunk?.(serializedToolCalls); } }; + // backends that do not stream their own timings report progress from wall + // clock time and the streamed delta count, throttled to keep updates cheap + const markToken = () => { + firstTokenAt ??= Date.now(); + lastTokenAt = Date.now(); + streamedTokens++; + + if ( + serverStore.capabilities.props || + Date.now() - liveTimingsAt < STREAM_LIVE_TIMINGS_INTERVAL_MS + ) { + return; + } + + liveTimingsAt = Date.now(); + + const liveTimings = buildTimingsFromUsage( + usage, + { firstTokenAt, lastTokenAt, startedAt }, + streamedTokens + ); + + if (liveTimings) { + ChatService.notifyTimings(liveTimings, undefined, onTimings); + } + }; const onVisibilityChange = () => { if (typeof document === 'undefined') return; @@ -659,59 +711,94 @@ export class ChatService { continue; } + let parsed: unknown; + try { - const parsed: ApiChatCompletionStreamChunk = JSON.parse(data); - const choice = parsed.choices?.[0]; + parsed = JSON.parse(data); + } catch (parseError) { + console.error('Error parsing JSON chunk:', parseError); - if (choice?.finish_reason) finishReason = choice.finish_reason; + continue; + } - const content = choice?.delta?.content; - const reasoningContent = choice?.delta?.reasoning_content; - const toolCalls = choice?.delta?.tool_calls; - const timings = parsed.timings; - const promptProgress = parsed.prompt_progress; - const chunkModel = ChatService.extractModelName(parsed); + for (const event of streamReader.readChunk(parsed)) { + switch (event.type) { + case 'done': + streamFinished = true; - if (chunkModel && !modelEmitted) { - modelEmitted = true; - onModel?.(chunkModel); - } + break; - if (parsed.id && !idEmitted) { - idEmitted = true; - onCompletionId?.(parsed.id); - } + case 'error': + throw new Error(event.message); - if (promptProgress) { - ChatService.notifyTimings(undefined, promptProgress, onTimings); - } + case 'finish_reason': + finishReason = event.reason; - if (timings) { - ChatService.notifyTimings(timings, promptProgress, onTimings); - lastTimings = timings; - } + break; - if (content) { - finalizeOpenToolCallBatch(); - aggregatedContent += content; + case 'id': + if (!idEmitted) { + idEmitted = true; + onCompletionId?.(event.id); + } - if (!abortSignal?.aborted) { - onChunk?.(content); - } - } + break; - if (reasoningContent) { - finalizeOpenToolCallBatch(); - fullReasoningContent += reasoningContent; + case 'model': + if (!modelEmitted) { + modelEmitted = true; + onModel?.(event.model); + } - if (!abortSignal?.aborted) { - onReasoningChunk?.(reasoningContent); - } - } + break; + + case 'prompt_progress': + ChatService.notifyTimings(undefined, event.progress, onTimings); + + break; + + case 'text': + finalizeOpenToolCallBatch(); + aggregatedContent += event.text; + + if (!abortSignal?.aborted) { + onChunk?.(event.text); + } + + markToken(); + + break; + + case 'thinking': + finalizeOpenToolCallBatch(); + fullReasoningContent += event.text; + + if (!abortSignal?.aborted) { + onReasoningChunk?.(event.text); + } + + markToken(); + + break; + + case 'timings': + ChatService.notifyTimings(event.timings, event.promptProgress, onTimings); + lastTimings = event.timings; + + break; - processToolCallDelta(toolCalls); - } catch (e) { - console.error('Error parsing JSON chunk:', e); + case 'tool_calls': + processToolCallDelta(event.deltas); + + break; + + case 'usage': + // providers may split usage across chunks (some report input + // tokens on message_start and output tokens on message_delta) + usage = { ...usage, ...event.usage }; + + break; + } } } } @@ -782,6 +869,20 @@ export class ChatService { if (streamFinished) { finalizeOpenToolCallBatch(); + // external backends report token counts only in the final usage chunk + if (!lastTimings && !serverStore.capabilities.props) { + lastTimings = + buildTimingsFromUsage( + usage, + { firstTokenAt, lastTokenAt, startedAt }, + streamedTokens + ) ?? undefined; + + if (lastTimings) { + ChatService.notifyTimings(lastTimings, undefined, onTimings); + } + } + if (conversationId) { ChatService.clearStreamState(conversationId); } @@ -822,7 +923,9 @@ export class ChatService { * conv::model identity when a model was bound at POST time. */ static async lookupStreamSessions(conversationIds: string[]): Promise { - const resp = await fetch(API_STREAM.LOOKUP, { + if (!serverStore.capabilities.resumableStreams) return []; + + const resp = await fetch(apiUrl(API_STREAM.LOOKUP), { body: JSON.stringify({ conversation_ids: conversationIds }), headers: getJsonHeaders(), method: 'POST' @@ -890,6 +993,9 @@ export class ChatService { signal?: AbortSignal, tools?: OpenAIToolDefinition[] ): Promise { + // pre-encode warms the llama.cpp KV cache and posts llama.cpp-only fields + if (!serverStore.capabilities.props) return null; + const normalizedMessages: ApiChatMessageData[] = await ChatService.normalizeMessagesForApi( messages, model @@ -919,7 +1025,7 @@ export class ChatService { } try { - const response = await fetch(API_CHAT.COMPLETIONS, { + const response = await fetch(apiChatUrl(), { body: JSON.stringify(requestBody), headers: getJsonHeaders(), method: 'POST', @@ -958,6 +1064,8 @@ export class ChatService { static async probeResumeStatus(streamId: string): Promise { if (!streamId) return 0; + if (!serverStore.capabilities.resumableStreams) return 0; + const ac = new AbortController(); try { @@ -981,6 +1089,8 @@ export class ChatService { ): Promise { if (!conversationId) return null; + if (!serverStore.capabilities.resumableStreams) return null; + const state = ChatService.getStreamState(conversationId); const from = state?.bytesReceived ?? 0; const id = streamIdentity(conversationId, model); @@ -1297,16 +1407,26 @@ export class ChatService { // tag streaming requests with the conversation id, this single header is the opt in for the // server side replay buffer and powers discoverActiveStream on tab reopen. with an explicit - // model the ::model suffix keeps the per model session distinct - if (stream && conversationId) { + // model the ::model suffix keeps the per model session distinct. external providers do not + // know the header and their CORS preflight rejects it, so only llama.cpp gets it + if (stream && conversationId && serverStore.capabilities.resumableStreams) { headers[HEADERS.X_CONVERSATION_ID_HEADER] = streamIdentity(conversationId, options.model); // persist the pending stream before the fetch: a reload during the model load or // the prompt processing must still find its way back to the session once it exists ChatService.saveStreamState(conversationId, 0, options.model ?? null); } - const response = await fetch(API_CHAT.COMPLETIONS, { - body: JSON.stringify(requestBody), + // the protocol adapter owns the wire format: it strips llama.cpp-only + // fields, applies the backend's token cap field and adds the usage chunk + const backend = getBackend(); + const wireBody = backend + ? getProtocolAdapter(backend).buildChatRequest( + requestBody as unknown as Record, + backend + ) + : (requestBody as unknown as Record); + const response = await fetch(apiChatUrl(), { + body: JSON.stringify(wireBody), headers, method: 'POST', signal @@ -1417,7 +1537,7 @@ export class ChatService { if (model) body.model = model; try { - const res = await fetch(API_CHAT.CONTROL, { + const res = await fetch(apiUrl(API_CHAT.CONTROL), { body: JSON.stringify(body), headers: getJsonHeaders(), method: 'POST' @@ -1448,62 +1568,7 @@ export class ChatService { const query = `${STREAM_QUERY_PARAMS.CONV_ID}=${encodeURIComponent(streamId)}`; const offset = from === undefined ? '' : `&${STREAM_QUERY_PARAMS.FROM}=${from}`; - return `${API_STREAM.BASE}?${query}${offset}`; - } - - /** - * Extracts model name from Chat Completions API response data. - * Handles various response formats including streaming chunks and final responses. - * - * WORKAROUND: In single model mode, llama-server returns a default/incorrect model name - * in the response. We override it with the actual model name from serverStore. - * - * @param data - Raw response data from the Chat Completions API - * @returns Model name string if found, undefined otherwise - * @private - */ - private static extractModelName(data: unknown): string | undefined { - const asRecord = (value: unknown): Record | undefined => { - return typeof value === 'object' && value !== null - ? (value as Record) - : undefined; - }; - const getTrimmedString = (value: unknown): string | undefined => { - return typeof value === 'string' && value.trim() ? value.trim() : undefined; - }; - const root = asRecord(data); - - if (!root) return undefined; - - // 1) root (some implementations provide `model` at the top level) - const rootModel = getTrimmedString(root.model); - - if (rootModel) { - return rootModel; - } - - // 2) streaming choice (delta) or final response (message) - const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; - - if (!firstChoice) { - return undefined; - } - - // priority: delta.model (first chunk) else message.model (final response) - const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); - - if (deltaModel) { - return deltaModel; - } - - const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); - - if (messageModel) { - return messageModel; - } - - // avoid guessing from non-standard locations (metadata, etc.) - return undefined; + return apiUrl(`${API_STREAM.BASE}?${query}${offset}`); } /** @@ -1539,7 +1604,7 @@ export class ChatService { } const data: ApiChatCompletionResponse = JSON.parse(responseText); - const responseModel = ChatService.extractModelName(data); + const responseModel = extractModelName(data); if (responseModel) { onModel?.(responseModel); diff --git a/ui/apps/chat/src/lib/services/models.service.ts b/ui/apps/chat/src/lib/services/models.service.ts index 87abe1a94245..d1529b5ed3a1 100644 --- a/ui/apps/chat/src/lib/services/models.service.ts +++ b/ui/apps/chat/src/lib/services/models.service.ts @@ -6,23 +6,49 @@ * modelsStore and its status manager. */ -import { API_MODELS, MODEL_ID, type ModelSidecar } from '#lib/constants/index.ts'; +import { + API_MODELS, + LOCAL_BACKEND_ID, + MODEL_ID, + type ModelSidecar, + SIDECAR_TOKENS +} from '#lib/constants/index.ts'; import { ServerModelStatus } from '#lib/enums/index.ts'; -import type { ModelSidecarBadge, ParsedModelId } from '#lib/types/models.js'; +import type { ModelSidecarBadge, ModelSidecarFile, ParsedModelId } from '#lib/types/models.js'; import { getAuthHeaders } from '#lib/utils/api-headers.ts'; -import { withBase } from '#lib/utils/base-path.ts'; import { apiDelete, apiFetch, + apiModelsUrl, apiPost, + apiUrl, extractSseDataPayload, isAuxSidecar, normalizeModelName, repoOf, sidecarFromFileToken, + sidecarFromTag, splitSseRecords } from '#lib/utils/index.ts'; +/** Sidecar token a file name carries, for the forms parsing an id as a model misses. */ +function sidecarTokenInFilename(modelId: string): ModelSidecar | null { + // the token can sit after a colon (`org/model:mtp`), a dash or an underscore + const name = modelId.toLowerCase(); + const match = SIDECAR_TOKENS.find((token) => + new RegExp(`(^|[-_:])${token}([-_.:]|$)`).test(name) + ); + + return (match as ModelSidecar | undefined) ?? null; +} + +/** Parameter label a parsed id reports, e.g. `35B-A3B`. */ +function paramsLabel(parsed: ParsedModelId): string | null { + if (!parsed.params) return null; + + return `${parsed.params}${parsed.activatedParams ? `-${parsed.activatedParams}` : ''}`; +} + export class ModelsService { private static readonly SSE_RECONNECT_MS = 1000; @@ -122,6 +148,10 @@ export class ModelsService { return model.status.value === ServerModelStatus.LOADED; } + static isModelLoading(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADING; + } + /** * * @@ -130,8 +160,24 @@ export class ModelsService { * */ - static isModelLoading(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADING; + /** + * True when a router entry id marks a downloaded sidecar file, e.g. + * `org/model:Q4_0-mtp` or `org/model:mmproj`, not a loadable model. + */ + static isSidecarEntry(modelId: string): boolean { + const idx = modelId.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR); + + if (idx !== MODEL_ID.NOT_FOUND && sidecarFromTag(modelId.slice(idx + 1))) return true; + + // the router also lists projector and draft files by filename, e.g. + // `org/model-mmproj-F16.gguf` or `mmproj-model.gguf` + const name = modelId.split('/').pop() ?? modelId; + + return ( + MODEL_ID.SIDECAR_INFIX_REGEX.test(name) || + MODEL_ID.SIDECAR_PREFIX_REGEX.test(name) || + MODEL_ID.SIDECAR_SUFFIX_REGEX.test(name) + ); } /** @@ -141,7 +187,7 @@ export class ModelsService { * @returns List of available models with basic metadata */ static async list(): Promise { - return apiFetch(API_MODELS.LIST); + return apiFetch(apiModelsUrl()); } /** @@ -151,16 +197,21 @@ export class ModelsService { * * @param modelId - Model identifier to load * @param extraArgs - Optional additional arguments to pass to the model instance + * @param backendId - Backend serving the model; the active one when omitted * @returns Load response from the server */ - static async load(modelId: string, extraArgs?: string[]): Promise { + static async load( + modelId: string, + extraArgs?: string[], + backendId?: string + ): Promise { const payload: { model: string; extra_args?: string[] } = { model: modelId }; if (extraArgs && extraArgs.length > 0) { payload.extra_args = extraArgs; } - return apiPost(API_MODELS.LOAD, payload); + return apiPost(API_MODELS.LOAD, payload, { backendId }); } /** @@ -334,16 +385,57 @@ export class ModelsService { return result; } + /** + * Sidecar files a listing carries as their own entries, keyed by the repo + * they belong to. The router lists a downloaded sidecar as a model of its + * own, so this keeps the pairing that the model list itself is filtered to + * drop. + */ + static sidecarFilesByRepo(response: ApiModelsListResponse): Record { + const byRepo: Record = {}; + + for (const entry of response.data ?? []) { + const parsed = ModelsService.parseModelId(entry.id); + const sidecar = parsed.sidecar ?? sidecarTokenInFilename(entry.id); + + if (!sidecar || isAuxSidecar(sidecar)) continue; + + // the repo is the id with its quant tag and sidecar token taken off; naming + // parts of the id are not touched, a model name may carry `-4b` for instance + const model = entry.id + .split(MODEL_ID.QUANTIZATION_SEPARATOR)[0] + .replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, '') + .replace(new RegExp(`[-_ ]?${sidecar}([-_ ]?draft)?$`, 'i'), ''); + + if (!model) continue; + + const files = (byRepo[model] ??= []); + + if (files.some((file) => file.kind === sidecar)) continue; + + files.push({ + id: entry.id, + kind: sidecar, + model, + params: paramsLabel(parsed), + quant: parsed.quantization + }); + } + + return byRepo; + } + /** * Unload a model (ROUTER mode only). * Sends POST request to `/models/unload`. Note: the endpoint returns success * before unloading completes — use polling to await actual unload status. * * @param modelId - Model identifier to unload + * @param backendId - Backend serving the model; the active one when omitted * @returns Unload response from the server */ - static async unload(modelId: string): Promise { - return apiPost(API_MODELS.UNLOAD, { model: modelId }); + static async unload(modelId: string, backendId?: string): Promise { + return apiPost(API_MODELS.UNLOAD, { model: modelId }, { backendId }); } /** @@ -360,8 +452,10 @@ export class ModelsService { while (!signal.aborted) { try { - const response = await fetch(withBase(API_MODELS.SSE), { - headers: getAuthHeaders(), + // the status feed only exists on the local llama.cpp server; pin the + // request so an active external backend cannot redirect it + const response = await fetch(apiUrl(API_MODELS.SSE, LOCAL_BACKEND_ID), { + headers: getAuthHeaders(LOCAL_BACKEND_ID), signal }); diff --git a/ui/apps/chat/src/lib/services/props.service.ts b/ui/apps/chat/src/lib/services/props.service.ts index 4a2f4e884799..097a8df4667e 100644 --- a/ui/apps/chat/src/lib/services/props.service.ts +++ b/ui/apps/chat/src/lib/services/props.service.ts @@ -15,17 +15,21 @@ export class PropsService { * In ROUTER mode, returns server-wide settings without model-specific modalities. * * @param autoload - If false, prevents automatic model loading (default: false) + * @param backendId - Backend to ask; defaults to the active one. * @returns Server properties including default generation settings and capabilities * @throws {Error} If the request fails or returns invalid data */ - static async fetch(autoload = false): Promise { + static async fetch(autoload = false, backendId?: string): Promise { const params: Record = {}; if (!autoload) { params.autoload = 'false'; } - return apiFetchWithParams('./props', params, { authOnly: true }); + return apiFetchWithParams('./props', params, { + authOnly: true, + backendId + }); } /** diff --git a/ui/apps/chat/src/lib/services/protocols/index.ts b/ui/apps/chat/src/lib/services/protocols/index.ts new file mode 100644 index 000000000000..9fab9c5c3670 --- /dev/null +++ b/ui/apps/chat/src/lib/services/protocols/index.ts @@ -0,0 +1,24 @@ +/** + * Protocol adapter registry. + * + * Resolves the wire mapping for a backend by its protocol. Unknown or + * not-yet-loaded backends fall back to the OpenAI-compatible adapter, which is + * what the local llama-server speaks. + */ + +import { openaiAdapter } from './openai'; +import type { ChatProtocolAdapter } from './types'; +import type { Backend, BackendProtocol } from '#lib/types/index.ts'; + +const ADAPTERS: Record = { + 'llama.cpp': openaiAdapter, + openai: openaiAdapter +}; + +export function getProtocolAdapter(backend?: Backend): ChatProtocolAdapter { + if (!backend) return openaiAdapter; + + return ADAPTERS[backend.protocol] ?? openaiAdapter; +} + +export type { ChatProtocolAdapter, ChatStreamEvent, ChatStreamReader } from './types'; diff --git a/ui/apps/chat/src/lib/services/protocols/openai.ts b/ui/apps/chat/src/lib/services/protocols/openai.ts new file mode 100644 index 000000000000..809f01fa3952 --- /dev/null +++ b/ui/apps/chat/src/lib/services/protocols/openai.ts @@ -0,0 +1,185 @@ +/** + * OpenAI-compatible protocol: llama-server, hosted OpenAI endpoints and any + * compatible gateway. llama-server accepts a superset of the wire format, so + * its only specialization is that nothing gets stripped. + */ + +import type { ChatProtocolAdapter, ChatStreamEvent, ChatStreamReader } from './types'; +import { HEADERS } from '#lib/constants/index.ts'; +import type { ApiChatCompletionStreamChunk } from '#lib/types/api.js'; +import type { Backend } from '#lib/types/index.ts'; +import { getBackendCompat } from '#lib/utils/backend.ts'; + +/** + * llama.cpp-only chat request fields. Strict OpenAI-compatible endpoints + * reject unknown parameters, so they are dropped for those backends. + */ +const COMPAT_ONLY_OMIT_REQUEST_FIELDS = [ + 'add_generation_prompt', + 'backend_sampling', + 'cache_prompt', + 'chat_template_kwargs', + 'continue_final_message', + 'dry_allowed_length', + 'dry_base', + 'dry_multiplier', + 'dry_penalty_last_n', + 'dynatemp_exponent', + 'dynatemp_range', + 'id_slot', + 'min_p', + 'n_keep', + 'n_predict', + 'reasoning_control', + 'reasoning_format', + 'repeat_last_n', + 'repeat_penalty', + 'return_progress', + 'samplers', + 'sse_ping_interval', + 'thinking_budget_tokens', + 'timings_per_token', + 'top_k', + 'typ_p', + 'xtc_probability', + 'xtc_threshold' +]; + +function authHeaders(backend: Backend): Record { + const headers: Record = { ...(backend.headers ?? {}) }; + const apiKey = backend.apiKey?.trim(); + + if (apiKey) { + headers[HEADERS.AUTHORIZATION] = `${HEADERS.BEARER}${apiKey}`; + } + + return headers; +} + +function buildChatRequest( + body: Record, + backend: Backend +): Record { + if (backend.protocol === 'llama.cpp') return body; + + const compat = getBackendCompat(backend); + const request: Record = { ...body }; + + for (const field of COMPAT_ONLY_OMIT_REQUEST_FIELDS) { + delete request[field]; + } + + // compatible endpoints reject the reasoning_content message extension + const messages = request.messages as { reasoning_content?: string }[] | undefined; + + for (const message of messages ?? []) { + delete message.reasoning_content; + } + + // -1 is llama.cpp's "no limit" sentinel; compatible endpoints reject it + if (typeof request.max_tokens === 'number' && request.max_tokens <= 0) { + delete request.max_tokens; + } + + // newer OpenAI models require max_completion_tokens, most compatible + // endpoints only understand max_tokens + if (compat.maxTokensField === 'max_completion_tokens' && request.max_tokens !== undefined) { + request.max_completion_tokens = request.max_tokens; + delete request.max_tokens; + } + + // a final usage chunk is what the client side timing fallback reads + if (request.stream && compat.supportsUsageInStreaming && request.stream_options === undefined) { + request.stream_options = { include_usage: true }; + } + + return request; +} + +/** + * Model name a payload reports. Streaming chunks carry it on the delta, final + * responses on the message, and some gateways on metadata or the choice itself. + */ +export function extractModelName(data: unknown): string | undefined { + const asRecord = (value: unknown): Record | undefined => + typeof value === 'object' && value !== null ? (value as Record) : undefined; + const getTrimmedString = (value: unknown): string | undefined => + typeof value === 'string' && value.trim() ? value.trim() : undefined; + const root = asRecord(data); + + if (!root) return undefined; + + const rootModel = getTrimmedString(root.model); + + if (rootModel) return rootModel; + + const firstChoice = Array.isArray(root.choices) ? asRecord(root.choices[0]) : undefined; + + if (!firstChoice) return undefined; + + const metadataModel = getTrimmedString(asRecord(firstChoice.metadata)?.model); + + if (metadataModel) return metadataModel; + + const deltaModel = getTrimmedString(asRecord(firstChoice.delta)?.model); + + if (deltaModel) return deltaModel; + + const messageModel = getTrimmedString(asRecord(firstChoice.message)?.model); + + if (messageModel) return messageModel; + + return getTrimmedString(firstChoice.model); +} + +function readChunk(payload: unknown): ChatStreamEvent[] { + if (!payload || typeof payload !== 'object') return []; + + const chunk = payload as ApiChatCompletionStreamChunk; + const events: ChatStreamEvent[] = []; + const model = extractModelName(chunk); + + if (chunk.id) events.push({ id: chunk.id, type: 'id' }); + + if (model) events.push({ model, type: 'model' }); + + if (chunk.prompt_progress) { + events.push({ progress: chunk.prompt_progress, type: 'prompt_progress' }); + } + + if (chunk.timings) { + events.push({ + promptProgress: chunk.prompt_progress, + timings: chunk.timings, + type: 'timings' + }); + } + + if (chunk.usage) events.push({ type: 'usage', usage: chunk.usage }); + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (choice?.finish_reason) events.push({ reason: choice.finish_reason, type: 'finish_reason' }); + + if (!delta) return events; + + if (delta.content) events.push({ text: delta.content, type: 'text' }); + + // endpoints disagree on the reasoning field name; take the first one set + const reasoning = delta.reasoning_content ?? delta.reasoning ?? delta.reasoning_text; + + if (reasoning) events.push({ text: reasoning, type: 'thinking' }); + + if (delta.tool_calls) events.push({ deltas: delta.tool_calls, type: 'tool_calls' }); + + return events; +} + +export const openaiAdapter: ChatProtocolAdapter = { + authHeaders, + buildChatRequest, + createStreamReader(): ChatStreamReader { + return { readChunk }; + } +}; diff --git a/ui/apps/chat/src/lib/services/protocols/types.ts b/ui/apps/chat/src/lib/services/protocols/types.ts new file mode 100644 index 000000000000..54f05e931121 --- /dev/null +++ b/ui/apps/chat/src/lib/services/protocols/types.ts @@ -0,0 +1,43 @@ +/** + * Backend protocol adapters. + * + * A backend speaks one wire protocol. ChatService owns the transport (fetch, + * SSE framing, resume offsets) and delegates the parts that differ per + * protocol here: credential headers, request shaping and stream decoding. + * + * Decoding is per-stream: a reader keeps per-stream state, so it must not be + * shared between concurrent requests. + */ + +import type { ApiChatCompletionToolCallDelta, ApiChatCompletionUsage } from '#lib/types/api.js'; +import type { ChatMessagePromptProgress, ChatMessageTimings } from '#lib/types/chat.js'; +import type { Backend } from '#lib/types/index.ts'; + +/** One canonical delta decoded from a backend's stream payload. */ +export type ChatStreamEvent = + | { type: 'done' } + | { type: 'error'; message: string } + | { type: 'finish_reason'; reason: string } + | { type: 'id'; id: string } + | { type: 'model'; model: string } + | { type: 'prompt_progress'; progress: ChatMessagePromptProgress } + | { type: 'text'; text: string } + | { type: 'thinking'; text: string } + | { type: 'timings'; timings: ChatMessageTimings; promptProgress?: ChatMessagePromptProgress } + | { type: 'tool_calls'; deltas: ApiChatCompletionToolCallDelta[] } + | { type: 'usage'; usage: ApiChatCompletionUsage }; + +/** Decodes one stream's payloads. Create one per request. */ +export interface ChatStreamReader { + /** Canonical deltas for one parsed SSE payload. */ + readChunk(payload: unknown): ChatStreamEvent[]; +} + +/** Wire mapping for one protocol. */ +export interface ChatProtocolAdapter { + /** Credential and protocol-required headers for a backend. */ + authHeaders(backend: Backend): Record; + /** Rewrite the canonical request body into this protocol's wire body. */ + buildChatRequest(body: Record, backend: Backend): Record; + createStreamReader(): ChatStreamReader; +} diff --git a/ui/apps/chat/src/lib/services/tools.service.ts b/ui/apps/chat/src/lib/services/tools.service.ts index dd401f79d34d..7cf585a81bd8 100644 --- a/ui/apps/chat/src/lib/services/tools.service.ts +++ b/ui/apps/chat/src/lib/services/tools.service.ts @@ -5,12 +5,17 @@ * No reactive state; consumed by toolsStore. */ -import { API_TOOLS, HEADERS, HOME_TILDE, SEARCH } from '#lib/constants/index.ts'; +import { API_TOOLS, HEADERS, HOME_TILDE, LOCAL_BACKEND_ID, SEARCH } from '#lib/constants/index.ts'; import { BuiltInTool, GlobSearchType, ToolResponseField } from '#lib/enums/index.ts'; import type { GlobEntry, ServerToolInfo, ToolExecutionResult } from '#lib/types/index.ts'; import { getJsonHeaders } from '#lib/utils/api-headers.ts'; -import { withBase } from '#lib/utils/base-path.ts'; -import { apiFetch, buildCaseInsensitiveGlob, joinPath, lastPathSegment } from '#lib/utils/index.ts'; +import { + apiFetch, + apiUrl, + buildCaseInsensitiveGlob, + joinPath, + lastPathSegment +} from '#lib/utils/index.ts'; import { parseSseJsonStream, type SseJsonEvent } from '#lib/utils/sse.ts'; export class ToolsService { @@ -28,6 +33,7 @@ export class ToolsService { cwd?: string ): Promise { const result = await apiFetch>(API_TOOLS.EXECUTE, { + backendId: LOCAL_BACKEND_ID, body: JSON.stringify({ params, tool: toolName }), headers: cwd ? { [HEADERS.X_TOOL_CWD_HEADER]: cwd } : undefined, method: 'POST', @@ -67,6 +73,7 @@ export class ToolsService { if (respType) headers[HEADERS.X_RESP_TYPE_HEADER] = respType; return apiFetch>(API_TOOLS.EXECUTE, { + backendId: LOCAL_BACKEND_ID, body: JSON.stringify({ params, tool: toolName }), headers: Object.keys(headers).length > 0 ? headers : undefined, method: 'POST', @@ -80,7 +87,8 @@ export class ToolsService { * @returns Array of tool definitions in OpenAI-compatible format */ static async list(): Promise { - return apiFetch(API_TOOLS.LIST); + // pinned to the local server: server tools do not exist on other backends + return apiFetch(API_TOOLS.LIST, { backendId: LOCAL_BACKEND_ID }); } /** @@ -133,11 +141,11 @@ export class ToolsService { signal?: AbortSignal, cwd?: string ): AsyncGenerator { - const headers = getJsonHeaders(); + const headers = getJsonHeaders(LOCAL_BACKEND_ID); if (cwd) headers[HEADERS.X_TOOL_CWD_HEADER] = cwd; - const response = await fetch(withBase(API_TOOLS.EXECUTE), { + const response = await fetch(apiUrl(API_TOOLS.EXECUTE, LOCAL_BACKEND_ID), { body: JSON.stringify({ params, stream: true, tool: toolName }), headers, method: 'POST', diff --git a/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts b/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts index e13b24860057..ea4ec0c7060f 100644 --- a/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts +++ b/ui/apps/chat/src/lib/stores/backendsModels.svelte.ts @@ -90,7 +90,7 @@ class BackendsModelsStore { const result = await BackendsService.listModels(backend); this.states[backend.id] = { - drafts: result.raw ? ModelsService.draftSidecarsByRepo(result.raw) : undefined, + drafts: result.raw ? ModelsService.sidecarFilesByRepo(result.raw) : undefined, error: result.error ?? null, loaded: result.ok, loading: false, diff --git a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts index 73c8d21d64e0..78c54b0e2760 100644 --- a/ui/apps/chat/src/lib/stores/chat/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/chat/index.svelte.ts @@ -451,6 +451,11 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { if (serverStore.isRouterMode) { const modelName = modelsStore.selectedModelName; + if (modelName) apiOptions.model = modelName; + } else if (!serverStore.capabilities.props) { + // external backends need an explicit model on every request + const modelName = modelsStore.activeModelId; + if (modelName) apiOptions.model = modelName; } @@ -956,6 +961,15 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { modelOverride?: string | null, firstUserMessageContent?: string ): Promise { + // a conversation keeps the model that generated it, which can belong to + // another backend; make that backend active so the request is not sent to + // a server that does not serve the model + const requestedModel = modelOverride ?? getConversationModel(allMessages); + + if (requestedModel) { + await modelsStore.ensureModelBackend(requestedModel); + } + // the ::model suffix in the stream identity is only for router mode, where it routes to the // owning child. in single-model mode the identity stays the bare conv id so that attach, stop // and reattach all agree, regardless of fresh send vs regenerate passing a resolved model @@ -965,6 +979,9 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { const conversationModel = getConversationModel(allMessages); effectiveModel = modelOverride || modelsStore.selectedModelName || conversationModel; + } else if (!serverStore.capabilities.props) { + // external backends are not router mode but still require a model + effectiveModel = modelOverride || modelsStore.activeModelId; } // a router serves the props the request options read once the model is loaded, so the @@ -1443,8 +1460,8 @@ class ChatStore implements ChatStreamHost, ChatFlowsHost { convId: string ): Promise { const effectiveModel = - serverStore.isRouterMode && modelsStore.selectedModelName - ? modelsStore.selectedModelName + serverStore.isRouterMode || !serverStore.capabilities.props + ? (modelsStore.activeModelId ?? undefined) : undefined; const configValue = settingsStore.config; const titlePromptTemplate = diff --git a/ui/apps/chat/src/lib/stores/index.ts b/ui/apps/chat/src/lib/stores/index.ts index 5b011b0fcf2d..eeb28f8aad60 100644 --- a/ui/apps/chat/src/lib/stores/index.ts +++ b/ui/apps/chat/src/lib/stores/index.ts @@ -35,6 +35,10 @@ export { agenticStore } from './agentic/index.svelte'; export { sidebarDragStore } from './sidebar-drag.svelte'; export { conversationsStore } from './conversations/index.svelte'; +// BACKENDS +export { backendsStore } from './backends.svelte'; +export { backendsModelsStore } from './backendsModels.svelte'; + // GROUPS (named sidebar sections for chats) export { groupsStore } from './groups.svelte'; diff --git a/ui/apps/chat/src/lib/stores/init.ts b/ui/apps/chat/src/lib/stores/init.ts index 532abfeb352a..5178864cc4d7 100644 --- a/ui/apps/chat/src/lib/stores/init.ts +++ b/ui/apps/chat/src/lib/stores/init.ts @@ -1,6 +1,10 @@ // direct imports, not via the barrel, to avoid circular deps +import { backendsStore } from './backends.svelte'; +import { backendsModelsStore } from './backendsModels.svelte'; import { conversationsStore } from './conversations/index.svelte'; import { groupsStore } from './groups.svelte'; +import { serverStore } from './llama-server.svelte'; +import { modelsStore } from './models/index.svelte'; import { permissionsStore } from './permissions.svelte'; import { projectsStore } from './projects.svelte'; import { settingsStore } from './settings/index.svelte'; @@ -13,6 +17,8 @@ import { MigrationService } from '#lib/services/migration.service.ts'; import { validateApiKey } from '#lib/utils/api-key-validation.ts'; import { browser } from '$app/env'; +let hydration: Promise | null = null; +let extensionsStarted: Promise | null = null; let startup: Promise | null = null; /** Ask the browser not to evict conversations and settings under storage pressure, best effort. */ @@ -26,22 +32,68 @@ async function requestPersistentStorage(): Promise { } } +/** + * Read the stored state the first request needs: migrations, the settings that + * carry the API key, and the backend that request goes to. + * + * Separate from {@link initStores} so a `load` can await it. The requests + * `initStores` starts belong after a load, where plain `window.fetch` is the + * intended one and SvelteKit raises no warning. + */ +export function hydrateStores(): Promise { + if (!browser) return Promise.resolve(); + + hydration ??= (async () => { + await MigrationService.runAllMigrations(); + + settingsStore.initialize(); + backendsStore.initialize(); + })(); + + return hydration; +} + +/** + * Register the built-in extensions and activate the ones that are switched + * on, once. The startup does it; the load of an extension's page does too, + * as it runs before the startup and needs the page to be there. + */ +export function startExtensions(): Promise { + if (!browser) return Promise.resolve(); + + extensionsStarted ??= (async () => { + // extensions start once the settings they read are in place + await hydrateStores(); + + for (const manifest of BUILT_IN_EXTENSIONS) extensions.register(manifest); + + await extensions.activateEnabled(); + })(); + + return extensionsStarted; +} + export function initStores(): Promise { if (!browser) return Promise.resolve(); startup ??= (async () => { - await MigrationService.runAllMigrations(); + await hydrateStores(); void requestPersistentStorage(); - settingsStore.initialize(); + // prefetch every backend's model list in the background; failures are + // per-backend and never block startup + void backendsModelsStore.loadAll().then(() => modelsStore.warmHubDetails()); + permissionsStore.initialize(); toolsStore.initialize(); - // extensions start once the settings they read are in place - for (const manifest of BUILT_IN_EXTENSIONS) extensions.register(manifest); + // the local server state backs the installation facts and decides whether + // /tools exists at all, so probe it first and only then list the tools; + // otherwise they stay empty until the tools menu is opened + void serverStore.prefetchLocalState().then(() => toolsStore.fetchServerTools()); - await extensions.activateEnabled(); + await startExtensions(); void versionStore.initialize(); void groupsStore.initialize(); @@ -57,10 +109,11 @@ export function initStores(): Promise { return startup; } -/** Load shared by the app's pages: start the stores, then check the API key. */ +/** Load shared by the app's pages: read the stored state, then check the API key. */ export async function loadAppRoute(fetch: typeof globalThis.fetch): Promise { // loads run before the root layout script, so the stored API key reaches - // the probe only once the settings store has read localStorage - await initStores(); + // the probe only once the settings store has read localStorage; the rest of + // the startup belongs to the layout, after this load + await hydrateStores(); await validateApiKey(fetch); } diff --git a/ui/apps/chat/src/lib/stores/llama-server.svelte.ts b/ui/apps/chat/src/lib/stores/llama-server.svelte.ts index 84968528f359..e5485ac8931d 100644 --- a/ui/apps/chat/src/lib/stores/llama-server.svelte.ts +++ b/ui/apps/chat/src/lib/stores/llama-server.svelte.ts @@ -6,8 +6,12 @@ * PropsService for the /props fetch. */ +import { BACKEND_CAPABILITIES, LOCAL_BACKEND_ID } from '#lib/constants/index.ts'; import { ServerRole } from '#lib/enums/index.ts'; import { PropsService } from '#lib/services/props.service.ts'; +import type { BackendCapabilities } from '#lib/types/index.ts'; +import { getBackend } from '#lib/utils/api-base.ts'; +import { getBackendCapabilities } from '#lib/utils/backend.ts'; import { ApiError } from '#lib/utils/index.ts'; const LOADING_RETRY_INTERVAL_MS = 1000; @@ -18,9 +22,25 @@ class ServerStore { props = $state(null); role = $state(null); status = $state(null); + private fetchBackendId: string | undefined; private fetchPromise: Promise | null = null; + /** + * Whether a local llama.cpp server answered. Null until the first probe + * resolves: the bundled build is served by one, the hosted PWA is not. + */ + private localAvailable: boolean | null = null; + /** Local server state kept alive while an external backend is active. */ + private localState: { props: ApiLlamaCppServerProps | null; role: ServerRole | null } | null = + null; private retryTimer: ReturnType | null = null; + /** Features of the active backend. Defaults to full llama.cpp support. */ + get capabilities(): BackendCapabilities { + const backend = getBackend(); + + return backend ? getBackendCapabilities(backend) : BACKEND_CAPABILITIES['llama.cpp']; + } + get contextSize(): number | null { const nCtx = this.props?.default_generation_settings?.n_ctx; @@ -28,7 +48,12 @@ class ServerStore { } get defaultParams(): ApiLlamaCppServerProps['default_generation_settings']['params'] | null { - return this.props?.default_generation_settings?.params || null; + return this.localProps?.default_generation_settings?.params || null; + } + + /** A local llama.cpp server answered, either now or earlier this session. */ + get hasLocalServer(): boolean { + return this.localAvailable === true; } get isModelMode(): boolean { @@ -39,10 +64,52 @@ class ServerStore { return this.role === ServerRole.ROUTER; } + /** + * Local server state. The local server is the base of the installation, so + * its props and role come from the mirror while another backend is active + * instead of following the selected model. + */ + private get localFacts(): { props: ApiLlamaCppServerProps | null; role: ServerRole | null } { + // no resolvable backend (early startup, non-browser call): props belong to + // the origin serving this UI, which is the local server + const backendId = getBackend()?.id ?? LOCAL_BACKEND_ID; + + if (backendId === LOCAL_BACKEND_ID) return { props: this.props, role: this.role }; + + return this.localState ?? { props: null, role: null }; + } + + /** The local server runs as a router, independent of the active backend. */ + get localIsRouter(): boolean { + return this.localFacts.role === ServerRole.ROUTER; + } + + /** Props of the local server, from the active backend or the cache. */ + get localProps(): ApiLlamaCppServerProps | null { + return this.localFacts.props; + } + + /** The local server is known to be unreachable: local-only UI stays out. */ + get localServerMissing(): boolean { + return this.localAvailable === false; + } + get uiSettings(): Record | undefined { return this.props?.ui_settings ?? this.props?.webui_settings; } + /** + * Keep the local server state before switching to an external backend, so + * switching back restores it instead of asking the server again. + */ + cacheLocalState(): void { + // props only exist while a llama.cpp server is active; an external to + // external switch must not overwrite the kept local state with blanks + if (!this.props) return; + + this.localState = { props: this.props, role: this.role }; + } + clear(): void { this.clearRetryTimer(); this.props = null; @@ -51,6 +118,7 @@ class ServerStore { this.loading = false; this.role = null; this.fetchPromise = null; + this.fetchBackendId = undefined; } /** @@ -59,10 +127,24 @@ class ServerStore { * splash and the chat screen every retry tick. */ async fetch({ background = false }: { background?: boolean } = {}): Promise { - if (this.fetchPromise) return this.fetchPromise; + // props and role describe one server. a fetch started for another backend + // must not be reused, and its response must not commit once the active + // backend has changed while it was in flight + const backendId = getBackend()?.id; + + if (this.fetchPromise && this.fetchBackendId === backendId) return this.fetchPromise; this.clearRetryTimer(); + // External backends expose no /props endpoint. Keep MODEL-mode defaults so + // role detection and generation defaults degrade instead of failing. + if (!this.capabilities.props) { + this.clear(); + this.role = ServerRole.MODEL; + + return; + } + if (!background) { this.loading = true; } @@ -73,19 +155,33 @@ class ServerStore { this.error = null; } - const fetchPromise = (async () => { + const promise = (async () => { try { const props = await PropsService.fetch(); + // the active backend changed while this request was in flight + if (getBackend()?.id !== backendId) return; + this.props = props; this.error = null; this.status = null; this.detectRole(props); + + if (backendId === LOCAL_BACKEND_ID) { + this.localAvailable = true; + // mirror the local facts as they arrive: a later clear() must not + // be able to wipe what the local server told us + this.localState = { props, role: this.role }; + } } catch (error: unknown) { + if (getBackend()?.id !== backendId) return; + this.error = error instanceof Error ? error.message : String(error); this.status = error instanceof ApiError ? error.status : null; console.error('Error fetching server properties:', error); + if (backendId === LOCAL_BACKEND_ID) this.localAvailable = false; + if (this.status === 503) { this.scheduleRetry(); } @@ -93,13 +189,56 @@ class ServerStore { if (!background) { this.loading = false; } - - this.fetchPromise = null; } })(); - this.fetchPromise = fetchPromise; - await fetchPromise; + this.fetchPromise = promise; + this.fetchBackendId = backendId; + + // a backend switch clears the in-flight handle; only the fetch that is + // still the current one may release it + void promise + .catch(() => {}) + .finally(() => { + if (this.fetchPromise === promise) { + this.fetchPromise = null; + this.fetchBackendId = undefined; + } + }); + + await promise; + } + + /** + * Load the local server state in the background at startup. The local tab + * then opens from memory instead of asking for props on the first click. + */ + async prefetchLocalState(): Promise { + if (this.localState) return; + + try { + const props = await PropsService.fetch(false, LOCAL_BACKEND_ID); + + this.localState = { + props, + role: props?.role === ServerRole.ROUTER ? ServerRole.ROUTER : ServerRole.MODEL + }; + this.localAvailable = true; + } catch { + // no local server in this deployment: mark it missing so the local + // backend drops out of the enabled list + this.localAvailable = false; + } + } + + /** Restore the state kept by {@link cacheLocalState}; no request is made. */ + restoreLocalState(): void { + if (!this.localState) return; + + this.props = this.localState.props; + this.role = this.localState.role; + this.error = null; + this.status = null; } private clearRetryTimer(): void { diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts index 7928defaad30..040d84e83195 100644 --- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts @@ -10,24 +10,30 @@ import { FAVORITE_MODELS_LOCALSTORAGE_KEY, HIDDEN_MODELS_LOCALSTORAGE_KEY, + LOCAL_BACKEND_ID, MODEL_GROUP_OPEN_LOCALSTORAGE_KEY, MODEL_REMOVE_BLOCKED_FALLBACK, MODEL_REMOVE_BLOCKED_REASONS, RECENT_MODEL_LIMIT, - RECENT_MODELS_LOCALSTORAGE_KEY + RECENT_MODELS_LOCALSTORAGE_KEY, + SELECTED_MODEL_LOCALSTORAGE_KEY } from '#lib/constants/index.ts'; import { ReasoningEffort, ServerModelSource, ServerModelStatus } from '#lib/enums/index.ts'; import { HuggingFaceService } from '#lib/services/huggingface.service.ts'; import { ModelsService } from '#lib/services/models.service.ts'; // direct imports between stores, not via the barrel, to avoid circular deps +import { backendsStore } from '#lib/stores/backends.svelte.ts'; +import { backendsModelsStore } from '#lib/stores/backendsModels.svelte.ts'; import { conversationsStore } from '#lib/stores/conversations/index.svelte.ts'; import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import { type ModelPropsHost, ModelPropsManager } from '#lib/stores/models/props.svelte.ts'; import { type ModelStatusHost, ModelStatusManager } from '#lib/stores/models/status.svelte.ts'; import type { ApiModelDataEntry } from '#lib/types/index.ts'; import type { ParsedModelId } from '#lib/types/models.js'; +import { getBackendCapabilities, readModelContextLength } from '#lib/utils/backend.ts'; import { getConversationModel } from '#lib/utils/conversation-utils.ts'; import { repoOf } from '#lib/utils/model-names.ts'; +import { backendIdFromModelId, qualifyModelId, rawModelId } from '#lib/utils/model-option-id.ts'; import { browser } from '$app/env'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { toast } from 'svelte-sonner'; @@ -41,6 +47,27 @@ const MAX_GROUP_OPEN_ENTRIES = 200; */ const HUB_WARM_LIMIT = 12; +/** Selection kept from the last session, so a reload does not drop the picked model. */ +function loadStoredSelection(): { id: string; model: string | null } | null { + if (!browser) return null; + + try { + const raw = localStorage.getItem(SELECTED_MODEL_LOCALSTORAGE_KEY); + + if (!raw) return null; + + const parsed = JSON.parse(raw) as { id?: unknown; model?: unknown }; + + if (typeof parsed?.id !== 'string' || !parsed.id) return null; + + return { id: parsed.id, model: typeof parsed.model === 'string' ? parsed.model : null }; + } catch { + return null; + } +} + +const storedSelection = loadStoredSelection(); + /** Models kept out of the selector. */ function loadHiddenModels(): Set { if (!browser) return new SvelteSet(); @@ -92,16 +119,59 @@ function loadRecentModels(): string[] { } class ModelsStore implements ModelPropsHost, ModelStatusHost { + activeModels = $state([]); error = $state(null); favoriteModelIds = $state>(this.loadFavoritesFromStorage()); groupOpenState = $state>(loadGroupOpenState()); hiddenModelIds = $state>(loadHiddenModels()); loading = $state(false); - models = $state([]); + /** + * Every selectable model across enabled backends. The active backend's + * models come from {@link activeModels}; the rest come from the background + * prefetch cache. Ids are backend-qualified so the same model name on two + * backends stays distinct. + */ + /** + * Computed once per state change. Rows reach this through per-model props + * lookups, so a getter that rebuilt the list on every read made opening the + * selector quadratic in the size of the catalog. + */ + models = $derived.by((): ModelOption[] => { + const activeBackendId = backendsStore.active.id; + const merged: ModelOption[] = []; + const seen = new SvelteSet(); + const push = (option: ModelOption, backendId: string) => { + const id = qualifyModelId(backendId, rawModelId(option.id)); + + // a backend can be listed twice while a switch is in flight: the rows + // of the previous backend are still in activeModels + if (seen.has(id)) return; + + seen.add(id); + merged.push({ ...option, backendId, id }); + }; + + for (const option of this.activeModels) { + // keep the backend an option was built for: rows from the previous + // backend must not be relabelled while a switch is in flight + push(option, option.backendId ?? activeBackendId); + } + + for (const backend of backendsStore.enabled) { + if (backend.id === activeBackendId) continue; + + for (const option of backendsModelsStore.get(backend.id).models) { + push(option, option.backendId ?? backend.id); + } + } + + return merged; + }); recentModelIds = $state(loadRecentModels()); routerModels = $state([]); - selectedModelId = $state(null); - selectedModelName = $state(null); + selectedModelId = $state(storedSelection?.id ?? null); + + selectedModelName = $state(storedSelection?.model ?? null); updating = $state(false); @@ -115,6 +185,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { // Without this, ?model= URL handler races an in-progress fetch and sees an empty list. private inflightFetch: Promise | null = null; + /** A restored selection loses to the active conversation's model, a fresh pick does not. */ + private selectionFromStorage = storedSelection !== null; + /** * Model the active conversation view resolves to. Router mode: the user's * selection first, then the conversation's own model. Otherwise the single @@ -122,14 +195,20 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { */ get activeModelId(): string | null { if (!serverStore.isRouterMode) { + // external backends expose a selectable list; prefer the user's pick + const selected = this.selectedModelId + ? this.models.find((m) => m.id === this.selectedModelId) + : undefined; + + if (selected) return selected.model; + return this.models.length > 0 ? this.models[0].model : this.singleModelName; } - if (this.selectedModelId) { - const selected = this.models.find((m) => m.id === this.selectedModelId); + const picked = this.selectedModelId && !this.selectionFromStorage ? this.selectedModelId : null; + const selected = picked ? this.models.find((m) => m.id === picked) : undefined; - if (selected) return selected.model; - } + if (selected) return selected.model; const conversationModel = getConversationModel(conversationsStore.activeMessages); @@ -139,7 +218,11 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { if (model) return model.model; } - return null; + const restored = this.selectedModelId + ? this.models.find((m) => m.id === this.selectedModelId) + : undefined; + + return restored?.model ?? null; } get loadedModelIds(): string[] { @@ -192,6 +275,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { clearSelection(): void { this.selectedModelId = null; this.selectedModelName = null; + this.persistSelection(); } /** Family keys folded away under one section, for a list that restores them. */ @@ -281,6 +365,19 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { await this.selectModelById(availableModels[0].id); } + /** + * Make the backend serving `modelName` active when it is not already. + * A conversation keeps the model that generated it, which can belong to a + * backend other than the active one. + */ + async ensureModelBackend(modelName: string): Promise { + const option = this.models.find((model) => model.model === modelName); + + if (!option?.backendId || option.backendId === backendsStore.active.id) return; + + await this.selectModelById(option.id); + } + /** * Fetch list of models from server and detect server role. * Also fetches modalities for MODEL mode (single model). @@ -288,7 +385,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { async fetch(force = false): Promise { if (this.inflightFetch) return this.inflightFetch; - if (this.models.length > 0 && !force) return; + if (this.activeModels.length > 0 && !force) return; this.inflightFetch = this.runFetch(); try { @@ -315,7 +412,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { this._status.syncPausedDownloads(); // keep the selector options in sync: a downloaded / deleted model shows // up here too, not only in the router model rows - this.models = this.buildModelOptions(response); + this.activeModels = this.buildModelOptions(response); this.warmHubDetails(); await this.props.fetchModalitiesForLoadedModels(); @@ -375,7 +472,19 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return null; } + /** + * Load state a model's own backend reports. Only the local server has a status + * feed, so an external backend answers from its own model listing. + */ getModelStatus(modelId: string): ServerModelStatus | null { + const backendId = this.models.find((model) => model.model === modelId)?.backendId; + + if (backendId && backendId !== LOCAL_BACKEND_ID) { + const option = backendsModelsStore.get(backendId).models.find((m) => m.model === modelId); + + return (option?.status?.value as ServerModelStatus) ?? null; + } + const model = this.routerModels.find((m) => m.id === modelId); return (model?.status?.value as ServerModelStatus) ?? null; @@ -386,7 +495,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { * hidden by the user, so an automatic pick never lands on one it doesn't show. */ getVisibleModels(): ModelOption[] { - return this.models.filter( + return this.activeModels.filter( (option) => this.props.getModelProps(option.model)?.ui !== false && !this.isHidden(option.id) ); } @@ -450,13 +559,26 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { async selectModelById(modelId: string, options?: { recordRecent?: boolean }): Promise { if (!modelId || this.updating) return; - if (this.selectedModelId === modelId) { - if (options?.recordRecent) this.recordRecentModel(modelId); + const backendId = backendIdFromModelId(modelId) ?? backendsStore.active.id; + const rawId = rawModelId(modelId); + // the selection is stored backend-qualified, matching the aggregated + // model list, no matter which form the caller passed + const qualifiedId = qualifyModelId(backendId, rawId); + + // a model from another backend makes that backend active first + if (backendId !== backendsStore.active.id) { + backendsStore.setActive(backendId); + await backendsModelsStore.ensureLoaded(backendId); + await this.switchBackend(); + } + + if (this.selectedModelId === qualifiedId) { + if (options?.recordRecent) this.recordRecentModel(qualifiedId); return; } - const option = this.models.find((model) => model.id === modelId); + const option = this.activeModels.find((model) => model.id === rawId); if (!option) throw new Error('Selected model is not available'); @@ -464,10 +586,12 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { this.error = null; try { - this.selectedModelId = option.id; + this.selectedModelId = qualifiedId; this.selectedModelName = option.model; + this.selectionFromStorage = false; + this.persistSelection(); - if (options?.recordRecent) this.recordRecentModel(modelId); + if (options?.recordRecent) this.recordRecentModel(qualifiedId); // reasoning levels belong to the model that answered: another model in the chat starts at Default const conversationModel = getConversationModel(conversationsStore.activeMessages); @@ -582,6 +706,53 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { } } + /** + * Activate a backend for the selector tabs. Everything comes from memory: + * the model list and router rows are prefetched at startup and the local + * server state is kept while an external backend is active. The selection + * is left alone, switching tabs must not pick a model. + */ + async switchBackend(): Promise { + this.error = null; + + const backend = backendsStore.active; + + // local props describe the server the UI is served from; keep them while + // an external backend is active instead of dropping and refetching + if (backend.protocol === 'llama.cpp') { + serverStore.restoreLocalState(); + } else { + serverStore.cacheLocalState(); + serverStore.clear(); + } + + const cached = backendsModelsStore.get(backend.id); + + if (!cached.loaded) { + // nothing prefetched for this backend (startup prefetch failed): load it once + await this.fetch(true); + + return; + } + + if (backend.protocol === 'llama.cpp' && !serverStore.props) { + // first visit to the local tab in this session + await serverStore.fetch({ background: true }); + } + + this.activeModels = cached.models; + this.loading = false; + + // the local router rows carry the load statuses; the startup prefetch + // already returned them, so a tab switch rebuilds the list from memory + if (backend.protocol === 'llama.cpp' && this.routerModels.length === 0 && cached.raw) { + this.routerModels = cached.raw.data; + this.activeModels = this.buildModelOptions(cached.raw); + } + + this.warmHubDetails(); + } + toDisplayName(id: string): string { const segments = id.split(/\\|\//); const candidate = segments.pop(); @@ -621,10 +792,14 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { const repos: string[] = []; - for (const option of this.models) { - const repo = repoOf(option.model); + for (const backend of backendsStore.enabled) { + if (!getBackendCapabilities(backend).props) continue; + + for (const option of backendsModelsStore.get(backend.id).models) { + const repo = repoOf(option.model); - if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo); + if (repo?.includes('/') && !repos.includes(repo)) repos.push(repo); + } } // the Hub rate limits aggressively and an unauthenticated burst of requests @@ -671,16 +846,15 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { return { aliases: item.aliases ?? [], + // stamp the backend here so the option keeps its origin even + // after another backend becomes active + backendId: backendsStore.active.id, capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value) ), - // 0 is the server's way of leaving the trained context unknown; a - // model-mode listing reports it as meta.n_ctx_train instead - contextLength: - item.context_length || - (typeof item.meta?.n_ctx_train === 'number' && item.meta.n_ctx_train > 0 - ? item.meta.n_ctx_train - : undefined), + // external backends report the context in their listing, so the + // gauge keeps working when the list is rebuilt on reload + contextLength: readModelContextLength(item), description: details?.description, details: details?.details, draftSidecars: this._status.getDraftSidecars(modelId), @@ -715,6 +889,23 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { } } + private persistSelection(): void { + if (!browser) return; + + try { + if (!this.selectedModelId) { + localStorage.removeItem(SELECTED_MODEL_LOCALSTORAGE_KEY); + } else { + localStorage.setItem( + SELECTED_MODEL_LOCALSTORAGE_KEY, + JSON.stringify({ id: this.selectedModelId, model: this.selectedModelName }) + ); + } + } catch { + console.warn('[ModelsStore] Failed to persist the model selection'); + } + } + /** Move a model to the front of the recently used list. */ private recordRecentModel(qualifiedId: string): void { this.recentModelIds = [ @@ -746,7 +937,7 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { const response = await ModelsService.list(); this.routerModels = response.data; - this.models = this.buildModelOptions(response); + this.activeModels = this.buildModelOptions(response); await this.props.fetchModalitiesForLoadedModels(); @@ -756,10 +947,16 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { this.selectModelById(visible[0].id); } } else { - this.models = await this.fetchModelModeInternal(); + this.activeModels = await this.fetchModelModeInternal(); + + // external backends expose a selectable list; pick a default so the + // first send and title generation have a model to target + if (!serverStore.capabilities.props && !this.selectedModelName) { + await this.ensureFirstModelSelected(); + } } } catch (error) { - this.models = []; + this.activeModels = []; this.error = error instanceof Error ? error.message : 'Failed to load models'; throw error; diff --git a/ui/apps/chat/src/lib/stores/models/props.svelte.ts b/ui/apps/chat/src/lib/stores/models/props.svelte.ts index d7c89c438624..26387b4d5f68 100644 --- a/ui/apps/chat/src/lib/stores/models/props.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/props.svelte.ts @@ -19,14 +19,19 @@ import { import { FileTypeCategory, ModelModality, ReasoningEffort } from '#lib/enums/index.ts'; import { PropsService } from '#lib/services/props.service.ts'; // direct imports between stores, not via the barrel, to avoid circular deps +import { backendsStore } from '#lib/stores/backends.svelte.ts'; import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import type { ReasoningEffortLevel } from '#lib/types/index.ts'; // deep imports, not the '#lib/utils/index.ts' barrel: it re-exports modules that reach back // into the stores, and going through it here would read a half-built module import { TTLCache } from '#lib/utils/cache-ttl.ts'; import { detectThinkingSupport } from '#lib/utils/chat-template-thinking-detector.ts'; +import { rawModelId } from '#lib/utils/model-option-id.ts'; import { SvelteMap } from 'svelte/reactivity'; +/** Grace period before /props is retried for a model after a failed fetch. */ +const PROPS_FETCH_RETRY_COOLDOWN_MS = 30_000; + /** * The slice of modelsStore the manager reads. Kept narrow on purpose so it * cannot reach around the host's full surface; modelsStore implements this @@ -34,7 +39,9 @@ import { SvelteMap } from 'svelte/reactivity'; */ export interface ModelPropsHost { /** Model rows the manager mirrors fetched modalities onto. */ - models: ModelOption[]; + activeModels: ModelOption[]; + /** Every enabled backend's models, used to resolve a model's owner. */ + readonly models: ModelOption[]; readonly selectedModelName: string | null; readonly loadedModelIds: string[]; isModelLoaded(modelId: string): boolean; @@ -51,6 +58,8 @@ export class ModelPropsManager { maxEntries: MODEL_PROPS_CACHE.MAX_ENTRIES, ttlMs: MODEL_PROPS_CACHE.TTL_MS }); + /** Last failed /props fetch per model; requests stay on cooldown after it. */ + private failedPropsAt = new SvelteMap(); /** Props requests in flight, a second caller awaits the same request. */ private fetching = new SvelteMap>(); @@ -130,7 +139,7 @@ export class ModelPropsManager { try { const results = await Promise.all(propsPromises); - this.host.models = this.host.models.map((model) => { + this.host.activeModels = this.host.activeModels.map((model) => { const modelIndex = loadedModelIds.indexOf(model.model); if (modelIndex === -1) return model; @@ -159,14 +168,36 @@ export class ModelPropsManager { * @returns Props data or null if fetch failed or model not loaded */ async fetchModelProps(modelId: string): Promise { + // /props only exists on llama.cpp servers + if (!serverStore.capabilities.props) return null; + + // props not loaded yet (mid backend switch): the server role and model + // load state are not trustworthy, so do not guess + if (!serverStore.props) return null; + const cached = this.cache.get(modelId); if (cached) return cached; + // never ask a server about a model another backend serves; a name that + // exists on two backends resolves to the active one's entry first + const option = this.host.models.find( + (m) => m.model === modelId || m.id === modelId || rawModelId(m.id) === modelId + ); + + if (option?.backendId && option.backendId !== backendsStore.active.id) return null; + if (serverStore.isRouterMode && !this.host.isModelLoaded(modelId)) { return null; } + // a failing model would otherwise be refetched on every reactive update + const failedAt = this.failedPropsAt.get(modelId); + + if (failedAt !== undefined && Date.now() - failedAt < PROPS_FETCH_RETRY_COOLDOWN_MS) { + return null; + } + const inFlight = this.fetching.get(modelId); if (inFlight) return inFlight; @@ -175,10 +206,12 @@ export class ModelPropsManager { .then((props) => { this.cache.set(modelId, props); this.cacheVersion++; + this.failedPropsAt.delete(modelId); return props; }) .catch((error) => { + this.failedPropsAt.set(modelId, Date.now()); console.warn(`Failed to fetch props for model ${modelId}:`, error); return null; @@ -194,7 +227,12 @@ export class ModelPropsManager { const props = this.getModelProps(modelId); const nCtx = props?.default_generation_settings?.n_ctx; - return typeof nCtx === 'number' ? nCtx : null; + if (typeof nCtx === 'number') return nCtx; + + // external backends report the context in their model listing, when they report one + const model = this.host.activeModels.find((m) => m.model === modelId || m.id === modelId); + + return model?.contextLength ?? null; } getModelModalities(modelId: string): ModelModalities | null { @@ -202,7 +240,7 @@ export class ModelPropsManager { return this.buildModalities(serverStore.props.modalities); } - const model = this.host.models.find((m) => m.model === modelId || m.id === modelId); + const model = this.host.activeModels.find((m) => m.model === modelId || m.id === modelId); if (model?.modalities) { return model.modalities; @@ -304,7 +342,7 @@ export class ModelPropsManager { if (!props?.modalities) return; - this.host.models = this.host.models.map((model) => + this.host.activeModels = this.host.activeModels.map((model) => model.model === modelId ? { ...model, modalities: this.buildModalities(props.modalities!) } : model diff --git a/ui/apps/chat/src/lib/stores/models/status.svelte.ts b/ui/apps/chat/src/lib/stores/models/status.svelte.ts index ade2b701a881..b3cb32b38812 100644 --- a/ui/apps/chat/src/lib/stores/models/status.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/status.svelte.ts @@ -8,6 +8,7 @@ import { CLI_FLAGS, HF_UD_QUANT_PREFIX_REGEX, + LOCAL_BACKEND_ID, MODEL_ID, type ModelSidecar, STORAGE_APP_NAME @@ -20,7 +21,9 @@ import { } from '#lib/enums/index.ts'; import { HuggingFaceService } from '#lib/services/huggingface.service.ts'; import { ModelsService } from '#lib/services/models.service.ts'; +import { backendsStore } from '#lib/stores/backends.svelte.ts'; // direct imports between stores, not via the barrel, to avoid circular deps +import { backendsModelsStore } from '#lib/stores/backendsModels.svelte.ts'; import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import type { ModelPropsManager } from '#lib/stores/models/props.svelte.ts'; // explicit type imports: the app.d.ts globals resolve to `any`, so import the real types @@ -32,6 +35,7 @@ import type { } from '#lib/types/index.ts'; import type { ModelSidecarBadge } from '#lib/types/models.js'; import { repoOf } from '#lib/utils/model-names.ts'; +import { backendIdFromModelId } from '#lib/utils/model-option-id.ts'; import { isAuxSidecar, isDraftSidecar, sidecarFromFileToken } from '#lib/utils/sidecars.ts'; import { browser } from '$app/env'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -42,13 +46,20 @@ import { toast } from 'svelte-sonner'; * cannot reach around the host's full surface; modelsStore implements this * structurally. */ +/** How long a remote operation is watched before its last answer is kept. */ +const REMOTE_STATUS_TIMEOUT_MS = 10 * 60 * 1000; +const REMOTE_STATUS_POLL_MS = 2000; + export interface ModelStatusHost { error: string | null; + /** Load state a model's own backend reports, local or external. */ + getModelStatus(modelId: string): ServerModelStatus | null; readonly props: ModelPropsManager; /** Router model rows the status feed updates. */ routerModels: ApiModelDataEntry[]; fetchRouterModels(): Promise; isModelLoaded(modelId: string): boolean; + switchBackend(): Promise; toDisplayName(id: string): string; } @@ -213,6 +224,8 @@ export class ModelStatusManager { * removes the model along with the partial files of its download. */ async cancelDownload(repoWithTag: string): Promise { + await this.ensureLocalTarget(); + if (!serverStore.isRouterMode) { toast.error('Model downloads are only available in router mode'); @@ -265,12 +278,14 @@ export class ModelStatusManager { * settled status, so no waiter is registered here. */ async cancelLoad(modelId: string): Promise { + const backendId = this.backendIdFor(modelId); + if (!serverStore.isRouterMode) return; this.subscribe(); try { - await ModelsService.unload(modelId); + await ModelsService.unload(modelId, backendId); toast.info(`Load cancelled: ${this.host.toDisplayName(modelId)}`); } catch (error) { toast.error(`Failed to cancel load: ${this.host.toDisplayName(modelId)}`); @@ -306,6 +321,28 @@ export class ModelStatusManager { return; } + // a second click must not post again: the server would answer that the model + // already exists, so a download already on its way is left alone + if (this.pendingDownloads.has(model) || this.isDownloadInProgress(model)) return; + + // the marker and its last reported progress survive until the request is + // accepted: dropping them first would drop the paused state, which reads as + // a lost download if the server rejects the resume + const snapshot = this.pausedDownloads.get(model) ?? null; + const wasPaused = this.pausedDownloads.has(model); + + // the download reads as in flight from here, so the chip shows it before the + // server answers; the feed refines it with the real counts. Both marks land + // ahead of the first await, so a second click in the same tick still coalesces + this.pendingDownloads.add(model); + this.downloadProgress.set(model, snapshot ?? emptyDownloadProgress()); + + // the local server owns the download endpoint, so the target moves only once + // the row already reads as downloading + await this.ensureLocalTarget(); + + // read once the target has moved: the listing describes the server taking the + // download, not the backend the row was clicked from const registered = this.host.routerModels.find( (m) => downloadIdKey(m.id) === downloadIdKey(model) ); @@ -317,25 +354,9 @@ export class ModelStatusManager { ...extraSidecars ].filter((kind, i, all) => SERVER_SIDECARS.includes(kind) && all.indexOf(kind) === i); - // a second click must not post again: the server would answer that the model - // already exists, so a download already on its way is left alone - if (this.pendingDownloads.has(model) || this.isDownloadInProgress(model)) return; - - this.pendingDownloads.add(model); - // the feed must be live so the resulting models_reload event refreshes the list this.subscribe(); - // the marker and its last reported progress survive until the request is - // accepted: dropping them first would drop the paused state, which reads as - // a lost download if the server rejects the resume - const snapshot = this.pausedDownloads.get(model) ?? null; - const wasPaused = this.pausedDownloads.has(model); - - // the download reads as in flight from here, so the chip shows it before the - // server answers; the feed refines it with the real counts - this.downloadProgress.set(model, snapshot ?? emptyDownloadProgress()); - try { const res = await ModelsService.downloadModel(model, sidecars); @@ -495,7 +516,10 @@ export class ModelStatusManager { return this.loadingStates.get(modelId) ?? false; } - async load(modelId: string): Promise { + async load(modelId: string, extraArgs?: string[]): Promise { + const backendId = this.backendIdFor(modelId); + const isLocal = backendId === LOCAL_BACKEND_ID; + if (this.host.isModelLoaded(modelId)) return; if (this.loadingStates.get(modelId)) return; @@ -506,13 +530,21 @@ export class ModelStatusManager { // the feed drives completion, so it must be live before the request this.subscribe(); - const reachedLoaded = this.waitForStatus(modelId, ServerModelStatus.LOADED); + const reachedLoaded = isLocal + ? this.waitForStatus(modelId, ServerModelStatus.LOADED) + : Promise.resolve(); reachedLoaded.catch(() => {}); try { - await ModelsService.load(modelId); - await reachedLoaded; + await ModelsService.load(modelId, extraArgs, backendId); + + if (isLocal) { + await reachedLoaded; + } else { + await this.waitForRemoteStatus(backendId, modelId, ServerModelStatus.LOADED); + } + toast.success(`Model loaded: ${this.host.toDisplayName(modelId)}`); } catch (error) { this.rejectStatus(modelId, error instanceof Error ? error : new Error('load failed')); @@ -532,6 +564,8 @@ export class ModelStatusManager { * before the request: an unconfirmed pause would leave the chip spinning. */ async pauseDownload(repoWithTag: string): Promise { + await this.ensureLocalTarget(); + if (!serverStore.isRouterMode) { toast.error('Model downloads are only available in router mode'); @@ -609,6 +643,9 @@ export class ModelStatusManager { } async unload(modelId: string): Promise { + const backendId = this.backendIdFor(modelId); + const isLocal = backendId === LOCAL_BACKEND_ID; + if (!this.host.isModelLoaded(modelId)) return; if (this.loadingStates.get(modelId)) return; @@ -618,13 +655,21 @@ export class ModelStatusManager { this.subscribe(); - const reachedUnloaded = this.waitForStatus(modelId, ServerModelStatus.UNLOADED); + const reachedUnloaded = isLocal + ? this.waitForStatus(modelId, ServerModelStatus.UNLOADED) + : Promise.resolve(); reachedUnloaded.catch(() => {}); try { - await ModelsService.unload(modelId); - await reachedUnloaded; + await ModelsService.unload(modelId, backendId); + + if (isLocal) { + await reachedUnloaded; + } else { + await this.waitForRemoteStatus(backendId, modelId, ServerModelStatus.UNLOADED); + } + toast.info(`Model unloaded: ${this.host.toDisplayName(modelId)}`); } catch (error) { this.rejectStatus(modelId, error instanceof Error ? error : new Error('unload failed')); @@ -808,9 +853,35 @@ export class ModelStatusManager { } } + /** + * Backend a model belongs to: the one its qualified id names, else whichever + * enabled backend lists it. Loading an external llama-server's model must not + * become a request to the server this UI is served from. + */ + private backendIdFor(modelId: string): string { + const qualified = backendIdFromModelId(modelId); + + if (qualified) return qualified; + + for (const backend of backendsStore.enabled) { + const state = backendsModelsStore.get(backend.id); + + if (state.models.some((option) => option.model === modelId)) return backend.id; + } + + return backendsStore.active.id; + } + private deletePausedDownload(repoWithTag: string): boolean { return this.pausedDownloads.delete(repoWithTag); } + private async ensureLocalTarget(): Promise { + if (backendsStore.active.id === LOCAL_BACKEND_ID) return; + + backendsStore.setActive(LOCAL_BACKEND_ID); + await backendsModelsStore.ensureLoaded(LOCAL_BACKEND_ID); + await this.host.switchBackend(); + } /** True when the option is the model itself, or a sidecar its download pulls. */ private pullsSidecar(repoWithTag: string): boolean { @@ -883,6 +954,42 @@ export class ModelStatusManager { for (const waiter of waiters) if (waiter.target === status) waiter.resolve(); } + /** + * Wait for an external backend to report a status. Its changes never reach the + * local feed, so its own listing is asked again until the status lands or the + * deadline passes, which leaves the last answer in place. + */ + private async waitForRemoteStatus( + backendId: string, + modelId: string, + target: ServerModelStatus + ): Promise { + const deadline = Date.now() + REMOTE_STATUS_TIMEOUT_MS; + + while (Date.now() < deadline) { + await backendsModelsStore.refresh(backendId); + + const status = this.host.getModelStatus(modelId); + + if (status === ServerModelStatus.FAILED) { + throw new Error( + target === ServerModelStatus.LOADED + ? 'the server failed to load it' + : 'the server failed to unload it' + ); + } + + const reached = + target === ServerModelStatus.LOADED + ? status === ServerModelStatus.LOADED || status === ServerModelStatus.SLEEPING + : status === ServerModelStatus.UNLOADED || status === null; + + if (reached) return; + + await new Promise((resolve) => setTimeout(resolve, REMOTE_STATUS_POLL_MS)); + } + } + // a load and the requests that wait on it can each hold a waiter for the same model private waitForStatus(modelId: string, target: ServerModelStatus): Promise { return new Promise((resolve, reject) => { diff --git a/ui/apps/chat/src/lib/stores/tools.svelte.ts b/ui/apps/chat/src/lib/stores/tools.svelte.ts index 0e926cd1673e..708ae30100b7 100644 --- a/ui/apps/chat/src/lib/stores/tools.svelte.ts +++ b/ui/apps/chat/src/lib/stores/tools.svelte.ts @@ -27,6 +27,8 @@ import { } from '#lib/extensions/index.ts'; import { ToolsService } from '#lib/services/tools.service.ts'; // direct imports between stores, not via the barrel, to avoid circular deps +import { backendsStore } from '#lib/stores/backends.svelte.ts'; +import { serverStore } from '#lib/stores/llama-server.svelte.ts'; import { modelsStore } from '#lib/stores/models/index.svelte.ts'; import { projectsStore } from '#lib/stores/projects.svelte.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; @@ -251,6 +253,15 @@ class ToolsStore { } async fetchServerTools(): Promise { + // the /tools endpoint only exists on the local llama.cpp server, so the + // list follows the built-in backend rather than the selected model + if (!serverStore.hasLocalServer || !backendsStore.local.enabled) { + this._serverTools = []; + this.cwdAwareTools = new SvelteSet(); + + return; + } + if (this._loading) return; this._loading = true; diff --git a/ui/apps/chat/src/lib/utils/api-headers.ts b/ui/apps/chat/src/lib/utils/api-headers.ts index 0c646eac7d1b..0581e685ec2e 100644 --- a/ui/apps/chat/src/lib/utils/api-headers.ts +++ b/ui/apps/chat/src/lib/utils/api-headers.ts @@ -2,7 +2,7 @@ import { getBackend } from './api-base'; import { redactValue } from './redact'; import { CORS_PROXY, HEADERS } from '#lib/constants/index.ts'; import { MimeTypeApplication } from '#lib/enums/index.ts'; -import { getProtocolAdapter } from '#lib/services/protocols'; +import { getProtocolAdapter } from '#lib/services/protocols/index.ts'; import { settingsStore } from '#lib/stores/settings/index.svelte.ts'; import type { Backend } from '#lib/types/index.ts'; diff --git a/ui/apps/chat/src/lib/utils/backend.ts b/ui/apps/chat/src/lib/utils/backend.ts index 896a65b8467d..316ea7009d31 100644 --- a/ui/apps/chat/src/lib/utils/backend.ts +++ b/ui/apps/chat/src/lib/utils/backend.ts @@ -234,6 +234,16 @@ export function readModelContextLength(value: unknown): number | undefined { if (sizes.length > 0) return Math.max(...sizes); } + // 0 is the server's way of leaving the trained context unknown; a model-mode + // listing reports it as meta.n_ctx_train instead + const meta = entry.meta; + + if (meta && typeof meta === 'object') { + const trained = (meta as Record).n_ctx_train; + + if (typeof trained === 'number' && trained > 0) return trained; + } + return undefined; } diff --git a/ui/apps/chat/src/routes/[extensionPage]/+page.ts b/ui/apps/chat/src/routes/[extensionPage]/+page.ts index af674fcc8f60..98440b7e2176 100644 --- a/ui/apps/chat/src/routes/[extensionPage]/+page.ts +++ b/ui/apps/chat/src/routes/[extensionPage]/+page.ts @@ -1,12 +1,14 @@ import type { PageLoad } from './$types'; import { error } from '@sveltejs/kit'; import { extensions } from '#lib/extensions/index.ts'; -import { loadAppRoute } from '#lib/stores/init.ts'; +import { loadAppRoute, startExtensions } from '#lib/stores/init.ts'; export const load: PageLoad = async ({ fetch, params }) => { await loadAppRoute(fetch); + // the rest of the startup runs after this load, so the extensions are started here + await startExtensions(); - // the extensions are active by now: a path none of them has a page for is the app's 404 + // a path none of them has a page for is the app's 404 if (!extensions.contributions('pages').some((page) => page.path === params.extensionPage)) { error(404, 'Not Found'); } diff --git a/ui/apps/chat/tests/unit/extension-page-route.test.ts b/ui/apps/chat/tests/unit/extension-page-route.test.ts index 7cb28d3344e1..595387e2285e 100644 --- a/ui/apps/chat/tests/unit/extension-page-route.test.ts +++ b/ui/apps/chat/tests/unit/extension-page-route.test.ts @@ -4,11 +4,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('$app/env', () => ({ browser: true })); -vi.mock('#lib/stores/init.ts', () => ({ loadAppRoute: vi.fn(async () => {}) })); +vi.mock('#lib/stores/init.ts', () => ({ + loadAppRoute: vi.fn(async () => {}), + startExtensions: vi.fn(async () => {}) +})); import { load } from '../../src/routes/[extensionPage]/+page'; import { extensions } from '#lib/extensions/index.ts'; -import { loadAppRoute } from '#lib/stores/init.ts'; +import { loadAppRoute, startExtensions } from '#lib/stores/init.ts'; import type { Component } from 'svelte'; const Page = (() => {}) as unknown as Component; @@ -51,8 +54,9 @@ describe('extension page route', () => { await expect(loadPath('fake-page')).resolves.toBeUndefined(); await expect(loadPath('other-page')).rejects.toMatchObject({ status: 404 }); - // the stores start, and the API key is checked, before the path is looked up + // the stores hydrate, the API key is checked and the extensions start before the path is looked up expect(loadAppRoute).toHaveBeenCalledWith(fetch); + expect(startExtensions).toHaveBeenCalled(); await extensions.setEnabled('fake-pages', false); From 55cf1a043a3e09bf8c3e37f16e097a95ff7c5adf Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 28 Sep 2026 15:46:45 +0200 Subject: [PATCH 03/34] ui : add the manage providers view A provider is added from a preset card or by hand: the dialog probes the endpoint, reads a refusal as a llama.cpp server that wants a key, and shows the connection test as a status block. Presets carry their official artwork and a saved backend keeps its branding through the favicon fallback. The manager dialog gains its third view, and the settings save stops clobbering keys that other surfaces write. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash --- .../app/backends/BackendCard.svelte | 163 ++++++++ .../app/backends/BackendForm.svelte | 190 ++++++++++ .../app/backends/BackendIcon.svelte | 37 ++ .../app/backends/BackendPresetCard.svelte | 41 ++ .../app/backends/BackendPresetIcon.svelte | 34 ++ .../app/backends/DialogBackendForm.svelte | 356 ++++++++++++++++++ .../src/lib/components/app/backends/index.ts | 6 + ui/apps/chat/src/lib/components/app/index.ts | 1 + .../models/ModelsManager/ModelsManager.svelte | 1 + .../ModelsManagerModelProviders.svelte | 108 ++++++ .../settings/SettingsChat/SettingsChat.svelte | 17 +- .../SettingsChat/SettingsChatFields.svelte | 2 +- .../src/lib/components/app/settings/index.ts | 5 + .../src/lib/constants/backend.constants.ts | 35 +- .../src/lib/constants/storage.constants.ts | 1 + ui/apps/chat/src/lib/utils/backend.ts | 25 ++ ui/apps/chat/src/lib/utils/index.ts | 1 + ui/apps/chat/src/routes/models/+page.svelte | 68 +++- .../chat/static/backend-presets/deepseek.svg | 1 + .../chat/static/backend-presets/google.svg | 10 + .../static/backend-presets/huggingface.svg | 37 ++ ui/apps/chat/static/backend-presets/kimi.png | Bin 0 -> 1835 bytes .../backend-presets/openrouter-dark.svg | 5 + .../backend-presets/openrouter-light.svg | 5 + ui/apps/chat/static/backend-presets/qwen.png | Bin 0 -> 1658 bytes ui/apps/chat/static/backend-presets/zai.png | Bin 0 -> 1064 bytes 26 files changed, 1132 insertions(+), 17 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte create mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte create mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte create mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendPresetCard.svelte create mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte create mode 100644 ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte create mode 100644 ui/apps/chat/src/lib/components/app/backends/index.ts create mode 100644 ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte create mode 100644 ui/apps/chat/static/backend-presets/deepseek.svg create mode 100644 ui/apps/chat/static/backend-presets/google.svg create mode 100644 ui/apps/chat/static/backend-presets/huggingface.svg create mode 100644 ui/apps/chat/static/backend-presets/kimi.png create mode 100644 ui/apps/chat/static/backend-presets/openrouter-dark.svg create mode 100644 ui/apps/chat/static/backend-presets/openrouter-light.svg create mode 100644 ui/apps/chat/static/backend-presets/qwen.png create mode 100644 ui/apps/chat/static/backend-presets/zai.png diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte new file mode 100644 index 000000000000..14cc49a10214 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte @@ -0,0 +1,163 @@ + + + +
+
+ + {#snippet fallback()} + {#if isLocal} + + {:else} + + {/if} + {/snippet} + + +
+

{backend.name}

+ +

{displayUrl}

+
+
+ + onToggle?.(value)} + /> +
+ + {#if isOffline && modelsState.error} + + +

{modelsState.error}

+
+ + +

{modelsState.error}

+
+
+ {/if} + +
+
+ {#if isChecking} + + + Checking + {:else if isOnline} + + + {modelCount === 1 ? '1 model' : `${modelCount} models`} + {:else if isOffline} + + + Unreachable + + + {/if} + + {#if hasStatus} + · + {/if} + + {metaText} +
+ + {#if !isLocal} +
+ + + +
+ {/if} +
+
+ + (showDelete = false)} + onConfirm={() => { + showDelete = false; + onDelete?.(); + }} + title="Delete backend?" + variant="destructive" +/> diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte new file mode 100644 index 000000000000..bd762ad89e69 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte @@ -0,0 +1,190 @@ + + +
+
+ + + onChange({ baseUrl: e.currentTarget.value })} + placeholder="https://api.example.com" + type="url" + value={backend.baseUrl} + /> + + {#if urlError} +

{urlError}

+ {:else if detectionLabel} +

{detectionLabel}

+ {/if} +
+ +
+
+ + + onChange({ name: e.currentTarget.value })} + placeholder="Name shown in the model selector" + type="text" + value={backend.name} + /> +
+ +
+ + + onChange({ apiKey: e.currentTarget.value || undefined })} + placeholder="Optional" + type="password" + value={backend.apiKey ?? ''} + /> + +

+ {#if apiKeyRequired} + Required by this endpoint. + {:else} + Sent as a Bearer token. + {/if} +

+
+
+ + + + {#if showAdvanced} + + {:else} + + {/if} + + Advanced + + + +
+
+ API format + + + onChange({ compat: undefined, protocol: value as BackendProtocol })} + type="single" + value={backend.protocol} + > + {protocolLabel} + + + {#each PROTOCOL_OPTIONS as option (option.value)} + {option.label} + {/each} + + + +

+ Detected from the endpoint. Override it when a server answers unusually. +

+
+ +
+
+ + + onChange({ chatPath: e.currentTarget.value || undefined })} + placeholder={DEFAULT_BACKEND_CHAT_PATH} + type="text" + value={backend.chatPath ?? ''} + /> +
+ +
+ + + onChange({ modelsPath: e.currentTarget.value || undefined })} + placeholder={DEFAULT_BACKEND_MODELS_PATH} + type="text" + value={backend.modelsPath ?? ''} + /> +
+
+
+
+
+
diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte new file mode 100644 index 000000000000..a22efe829268 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte @@ -0,0 +1,37 @@ + + +{#if preset} + +{:else if faviconUrl && !failedFavicons.has(faviconUrl)} + failedFavicons.add(faviconUrl)} + src={faviconUrl} + /> +{:else} + {@render fallback?.()} +{/if} diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendPresetCard.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendPresetCard.svelte new file mode 100644 index 000000000000..3ba61a75f97c --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/BackendPresetCard.svelte @@ -0,0 +1,41 @@ + + + + {#if added} + + + Added + + {/if} + +
+ + +

{preset.name}

+
+ +

{preset.description}

+
diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte new file mode 100644 index 000000000000..2b5fc1850b3d --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte @@ -0,0 +1,34 @@ + + +{#if iconUrl} + +{:else} + +{/if} diff --git a/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte b/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte new file mode 100644 index 000000000000..6d3743c5499b --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte @@ -0,0 +1,356 @@ + + + + + + + {isEdit ? 'Edit Provider' : 'Add New Provider'} + + + + Point at another llama-server, or connect an OpenAI-compatible endpoint. + + + + {#if !isEdit && presetsToShow.length > 0} +
+
+

Recommended providers

+ + +
+ +
+ {#each presetsToShow as preset (preset.id)} + applyPreset(preset)} + {preset} + selected={selectedPresetId === preset.id} + /> + {/each} +
+
+ {/if} + +
+
+ +
+ + +
+ {#if testing} + + +
+

Testing connection

+ +

+ Asking {hostOf(draft.baseUrl)} for the models it serves. +

+
+ {:else if testResult?.ok} + + +
+

Connected

+ +

+ {testResult.modelCount ?? 0} + model{(testResult.modelCount ?? 0) === 1 ? '' : 's'} available. +

+
+ {:else if testResult} + + +
+

Connection failed

+ +

+ {testResult.error ?? 'The endpoint did not answer.'}{testResult.status + ? ` (HTTP ${testResult.status})` + : ''} +

+
+ {:else} + + +
+

Connection not tested

+ +

+ Test it to list the models this provider serves. +

+
+ {/if} +
+ + + + + + + + +
+
+
diff --git a/ui/apps/chat/src/lib/components/app/backends/index.ts b/ui/apps/chat/src/lib/components/app/backends/index.ts new file mode 100644 index 000000000000..529bc7fc3649 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/backends/index.ts @@ -0,0 +1,6 @@ +export { default as BackendCard } from './BackendCard.svelte'; +export { default as BackendForm } from './BackendForm.svelte'; +export { default as BackendIcon } from './BackendIcon.svelte'; +export { default as BackendPresetCard } from './BackendPresetCard.svelte'; +export { default as BackendPresetIcon } from './BackendPresetIcon.svelte'; +export { default as DialogBackendForm } from './DialogBackendForm.svelte'; diff --git a/ui/apps/chat/src/lib/components/app/index.ts b/ui/apps/chat/src/lib/components/app/index.ts index 302f0a2f06ad..24b717cb69f0 100644 --- a/ui/apps/chat/src/lib/components/app/index.ts +++ b/ui/apps/chat/src/lib/components/app/index.ts @@ -1,4 +1,5 @@ export * from './actions'; +export * from './backends'; export * from './badges'; export * from './chat'; export * from './content'; diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte index 4cdf3112bf7c..df70e68070ae 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -21,6 +21,7 @@ import { filterModelOptions } from '#lib/utils/index.ts'; import type { Snippet } from 'svelte'; import { SvelteMap } from 'svelte/reactivity'; + import { toast } from 'svelte-sonner'; interface Props { class?: string; diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte new file mode 100644 index 000000000000..a3ee433396b4 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte @@ -0,0 +1,108 @@ + + +
+ void backendsModelsStore.loadAll()} + /> + + {#if showsLocal && !serverStore.localServerMissing} + backendsStore.setLocalEnabled(enabled)} + /> + {/if} + + {#each backends as backend (backend.id)} + { + backendsStore.removeBackend(backend.id); + void backendsModelsStore.loadAll(); + }} + onEdit={() => handleEdit(backend)} + onToggle={(enabled) => { + backendsStore.updateBackend(backend.id, { enabled }); + void backendsModelsStore.loadAll(); + }} + /> + {/each} + +
+ + +

{addDescription}

+
+
diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 75b9ac341e9c..78b85bc1cc1a 100644 --- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -68,12 +68,17 @@ return false; }); + // Snapshot used to persist only the fields edited in this dialog. Config keys + // that other surfaces write directly (backends, MCP servers) must not be + // clobbered by the dialog's save. + let baseline: SettingsConfigType = $state({ ...settingsStore.config }); + let mobileHeader: { updateCarousel: () => void } | undefined; let fetchInitiated = false; $effect(() => { - if (serverStore.isRouterMode && currentSection.fields?.length && !fetchInitiated) { + if (serverStore.localIsRouter && currentSection.fields?.length && !fetchInitiated) { fetchInitiated = true; void modelsStore @@ -108,6 +113,7 @@ function handleReset() { localConfig = { ...settingsStore.config }; + baseline = { ...settingsStore.config }; setMode(localConfig.theme as ColorMode); applySurfaceTheme(localConfig[SETTINGS_KEYS.SURFACE_THEME]); mobileHeader?.updateCarousel(); @@ -129,7 +135,13 @@ } } - const processedConfig = { ...localConfig }; + const processedConfig: SettingsConfigType = {}; + + for (const [key, value] of Object.entries(localConfig)) { + if (value !== baseline[key]) { + processedConfig[key] = value; + } + } for (const field of NUMERIC_FIELDS) { if (processedConfig[field] !== undefined && processedConfig[field] !== '') { @@ -161,6 +173,7 @@ export function reset() { localConfig = { ...settingsStore.config }; + baseline = { ...settingsStore.config }; } diff --git a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte index 9d864ca7cd49..1152b1eaa812 100644 --- a/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte +++ b/ui/apps/chat/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte @@ -47,7 +47,7 @@ let currentModelParams = $derived.by(() => { void modelsStore.props.cacheVersion; - if (serverStore.isRouterMode) { + if (serverStore.localIsRouter) { const currentModelName = modelsStore.selectedModelName; if (currentModelName) { diff --git a/ui/apps/chat/src/lib/components/app/settings/index.ts b/ui/apps/chat/src/lib/components/app/settings/index.ts index 184a34fc7f51..fe06f197c377 100644 --- a/ui/apps/chat/src/lib/components/app/settings/index.ts +++ b/ui/apps/chat/src/lib/components/app/settings/index.ts @@ -91,3 +91,8 @@ export { default as SettingsChatFields } from './SettingsChat/SettingsChatFields * server favicons and permission management controls. */ export { default as SettingsChatToolsTab } from './SettingsChat/SettingsChatToolsTab.svelte'; + +/** + * Providers panel for chat settings. Switches between the llama.cpp and + * OpenAI-compatible protocols and lists the endpoints configured for each. + */ diff --git a/ui/apps/chat/src/lib/constants/backend.constants.ts b/ui/apps/chat/src/lib/constants/backend.constants.ts index 5ab3ab756822..cef4dba928f6 100644 --- a/ui/apps/chat/src/lib/constants/backend.constants.ts +++ b/ui/apps/chat/src/lib/constants/backend.constants.ts @@ -1,4 +1,9 @@ -import type { BackendCapabilities, BackendCompat, BackendProtocol } from '#lib/types/index.ts'; +import type { + BackendCapabilities, + BackendCompat, + BackendPreset, + BackendProtocol +} from '#lib/types/index.ts'; /** Prefix for generated ids of user-added backends. */ export const BACKEND_ID_PREFIX = 'backend'; @@ -64,3 +69,31 @@ export const MODEL_CONTEXT_LENGTH_FIELDS = [ /** Favicon extract keyed by domain, for backends with no bundled mark. */ export const FAVICON_SERVICE_URL = 'https://www.google.com/s2/favicons?domain='; + +/** + * Ready-made endpoints offered when adding a backend. + * + * TODO: pair a llama.cpp server by scanning its QR code (WebRTC transport, so a + * hosted PWA can reach a server on the user's network), see + * ggml-org/llama.cpp#24577. + */ +export const BACKEND_PRESETS: readonly BackendPreset[] = [ + { + baseUrl: 'https://router.huggingface.co', + description: 'Open models from the Hub, served by inference providers.', + iconUrl: '/backend-presets/huggingface.svg', + id: 'huggingface', + name: 'Hugging Face', + protocol: 'openai' + }, + { + baseUrl: 'https://openrouter.ai/api', + description: 'Models from many providers behind a single API.', + // v2 brand: purple on light, lime on dark + iconUrlDark: '/backend-presets/openrouter-dark.svg', + iconUrlLight: '/backend-presets/openrouter-light.svg', + id: 'openrouter', + name: 'OpenRouter', + protocol: 'openai' + } +]; diff --git a/ui/apps/chat/src/lib/constants/storage.constants.ts b/ui/apps/chat/src/lib/constants/storage.constants.ts index 0290b3511f63..c0345c8942ee 100644 --- a/ui/apps/chat/src/lib/constants/storage.constants.ts +++ b/ui/apps/chat/src/lib/constants/storage.constants.ts @@ -56,6 +56,7 @@ export const SIDEBAR_EXPANDED_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.sidebarExp export const SIDEBAR_WIDTH_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.sidebarWidth`; export const USER_OVERRIDES_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.userOverrides`; export const DISMISSED_RECOMMENDED_MCP_SERVERS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.dismissedRecommendedMcpServers`; +export const DISMISSED_RECOMMENDED_BACKENDS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.dismissedRecommendedBackends`; /** Key prefix for per-conversation resumable stream state, conversationId is appended */ export const STREAM_RESUME_LOCALSTORAGE_KEY_PREFIX = `${STORAGE_APP_NAME}.streamResume.`; diff --git a/ui/apps/chat/src/lib/utils/backend.ts b/ui/apps/chat/src/lib/utils/backend.ts index 316ea7009d31..7f5c765a4264 100644 --- a/ui/apps/chat/src/lib/utils/backend.ts +++ b/ui/apps/chat/src/lib/utils/backend.ts @@ -10,6 +10,7 @@ import { BACKEND_CAPABILITIES, BACKEND_COMPAT, BACKEND_ID_PREFIX, + BACKEND_PRESETS, BACKEND_PROTOCOLS, DEFAULT_BACKEND_CHAT_PATH, DEFAULT_BACKEND_MODELS_PATH, @@ -21,6 +22,7 @@ import type { Backend, BackendCapabilities, BackendCompat, + BackendPreset, BackendProtocol } from '#lib/types/index.ts'; @@ -54,6 +56,29 @@ export function backendFaviconUrl(baseUrl: string): string | null { } } +/** + * Preset a backend was created from, matched on origin and path so a saved + * backend keeps its branding. Returns undefined for edited or custom URLs. + */ +export function findBackendPreset(baseUrl: string): BackendPreset | undefined { + const target = normalizeBaseUrl(baseUrl); + + if (!target) return undefined; + + return BACKEND_PRESETS.find((preset) => normalizeBaseUrl(preset.baseUrl) === target); +} + +/** Origin and path of a URL, without a trailing slash; null when unparsable. */ +function normalizeBaseUrl(url: string): string | null { + try { + const parsed = new URL(url); + + return `${parsed.origin}${parsed.pathname.replace(/\/+$/, '')}`.toLowerCase(); + } catch { + return null; + } +} + /** * Features a backend supports, derived from its protocol. A missing backend * (unknown model, early startup) gets the plain compatible defaults. diff --git a/ui/apps/chat/src/lib/utils/index.ts b/ui/apps/chat/src/lib/utils/index.ts index 441e1dc20a95..0aef6056c67e 100644 --- a/ui/apps/chat/src/lib/utils/index.ts +++ b/ui/apps/chat/src/lib/utils/index.ts @@ -30,6 +30,7 @@ export { backendFaviconUrl, backendModelsUrl, createLocalBackend, + findBackendPreset, getBackendCapabilities, parseBackendsSettings } from './backend'; diff --git a/ui/apps/chat/src/routes/models/+page.svelte b/ui/apps/chat/src/routes/models/+page.svelte index e8295430845b..95288dffae85 100644 --- a/ui/apps/chat/src/routes/models/+page.svelte +++ b/ui/apps/chat/src/routes/models/+page.svelte @@ -1,10 +1,15 @@ {#if hasError || isLoadingModel} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte index eb20342ce3cd..883d2daab846 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte @@ -1,9 +1,18 @@ -{#if orgName && hubEnabled} +{#if useProviderIcon} + + + {#snippet fallback()} + {#if isLocal} + + {:else} + + {/if} + {/snippet} + + +{:else if orgName && hubEnabled} (isNearViewport = true))} class={['inline-flex shrink-0', className]} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte b/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte index 023412df889d..d3339c44a0b7 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelCapabilities.svelte @@ -1,4 +1,5 @@
- {#if isLoading} + {#if !canLoad} + {#if showBackendMark} + + {:else if showRemoteMark} + + + + {#if isBackendFailed} + + {/if} + + {/if} + {:else if isLoading} {:else} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte index df70e68070ae..0a85be100523 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -15,13 +15,18 @@ type ModalityKey, MODELS_TABLE_GROUP_LABELS } from '#lib/constants/index.ts'; - import { ModelCapability, ModelsTableGroupKind } from '#lib/enums/index.ts'; - import { modelsStore, uiStore } from '#lib/stores/index.ts'; + import { + ModelCapability, + ModelsTableGroupKind, + ModelsTableProviderKind + } from '#lib/enums/index.ts'; + import { backendsStore, modelsStore, uiStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; + import { getBackend } from '#lib/utils/api-base.ts'; + import { getBackendCapabilities } from '#lib/utils/backend.ts'; import { filterModelOptions } from '#lib/utils/index.ts'; import type { Snippet } from 'svelte'; import { SvelteMap } from 'svelte/reactivity'; - import { toast } from 'svelte-sonner'; interface Props { class?: string; @@ -32,6 +37,7 @@ let { class: className, toolbarEnd }: Props = $props(); let filter = $state(''); + let providerFilter = $state([]); let contextLimit = $state(0); let modalityFilter = $state([]); let capabilityFilter = $state([]); @@ -51,6 +57,8 @@ let isFavorite = $derived((option: ModelOption) => modelsStore.favoriteModelIds.has(option.model) ); + // every filter but the provider one, so a provider count does not fall to zero + // the moment that provider is the one being looked at let matching = $derived.by(() => { // the term matches what the selector search matches: name, model, aliases and tags const searched = filterModelOptions(allModels, filter); @@ -80,6 +88,24 @@ return context === null || context >= contextLimit; }); }); + let visible = $derived.by(() => + providerFilter.length === 0 + ? matching + : matching.filter((option) => providerFilter.includes(option.backendId ?? LOCAL_BACKEND_ID)) + ); + // the rail counts follow the active view and filter, so it always says how many + // repos each provider contributes to what the table is showing + let providerCounts = $derived.by(() => { + const counts: Record = {}; + + for (const entry of groupModelQuants(matching)) { + const backendId = entry.base.backendId ?? LOCAL_BACKEND_ID; + + counts[backendId] = (counts[backendId] ?? 0) + 1; + } + + return counts; + }); // recently used models lead their section, the rest keep the server's order let rank = $derived.by(() => { @@ -94,11 +120,15 @@ Math.min(...entry.quants.map((quant) => rank.get(quant.id) ?? Number.MAX_SAFE_INTEGER)); const byRecency = (list: ModelQuantGroup[]) => rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b)); - // one entry per repo, so a model with several quants takes a single table row - let entries = $derived(byRecency(groupModelQuants(matching))); + // one entry per repo, so a model with several quants takes a single table row; + // loaded models lead the table, then favorites, then one block per provider + let entries = $derived(byRecency(groupModelQuants(visible))); let groups = $derived.by(() => { - // a loaded quant is a model of its own: its repo keeps the quants left behind - const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model); + // a loaded quant is a model of its own: its repo keeps the quants left behind. + // Only llama-compat servers report a load state. + const isLoaded = (option: ModelOption) => + getBackendCapabilities(getBackend(option.backendId)).loadUnload && + modelsStore.isModelLoaded(option.model); // a tracked download stands in its own section, so it is not listed twice const isDownload = (option: ModelOption) => modelsStore.status.isDownloadInProgress(option.model) || @@ -142,23 +172,57 @@ const { hidden, local } = splitHiddenQuants(localRest, (option) => modelsStore.isHidden(option.id) ); + const byBackend = new SvelteMap(); + + for (const entry of local) { + const backendId = entry.base.backendId ?? LOCAL_BACKEND_ID; + + if (!byBackend.has(backendId)) byBackend.set(backendId, []); + + byBackend.get(backendId)!.push(entry); + } + const ordered: ModelsTableGroup[] = []; - // loaded models lead the table, then favorites, then the local block - const pushSection = (kind: ModelsTableGroupKind, items: ModelQuantGroup[]): void => { + // loaded models lead the table, then favorites, then one block per backend + const pushSection = ( + kind: ModelsTableGroup['kind'], + items: ModelQuantGroup[], + provider?: { id: string; label: string } + ): void => { if (items.length === 0) return; + const isLocal = kind === ModelsTableGroupKind.LOCAL; + ordered.push({ + backendId: provider?.id ?? (isLocal ? LOCAL_BACKEND_ID : null), + isLocal, items, - key: kind === ModelsTableGroupKind.LOCAL ? LOCAL_BACKEND_ID : kind, + key: provider?.id ?? (isLocal ? LOCAL_BACKEND_ID : kind), kind, - label: MODELS_TABLE_GROUP_LABELS[kind] + label: provider?.label ?? MODELS_TABLE_GROUP_LABELS[kind as ModelsTableGroupKind] }); }; pushSection(ModelsTableGroupKind.LOADED, loaded); pushSection(ModelsTableGroupKind.DOWNLOADING, downloads); pushSection(ModelsTableGroupKind.FAVORITES, favorites); - pushSection(ModelsTableGroupKind.LOCAL, local); + + const localItems = byBackend.get(LOCAL_BACKEND_ID); + + if (localItems?.length) pushSection(ModelsTableGroupKind.LOCAL, localItems); + + for (const backend of backendsStore.enabled) { + if (backend.id === LOCAL_BACKEND_ID) continue; + + const items = byBackend.get(backend.id); + + if (items?.length) + pushSection(ModelsTableProviderKind.PROVIDER, items, { + id: backend.id, + label: backend.name + }); + } + pushSection(ModelsTableGroupKind.HIDDEN, hidden); return ordered; @@ -201,9 +265,11 @@ bind:contextLimit bind:filter bind:modalities={modalityFilter} + bind:providers={providerFilter} {groups} {isFavorite} onSelect={(option) => uiStore.openModelDetails(option.id)} + {providerCounts} {selectedId} {toolbarEnd} /> diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte index 27aaf8b2bdc0..69ebd9a9e615 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte @@ -1,12 +1,16 @@ +{#snippet providerMark(backend: Backend)} + {#if backend.id === LOCAL_BACKEND_ID} + + {#snippet fallback()} + + {/snippet} + + {:else} + + {/if} +{/snippet} + @@ -98,6 +135,53 @@ gapSize="2" innerClass="items-center" > + {#if backends.length > 1} + + + {#snippet child({ props })} + + {/snippet} + + + + + Providers + + {#each backends as backend (backend.id)} + toggleProvider(backend.id, checked)} + > + {@render providerMark(backend)} + + {backend.name} + + {providerCounts[backend.id] ?? 0} + + {/each} + + + {#if providers.length > 0} + + + (providers = [])} + >Show every provider + {/if} + + + {/if} + (contextLimit = Number(value))} type="single" diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte index a3ee433396b4..9a1d548d63c3 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte @@ -2,8 +2,9 @@ import { Button } from '@ggml-org/llama-ui/ui/button'; import { Plus } from '@lucide/svelte'; import { BackendCard, DialogBackendForm } from '#lib/components/app/backends/index.ts'; + import { BackendProtocol } from '#lib/constants/index.ts'; import { backendsModelsStore, backendsStore, serverStore } from '#lib/stores/index.ts'; - import type { Backend, BackendProtocol } from '#lib/types/index.ts'; + import type { Backend } from '#lib/types/index.ts'; import { fade } from 'svelte/transition'; interface Props { @@ -20,18 +21,18 @@ : backendsStore.external ); // the bundled server belongs to the list whenever llama.cpp providers are in it - let showsLocal = $derived(protocol === undefined || protocol === 'llama.cpp'); + let showsLocal = $derived(protocol === undefined || protocol === BackendProtocol.COMPAT); let addTitle = $derived( - protocol === 'openai' + protocol === BackendProtocol.OPENAI ? 'Add a backend' - : protocol === 'llama.cpp' + : protocol === BackendProtocol.COMPAT ? 'Add a Llama-compatible backend' : 'Add new provider' ); let addDescription = $derived( - protocol === 'openai' + protocol === BackendProtocol.OPENAI ? 'Connect an OpenAI-compatible endpoint.' - : protocol === 'llama.cpp' + : protocol === BackendProtocol.COMPAT ? 'Point at another llama-server.' : 'External llama-server or connect an OpenAI-compatible API.' ); @@ -66,7 +67,7 @@ void backendsModelsStore.loadAll()} /> diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte index 1f25c3a5553c..a1f184f17124 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte @@ -6,7 +6,12 @@ import ModelId from '../ModelId.svelte'; import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte'; import { modelRowActions } from './row-actions'; - import { configuredContext, downloadProgressFor, isRowControlClick } from './utils'; + import { + canLoadOption, + configuredContext, + downloadProgressFor, + isRowControlClick + } from './utils'; import { ModelDownloadProgressBar } from '@ggml-org/llama-ui/models'; import { Heart, MoreHorizontal } from '@lucide/svelte'; import { ContextMenuActions, DropdownMenuActions } from '#lib/components/app/index.ts'; @@ -55,7 +60,13 @@ } // the "more actions" dropdown and the right-click menu offer the same actions - let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete, download)); + let actions = $derived( + modelRowActions( + option, + { canLoad: canLoadOption(option), download, favorite, isHidden }, + { onDelete } + ) + ); let contextMenuOpen = $state(false); diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte index 1f9bd6046b77..0f5f3ffd7761 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte @@ -45,9 +45,10 @@ ModelCapability, ModelDownloadConfirmAction, ModelsTableGroupKind, + ModelsTableProviderKind, ModelsTableSortKey } from '#lib/enums/index.ts'; - import { modelsStore, settingsStore } from '#lib/stores/index.ts'; + import { backendsModelsStore, modelsStore, settingsStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; import { groupModelFamilies, type ModelFamilyGroup } from '#lib/utils/model-families.ts'; import type { Snippet } from 'svelte'; @@ -64,6 +65,10 @@ onSelect: (option: ModelOption) => void; /** Modalities a model must support at least one of. */ modalities?: ModalityKey[]; + /** Backend ids to keep; empty keeps every provider. */ + providers?: string[]; + /** Repos each provider contributes to the current search, for the filter menu. */ + providerCounts?: Record; selectedId: string | null; /** Rendered at the toolbar's right end, past the filters. */ toolbarEnd?: Snippet; @@ -77,12 +82,16 @@ isFavorite, modalities = $bindable([]), onSelect, + providerCounts = {}, + providers = $bindable([]), selectedId, toolbarEnd }: Props = $props(); let isEmpty = $derived(groups.every((group) => group.items.length === 0)); - let hasFilters = $derived(hasActiveFilters(contextLimit, modalities, capabilities)); + let hasFilters = $derived( + hasActiveFilters(contextLimit, modalities, capabilities) || providers.length > 0 + ); /** Noun the show-more row counts in, per unit of the grouped list. */ const SHOW_MORE_NOUNS: Record = { @@ -244,6 +253,7 @@ {onSelect} option={quant} selected={selectedId === quant.id} + showProvider={entry.kind === 'providers'} /> {/each} @@ -348,6 +358,8 @@ bind:contextLimit bind:filter bind:modalities + bind:providers + {providerCounts} {toolbarEnd} /> @@ -382,12 +394,19 @@ {/if} {/snippet} + {@const backendState = group.backendId ? backendsModelsStore.get(group.backendId) : null} + void; onSelect: (option: ModelOption) => void; selected: boolean; + /** Badge the backend name instead of the quant, for a provider group. */ + showProvider?: boolean; /** Left padding in px, from the nesting depth. */ indent?: number; } - let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props(); + let { + indent = 0, + isFavorite, + onDelete, + onSelect, + option, + selected, + showProvider = false + }: Props = $props(); let favorite = $derived(isFavorite(option)); let isHidden = $derived(modelsStore.isHidden(option.id)); let quant = $derived(option.parsedId?.quantization ?? option.model); // the "more actions" dropdown and the right-click menu offer the same actions - let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete)); + let actions = $derived( + modelRowActions(option, { canLoad: canLoadOption(option), favorite, isHidden }, { onDelete }) + ); let contextMenuOpen = $state(false); @@ -59,7 +72,9 @@ style="padding-left: {indent}px" type="button" > - {quant} + + {showProvider ? (getBackend(option.backendId)?.name ?? quant) : quant} + !isAuxSidecar(badge.kind))} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte index eee8d9ebba44..231a4c00a020 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte @@ -20,10 +20,13 @@ let { entry, expanded, indent = 0, onToggle }: Props = $props(); + let providerCount = $derived(new Set(entry.quants.map((option) => option.backendId ?? '')).size); let groupLabel = $derived( - entry.kind === ModelGroupKind.VARIANTS - ? `${entry.quants.length} variants` - : `${entry.quants.length} quants available` + entry.kind === ModelGroupKind.PROVIDERS + ? `${providerCount} provider${providerCount === 1 ? '' : 's'}` + : entry.kind === ModelGroupKind.VARIANTS + ? `${entry.quants.length} variants` + : `${entry.quants.length} quants available` ); let anyLoaded = $derived(entry.quants.some((quant) => modelsStore.isModelRunning(quant.model))); // a repo row stands for its quants, so it reports what they agree on diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte index 7e0eaf5eb746..0ec3a877f229 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte @@ -1,6 +1,7 @@ {#if download} -{:else if isRouter} +{:else if canToggleLoad} +{:else if option.backendId && option.backendId !== LOCAL_BACKEND_ID} + {:else} diff --git a/ui/apps/chat/src/lib/stores/models/index.svelte.ts b/ui/apps/chat/src/lib/stores/models/index.svelte.ts index 66ce649be7d5..b5e04dcf85e0 100644 --- a/ui/apps/chat/src/lib/stores/models/index.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/index.svelte.ts @@ -11,7 +11,6 @@ import { BackendProtocol, FAVORITE_MODELS_LOCALSTORAGE_KEY, HIDDEN_MODELS_LOCALSTORAGE_KEY, - LOCAL_BACKEND_ID, MODEL_GROUP_OPEN_LOCALSTORAGE_KEY, MODEL_REMOVE_BLOCKED_FALLBACK, MODEL_REMOVE_BLOCKED_REASONS, @@ -478,17 +477,21 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { * feed, so an external backend answers from its own model listing. */ getModelStatus(modelId: string): ServerModelStatus | null { - const backendId = this.models.find((model) => model.model === modelId)?.backendId; + // the local feed reports the models its router serves: a same-named entry + // on a remote provider must not shadow it + const model = this.routerModels.find((m) => m.id === modelId); - if (backendId && backendId !== LOCAL_BACKEND_ID) { - const option = backendsModelsStore.get(backendId).models.find((m) => m.model === modelId); + if (model) return (model.status?.value as ServerModelStatus) ?? null; - return (option?.status?.value as ServerModelStatus) ?? null; - } + // a model the local router does not serve belongs to a backend, and the + // first enabled backend listing it owns the state + for (const backend of backendsStore.enabled) { + const option = backendsModelsStore.get(backend.id).models.find((m) => m.model === modelId); - const model = this.routerModels.find((m) => m.id === modelId); + if (option) return (option.status?.value as ServerModelStatus) ?? null; + } - return (model?.status?.value as ServerModelStatus) ?? null; + return null; } /** diff --git a/ui/apps/chat/src/lib/stores/models/status.svelte.ts b/ui/apps/chat/src/lib/stores/models/status.svelte.ts index b3cb32b38812..3cfe344426ce 100644 --- a/ui/apps/chat/src/lib/stores/models/status.svelte.ts +++ b/ui/apps/chat/src/lib/stores/models/status.svelte.ts @@ -280,7 +280,9 @@ export class ModelStatusManager { async cancelLoad(modelId: string): Promise { const backendId = this.backendIdFor(modelId); - if (!serverStore.isRouterMode) return; + // the local server takes unload requests in router mode only; a remote + // backend cancels on its own + if (backendId === LOCAL_BACKEND_ID && !serverStore.localIsRouter) return; this.subscribe(); @@ -605,7 +607,9 @@ export class ModelStatusManager { subscribe(): void { if (this.statusReaderActive) return; - if (!serverStore.isRouterMode) return; + // the feed reports the local server's models, so the local server's role + // decides, not the active backend's + if (!serverStore.localIsRouter) return; this.statusReaderActive = true; this.statusAbort = new AbortController(); From 82cae97ebd7ce34edcee116916f24c2ac33f1ac7 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 19 Oct 2026 12:48:28 +0200 Subject: [PATCH 12/34] ui : keep llama.cpp servers as the only providers Drop the OpenAI-compatible backend dialect from the providers work: every backend now speaks the llama.cpp API, so there is nothing to negotiate. Goes with it: the protocol enum and its capability and wire-quirk tables, the adapter registry (it had a single implementation, and both protocols resolved to it), the chat and models path overrides, the endpoint probe that classified a URL as llama or OpenAI, and the hosted-provider presets with their recommendation block. ChatService now builds llama.cpp requests directly and decodes its stream through chat-wire, keeping the timings the server reports instead of the wall-clock fallback a foreign endpoint needs. The gates that read those capabilities collapse to their llama.cpp branch: load, unload, slots, props, the resumable-stream endpoints and pre-encode are simply available. A row of another server loads through that server, and the switch that keeps local server state now keys on the built-in backend rather than on a protocol. --- .../app/backends/BackendCard.svelte | 9 +- .../app/backends/BackendForm.svelte | 121 +---------- .../app/backends/BackendIcon.svelte | 12 +- .../app/backends/BackendPresetCard.svelte | 41 ---- .../app/backends/BackendPresetIcon.svelte | 34 ---- .../app/backends/DialogBackendForm.svelte | 152 +------------- .../src/lib/components/app/backends/index.ts | 2 - .../components/app/models/ModelAvatar.svelte | 14 +- .../app/models/ModelLoadControl.svelte | 38 +--- .../models/ModelsManager/ModelsManager.svelte | 27 +-- .../ModelsManagerModelConfiguration.svelte | 3 +- ...delsManagerModelConfigurationHeader.svelte | 2 - ...anagerModelConfigurationInformation.svelte | 31 ++- .../ModelsManagerModelProviders.svelte | 34 +--- .../ModelsManagerModelRow.svelte | 15 +- .../ModelsManagerModelsTable.svelte | 3 - .../ModelsManagerQuantRow.svelte | 6 +- .../ModelsManagerStatusCell.svelte | 20 +- .../app/models/ModelsManager/row-actions.ts | 26 +-- .../app/models/ModelsManager/utils.ts | 6 - .../ModelsSelector/ModelsSelector.svelte | 1 - .../ModelsSelectorDropdown.svelte | 1 - .../ModelsSelector/ModelsSelectorList.svelte | 23 +-- .../ModelsSelectorOption.svelte | 15 +- .../src/lib/constants/backend.constants.ts | 109 +--------- .../lib/constants/models-manager.constants.ts | 3 +- ui/apps/chat/src/lib/enums/model.enums.ts | 3 +- .../lib/hooks/use-models-selector.svelte.ts | 76 +------ .../chat/src/lib/services/backends.service.ts | 58 ++---- ui/apps/chat/src/lib/services/chat-wire.ts | 119 +++++++++++ ui/apps/chat/src/lib/services/chat.service.ts | 89 +-------- .../chat/src/lib/services/protocols/index.ts | 25 --- .../chat/src/lib/services/protocols/openai.ts | 188 ------------------ .../chat/src/lib/services/protocols/types.ts | 43 ---- .../chat/src/lib/stores/chat/index.svelte.ts | 15 +- .../src/lib/stores/llama-server.svelte.ts | 20 +- .../src/lib/stores/models/index.svelte.ts | 22 +- .../src/lib/stores/models/props.svelte.ts | 3 - ui/apps/chat/src/lib/types/backend.d.ts | 100 ++-------- ui/apps/chat/src/lib/types/index.ts | 8 +- ui/apps/chat/src/lib/utils/api-headers.ts | 11 +- .../chat/src/lib/utils/api-key-validation.ts | 10 +- ui/apps/chat/src/lib/utils/backend.ts | 160 ++------------- ui/apps/chat/src/lib/utils/index.ts | 2 - ui/apps/chat/src/routes/+layout.svelte | 5 - 45 files changed, 265 insertions(+), 1440 deletions(-) delete mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendPresetCard.svelte delete mode 100644 ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte create mode 100644 ui/apps/chat/src/lib/services/chat-wire.ts delete mode 100644 ui/apps/chat/src/lib/services/protocols/index.ts delete mode 100644 ui/apps/chat/src/lib/services/protocols/openai.ts delete mode 100644 ui/apps/chat/src/lib/services/protocols/types.ts diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte index a85a54ad2a09..b451283abb61 100644 --- a/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte +++ b/ui/apps/chat/src/lib/components/app/backends/BackendCard.svelte @@ -7,15 +7,9 @@ import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip'; import { Pencil, RotateCw, Server, Trash2 } from '@lucide/svelte'; import { DialogConfirmation } from '#lib/components/app/dialogs/index.ts'; - import { BackendProtocol } from '#lib/constants/index.ts'; import { backendsModelsStore } from '#lib/stores/backendsModels.svelte.ts'; import type { Backend } from '#lib/types/index.ts'; - const PROTOCOL_LABELS: Record = { - [BackendProtocol.COMPAT]: 'Llama-compatible', - [BackendProtocol.OPENAI]: 'OpenAI-compatible' - }; - const CARD_ICON_CLASS = 'h-5 w-5'; interface Props { @@ -29,7 +23,6 @@ let { backend, isLocal = false, onDelete, onEdit, onToggle }: Props = $props(); let showDelete = $state(false); - let protocolLabel = $derived(PROTOCOL_LABELS[backend.protocol]); let displayUrl = $derived(backend.baseUrl || 'This server'); // the model listing doubles as the reachability probe @@ -39,7 +32,7 @@ let isOffline = $derived(backend.enabled && !!modelsState.error); let hasStatus = $derived(isChecking || isOnline || isOffline); let modelCount = $derived(modelsState.models.length); - let metaText = $derived(isLocal ? `Built-in · ${protocolLabel}` : protocolLabel); + let metaText = $derived(isLocal ? 'Built-in' : 'llama.cpp server'); // kick off the probe for backends the prefetch has not covered yet $effect(() => { diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte index dbf7ec9ed339..9a4dd9254920 100644 --- a/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte +++ b/ui/apps/chat/src/lib/components/app/backends/BackendForm.svelte @@ -1,60 +1,17 @@
@@ -74,8 +31,6 @@ {#if urlError}

{urlError}

- {:else if detectionLabel} -

{detectionLabel}

{/if}
@@ -117,78 +72,4 @@

- - - - {#if showAdvanced} - - {:else} - - {/if} - - Advanced - - - -
-
- API format - - - onChange({ compat: undefined, protocol: value as BackendProtocol })} - type="single" - value={backend.protocol} - > - {protocolLabel} - - - {#each PROTOCOL_OPTIONS as option (option.value)} - {option.label} - {/each} - - - -

- Detected from the endpoint. Override it when a server answers unusually. -

-
- -
-
- - - onChange({ chatPath: e.currentTarget.value || undefined })} - placeholder={DEFAULT_BACKEND_CHAT_PATH} - type="text" - value={backend.chatPath ?? ''} - /> -
- -
- - - onChange({ modelsPath: e.currentTarget.value || undefined })} - placeholder={DEFAULT_BACKEND_MODELS_PATH} - type="text" - value={backend.modelsPath ?? ''} - /> -
-
-
-
-
diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte index a22efe829268..858f5111ea70 100644 --- a/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte +++ b/ui/apps/chat/src/lib/components/app/backends/BackendIcon.svelte @@ -1,7 +1,6 @@ -{#if preset} - -{:else if faviconUrl && !failedFavicons.has(faviconUrl)} +{#if faviconUrl && !failedFavicons.has(faviconUrl)} - import BackendPresetIcon from './BackendPresetIcon.svelte'; - import { Badge } from '@ggml-org/llama-ui/ui/badge'; - import * as Card from '@ggml-org/llama-ui/ui/card'; - import { Check } from '@lucide/svelte'; - import type { BackendPreset } from '#lib/types/index.ts'; - - interface Props { - preset: BackendPreset; - onClick?: () => void; - selected?: boolean; - dimmed?: boolean; - /** A backend already points at this provider. */ - added?: boolean; - } - - let { added = false, dimmed = false, onClick, preset, selected = false }: Props = $props(); - - // an added provider is a label, not an action: no hover, no selection - let interactive = $derived(!added && Boolean(onClick)); - - - - {#if added} - - - Added - - {/if} - -
- - -

{preset.name}

-
- -

{preset.description}

-
diff --git a/ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte b/ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte deleted file mode 100644 index 2b5fc1850b3d..000000000000 --- a/ui/apps/chat/src/lib/components/app/backends/BackendPresetIcon.svelte +++ /dev/null @@ -1,34 +0,0 @@ - - -{#if iconUrl} - -{:else} - -{/if} diff --git a/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte b/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte index d2fca2eb6a6e..4cc52b967bc8 100644 --- a/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte +++ b/ui/apps/chat/src/lib/components/app/backends/DialogBackendForm.svelte @@ -1,59 +1,28 @@
- {#if !canLoad} - {#if showBackendMark} - - {:else if showRemoteMark} - - - - {#if isBackendFailed} - - {/if} - - {/if} - {:else if isLoading} + {#if isLoading} {:else} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte index d5f3df9acf96..75e940dfbe36 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -22,8 +22,6 @@ } from '#lib/enums/index.ts'; import { backendsStore, modelsStore, uiStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; - import { getBackend } from '#lib/utils/api-base.ts'; - import { getBackendCapabilities } from '#lib/utils/backend.ts'; import { filterModelOptions } from '#lib/utils/index.ts'; import type { Snippet } from 'svelte'; import { SvelteMap } from 'svelte/reactivity'; @@ -125,35 +123,18 @@ let entries = $derived(byRecency(groupModelQuants(visible))); let groups = $derived.by(() => { // a loaded quant is a model of its own: its repo keeps the quants left behind. - // Only llama-compat servers report a load state. - const isLoaded = (option: ModelOption) => - getBackendCapabilities(getBackend(option.backendId)).loadUnload && - modelsStore.isModelLoaded(option.model); + const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model); // a tracked download stands in its own section, so it is not listed twice const isDownload = (option: ModelOption) => modelsStore.status.isDownloadInProgress(option.model) || modelsStore.status.isDownloadPaused(option.model); - // a compat backend serves the selection itself: its selected model is a - // section of its own, since it never reports a load state - const isProviderSelected = (option: ModelOption) => - option.id === modelsStore.selectedModelId && - !getBackendCapabilities(getBackend(option.backendId)).loadUnload; - const selected: ModelQuantGroup[] = []; const loaded: ModelQuantGroup[] = []; const rest: ModelQuantGroup[] = []; for (const entry of entries) { - const remaining = entry.quants.filter( - (quant) => !isLoaded(quant) && !isDownload(quant) && !isProviderSelected(quant) - ); + const remaining = entry.quants.filter((quant) => !isLoaded(quant) && !isDownload(quant)); for (const quant of entry.quants) { - if (isProviderSelected(quant)) { - selected.push({ ...entry, base: quant, key: quant.id, quants: [quant] }); - - continue; - } - if (!isLoaded(quant)) continue; loaded.push({ ...entry, base: quant, key: quant.id, quants: [quant] }); @@ -197,8 +178,7 @@ } const ordered: ModelsTableGroup[] = []; - // the selected compat model leads the table, then the loaded models, then - // favorites, then one block per backend + // loaded models lead the table, then favorites, then one block per backend const pushSection = ( kind: ModelsTableGroup['kind'], items: ModelQuantGroup[], @@ -218,7 +198,6 @@ }); }; - pushSection(ModelsTableGroupKind.SELECTED, selected); pushSection(ModelsTableGroupKind.LOADED, loaded); pushSection(ModelsTableGroupKind.DOWNLOADING, downloads); pushSection(ModelsTableGroupKind.FAVORITES, favorites); diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte index 7f07272ae2d4..4af8e0d2437a 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte @@ -1,5 +1,5 @@ diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte index 7168a5bce6a5..0f5f3ffd7761 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte @@ -13,7 +13,6 @@ import { ArrowDown, ArrowUp, - CheckCircle2, ChevronDown, ChevronUp, Download, @@ -388,8 +387,6 @@ {:else if group.kind === ModelsTableGroupKind.LOADED} - {:else if group.kind === ModelsTableGroupKind.SELECTED} - {:else if group.kind === ModelsTableGroupKind.HIDDEN} {:else if group.kind === ModelsTableGroupKind.LOCAL} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte index e9c1219dc5e9..87bdc65ae0cc 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte @@ -2,7 +2,7 @@ import ModelContext from '../ModelContext.svelte'; import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte'; import { modelRowActions } from './row-actions'; - import { canLoadOption, configuredContext, isRowControlClick } from './utils'; + import { configuredContext, isRowControlClick } from './utils'; import { ModelDraftSidecars } from '@ggml-org/llama-ui/models'; import { Badge } from '@ggml-org/llama-ui/ui/badge'; import { MoreHorizontal } from '@lucide/svelte'; @@ -40,9 +40,7 @@ let quant = $derived(option.parsedId?.quantization ?? option.model); // the "more actions" dropdown and the right-click menu offer the same actions - let actions = $derived( - modelRowActions(option, { canLoad: canLoadOption(option), favorite, isHidden }, { onDelete }) - ); + let actions = $derived(modelRowActions(option, { favorite, isHidden }, { onDelete })); let contextMenuOpen = $state(false); diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte index 0a1402cfbb62..43b503a59a89 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte @@ -1,7 +1,6 @@ @@ -35,7 +30,6 @@ {:else if canToggleLoad} -{:else if option.backendId && option.backendId !== LOCAL_BACKEND_ID} - {:else} :`; the entry exists only // while Discover is on const viewInDiscover = uiStore.isDiscoverEnabled @@ -71,19 +69,15 @@ export function modelRowActions( label: favorite ? 'Remove from favorites' : 'Add to favorites', onclick: () => modelsStore.toggleFavorite(option.model) }, - ...(canLoad - ? [ - { - disabled: removeBlockedReason !== null, - icon: Trash2, - label: 'Delete from disk', - onclick: () => handlers.onDelete(option), - separator: true, - tooltip: removeBlockedReason ?? undefined, - variant: 'destructive' as const - } - ] - : []), + { + disabled: removeBlockedReason !== null, + icon: Trash2, + label: 'Delete from disk', + onclick: () => handlers.onDelete(option), + separator: true, + tooltip: removeBlockedReason ?? undefined, + variant: 'destructive' as const + }, { icon: isHidden ? Eye : EyeOff, label: isHidden ? 'Unhide model' : 'Hide model', diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts b/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts index 9504d18f7853..1a2b420dd2f4 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/utils.ts @@ -14,7 +14,6 @@ import type { ModelSidecarBadge } from '#lib/types/models.js'; import { getBackend } from '#lib/utils/api-base.ts'; -import { getBackendCapabilities } from '#lib/utils/backend.ts'; import { detectThinkingSupport, detectToolUseSupport, repoOf } from '#lib/utils/index.ts'; /** One repo of the table, with the rows it ships as. */ @@ -142,11 +141,6 @@ export function configuredContext(option: ModelOption): number | null { : null; } -/** True when the backend that serves the model can load and unload it. */ -export function canLoadOption(option: ModelOption): boolean { - return getBackendCapabilities(getBackend(option.backendId)).loadUnload; -} - export function servedByLabel(option: ModelOption): string { const backend = getBackend(option.backendId); diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte index a976bb66295c..a2742c334e12 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte @@ -356,7 +356,6 @@ onSelect={ms.handleSelect} renderOption={modelOption} sectionHeaderClass="[&:not(:first-child)]:mt-2 mb-1 px-2 py-2.5 text-sm font-semibold text-foreground/80 select-none" - selected={ms.selectedItems} {showOrgName} />
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte index 5f2dcd85985e..1ea0dcb97419 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte @@ -290,7 +290,6 @@ manager's other entry points --> onSelect={ms.handleSelect} renderOption={modelOption} sectionHeaderClass="[&:not(:first-child)]:mt-1 mb-1 px-2 py-2 text-[13px] font-semibold text-foreground/80 select-none" - selected={ms.selectedItems} {showOrgName} /> diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte index c1a6fdef48a8..268696a833da 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte @@ -1,7 +1,7 @@
-
- - - onChange({ baseUrl: e.currentTarget.value })} - placeholder="https://api.example.com" - type="url" - value={backend.baseUrl} - /> - - {#if urlError} -

{urlError}

- {/if} +
+ {#each [BackendKind.URL, BackendKind.REMOTE] as kind (kind)} + + {/each}
+ {#if isRemote} +
+ + +
+ onChange({ code: e.currentTarget.value })} + placeholder="Paste the {CODE_LENGTHS.SHARE}-character code" + spellcheck="false" + type="text" + value={backend.code ?? ''} + /> + + {#if canScan} + + {/if} +
+ + {#if codeError} +

{codeError}

+ {:else} +

+ Printed by llama-server --connect. It is a credential: + anyone who has it can use that server. +

+ {/if} +
+ {:else} +
+ + + onChange({ baseUrl: e.currentTarget.value })} + placeholder="https://api.example.com" + type="url" + value={backend.baseUrl} + /> + + {#if urlError} +

{urlError}

+ {/if} +
+ {/if} +
+ handleChange({ code: scanned })} + /> +
- {#if testing} + {#if pairing} + + +
+

Pairing

+ +

Opening the tunnel to that server.

+
+ {:else if testing}
@@ -171,6 +268,18 @@ Asking {hostOf(draft.baseUrl)} for the models it serves.

+ {:else if pairError} + + +
+

Pairing failed

+ +

{pairError}

+ +

+ The provider is saved; retry from its card. +

+
{:else if testResult?.ok} @@ -195,13 +304,21 @@

{:else} - + {#if isRemote} + + {:else} + + {/if}
-

Connection not tested

+

+ {isRemote ? 'Not paired yet' : 'Connection not tested'} +

- Test it to list the models this provider serves. + {isRemote + ? 'Pairing saves the provider and connects to it.' + : 'Test it to list the models this provider serves.'}

{/if} @@ -212,13 +329,25 @@ Cancel - - - + {#if isRemote} + + {:else} + + + + {/if} diff --git a/ui/apps/chat/src/lib/components/app/backends/index.ts b/ui/apps/chat/src/lib/components/app/backends/index.ts index 7b18cf2c44d2..3da88d367307 100644 --- a/ui/apps/chat/src/lib/components/app/backends/index.ts +++ b/ui/apps/chat/src/lib/components/app/backends/index.ts @@ -1,4 +1,5 @@ export { default as BackendCard } from './BackendCard.svelte'; export { default as BackendForm } from './BackendForm.svelte'; export { default as BackendIcon } from './BackendIcon.svelte'; +export { default as DialogBackendCodeScanner } from './DialogBackendCodeScanner.svelte'; export { default as DialogBackendForm } from './DialogBackendForm.svelte'; diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte index f8b7cc9c1625..637cab82a5a8 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte @@ -1,8 +1,14 @@ {#if hasError || isLoadingModel} diff --git a/ui/apps/chat/src/lib/components/app/index.ts b/ui/apps/chat/src/lib/components/app/index.ts index 24b717cb69f0..302f0a2f06ad 100644 --- a/ui/apps/chat/src/lib/components/app/index.ts +++ b/ui/apps/chat/src/lib/components/app/index.ts @@ -1,5 +1,4 @@ export * from './actions'; -export * from './backends'; export * from './badges'; export * from './chat'; export * from './content'; diff --git a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte index 2218f42cd08e..994101386db8 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelAvatar.svelte @@ -1,7 +1,7 @@ -{#if useProviderIcon} +{#if useServerIcon} - + {#snippet fallback()} {#if isLocal} @@ -109,7 +109,7 @@ {/if} {/snippet} - + {:else if orgName && hubEnabled} ([]); + let serverFilter = $state([]); let contextLimit = $state(0); let modalityFilter = $state([]); let capabilityFilter = $state([]); @@ -55,8 +55,8 @@ let isFavorite = $derived((option: ModelOption) => modelsStore.favoriteModelIds.has(option.model) ); - // every filter but the provider one, so a provider count does not fall to zero - // the moment that provider is the one being looked at + // every filter but the server one, so a server count does not fall to zero + // the moment that server is the one being looked at let matching = $derived.by(() => { // the term matches what the selector search matches: name, model, aliases and tags const searched = filterModelOptions(allModels, filter); @@ -87,19 +87,19 @@ }); }); let visible = $derived.by(() => - providerFilter.length === 0 + serverFilter.length === 0 ? matching - : matching.filter((option) => providerFilter.includes(option.backendId ?? LOCAL_BACKEND_ID)) + : matching.filter((option) => serverFilter.includes(option.serverId ?? LOCAL_BACKEND_ID)) ); // the rail counts follow the active view and filter, so it always says how many - // repos each provider contributes to what the table is showing - let providerCounts = $derived.by(() => { + // repos each server contributes to what the table is showing + let serverCounts = $derived.by(() => { const counts: Record = {}; for (const entry of groupModelQuants(matching)) { - const backendId = entry.base.backendId ?? LOCAL_BACKEND_ID; + const serverId = entry.base.serverId ?? LOCAL_BACKEND_ID; - counts[backendId] = (counts[backendId] ?? 0) + 1; + counts[serverId] = (counts[serverId] ?? 0) + 1; } return counts; @@ -119,7 +119,7 @@ const byRecency = (list: ModelQuantGroup[]) => rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b)); // one entry per repo, so a model with several quants takes a single table row; - // loaded models lead the table, then favorites, then one block per provider + // loaded models lead the table, then favorites, then one block per server let entries = $derived(byRecency(groupModelQuants(visible))); let groups = $derived.by(() => { // a loaded quant is a model of its own: its repo keeps the quants left behind. @@ -167,34 +167,34 @@ const { hidden, local } = splitHiddenQuants(localRest, (option) => modelsStore.isHidden(option.id) ); - const byBackend = new SvelteMap(); + const byServer = new SvelteMap(); for (const entry of local) { - const backendId = entry.base.backendId ?? LOCAL_BACKEND_ID; + const serverId = entry.base.serverId ?? LOCAL_BACKEND_ID; - if (!byBackend.has(backendId)) byBackend.set(backendId, []); + if (!byServer.has(serverId)) byServer.set(serverId, []); - byBackend.get(backendId)!.push(entry); + byServer.get(serverId)!.push(entry); } const ordered: ModelsTableGroup[] = []; - // loaded models lead the table, then favorites, then one block per backend + // loaded models lead the table, then favorites, then one block per server const pushSection = ( kind: ModelsTableGroup['kind'], items: ModelQuantGroup[], - provider?: { id: string; label: string } + server?: { id: string; label: string } ): void => { if (items.length === 0) return; const isLocal = kind === ModelsTableGroupKind.LOCAL; ordered.push({ - backendId: provider?.id ?? (isLocal ? LOCAL_BACKEND_ID : null), isLocal, items, - key: provider?.id ?? (isLocal ? LOCAL_BACKEND_ID : kind), + key: server?.id ?? (isLocal ? LOCAL_BACKEND_ID : kind), kind, - label: provider?.label ?? MODELS_TABLE_GROUP_LABELS[kind as ModelsTableGroupKind] + label: server?.label ?? MODELS_TABLE_GROUP_LABELS[kind as ModelsTableGroupKind], + serverId: server?.id ?? (isLocal ? LOCAL_BACKEND_ID : null) }); }; @@ -202,19 +202,19 @@ pushSection(ModelsTableGroupKind.DOWNLOADING, downloads); pushSection(ModelsTableGroupKind.FAVORITES, favorites); - const localItems = byBackend.get(LOCAL_BACKEND_ID); + const localItems = byServer.get(LOCAL_BACKEND_ID); if (localItems?.length) pushSection(ModelsTableGroupKind.LOCAL, localItems); - for (const backend of backendsStore.enabled) { - if (backend.id === LOCAL_BACKEND_ID) continue; + for (const server of serversStore.enabled) { + if (server.id === LOCAL_BACKEND_ID) continue; - const items = byBackend.get(backend.id); + const items = byServer.get(server.id); if (items?.length) - pushSection(ModelsTableProviderKind.PROVIDER, items, { - id: backend.id, - label: backend.name + pushSection(ModelsTableServerKind.SERVER, items, { + id: server.id, + label: server.name }); } @@ -260,12 +260,12 @@ bind:contextLimit bind:filter bind:modalities={modalityFilter} - bind:providers={providerFilter} + bind:servers={serverFilter} {groups} {isFavorite} onSelect={(option) => uiStore.openModelDetails(option.id)} - {providerCounts} {selectedId} + {serverCounts} {toolbarEnd} />
diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte index 69ebd9a9e615..24936fa651e5 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte @@ -2,9 +2,9 @@ import * as DropdownMenu from '@ggml-org/llama-ui/ui/dropdown-menu'; import * as Select from '@ggml-org/llama-ui/ui/select'; import * as ToggleGroup from '@ggml-org/llama-ui/ui/toggle-group'; - import { ChevronDown, Server } from '@lucide/svelte'; - import { BackendIcon } from '#lib/components/app/backends/index.ts'; + import { ChevronDown, Server as ServerGlyph } from '@lucide/svelte'; import { Logo, ScrollCarousel } from '#lib/components/app/index.ts'; + import { ServerIcon } from '#lib/components/app/servers/index.ts'; import { CAPABILITY_ICONS, CAPABILITY_LABELS, @@ -20,30 +20,31 @@ } from '#lib/constants/index.ts'; import { ModelCapability } from '#lib/enums/index.ts'; import { modelsStore } from '#lib/stores/index.ts'; - import type { Backend } from '#lib/types/backend.js'; + import type { Server } from '#lib/types/servers.js'; import { SvelteSet } from 'svelte/reactivity'; interface Props { - backends: Backend[]; + /** Servers the filter can offer: the enabled ones. */ + availableServers: Server[]; /** Capabilities a model must have every one of. */ capabilities?: ModelCapability[]; /** Smallest context a model must support; 0 keeps every model. */ contextLimit?: number; /** Modalities a model must support at least one of. */ modalities?: ModalityKey[]; - /** Backend ids to keep; empty keeps every provider. */ - providers?: string[]; - /** Repos each provider contributes to the current search, shown in the menu. */ - providerCounts?: Record; + /** Server ids to keep; empty keeps every server. */ + servers?: string[]; + /** Repos each server contributes to the current search, shown in the menu. */ + serverCounts?: Record; } let { - backends, + availableServers, capabilities = $bindable([]), contextLimit = $bindable(0), modalities = $bindable([]), - providerCounts = {}, - providers = $bindable([]) + serverCounts = {}, + servers = $bindable([]) }: Props = $props(); const CONTEXT_STEPS: { label: string; value: number }[] = [ @@ -96,32 +97,32 @@ modalities = values.filter((value): value is ModalityKey => MODALITY_VALUES.has(value)); } - // none selected means every provider, so the label names the selection - let providerLabel = $derived( - providers.length === 0 - ? 'All providers' - : providers.length === 1 - ? (backends.find((backend) => backend.id === providers[0])?.name ?? '1 provider') - : `${providers.length} providers` + // none selected means every server, so the label names the selection + let serverLabel = $derived( + servers.length === 0 + ? 'All servers' + : servers.length === 1 + ? (availableServers.find((server) => server.id === servers[0])?.name ?? '1 server') + : `${servers.length} servers` ); let contextLabel = $derived( CONTEXT_STEPS.find((step) => step.value === contextLimit)?.label ?? CONTEXT_STEPS[0].label ); - function toggleProvider(id: string, checked: boolean | 'indeterminate'): void { - providers = checked === true ? [...providers, id] : providers.filter((entry) => entry !== id); + function toggleServer(id: string, checked: boolean | 'indeterminate'): void { + servers = checked === true ? [...servers, id] : servers.filter((entry) => entry !== id); } -{#snippet providerMark(backend: Backend)} - {#if backend.id === LOCAL_BACKEND_ID} - +{#snippet serverMark(server: Server)} + {#if server.id === LOCAL_BACKEND_ID} + {#snippet fallback()} {/snippet} - + {:else} - + {/if} {/snippet} @@ -135,7 +136,7 @@ gapSize="2" innerClass="items-center" > - {#if backends.length > 1} + {#if availableServers.length > 1} {#snippet child({ props })} @@ -144,9 +145,9 @@ class="inline-flex items-center whitespace-nowrap {FILTER_TRIGGER_CLASS}" type="button" > - + - {providerLabel} + {serverLabel} @@ -155,28 +156,26 @@ - Providers + Servers - {#each backends as backend (backend.id)} + {#each availableServers as server (server.id)} toggleProvider(backend.id, checked)} + checked={servers.includes(server.id)} + onCheckedChange={(checked) => toggleServer(server.id, checked)} > - {@render providerMark(backend)} + {@render serverMark(server)} - {backend.name} + {server.name} - {providerCounts[backend.id] ?? 0} + {serverCounts[server.id] ?? 0} {/each} - {#if providers.length > 0} + {#if servers.length > 0} - (providers = [])} - >Show every provider + (servers = [])}>Show every server {/if} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte index 9f0183e3188f..f9dbab94f505 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte @@ -14,7 +14,7 @@ import { conversationsStore, deviceStore, uiStore } from '#lib/stores/index.ts'; interface Props { - /** A provider that cannot load or unload has no button to offer. */ + /** A server that cannot load or unload has no button to offer. */ canToggleLoad?: boolean; isDownload?: boolean; isDownloadPaused?: boolean; diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte index 0f5f3ffd7761..9f66a93d75a9 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte @@ -45,10 +45,10 @@ ModelCapability, ModelDownloadConfirmAction, ModelsTableGroupKind, - ModelsTableProviderKind, + ModelsTableServerKind, ModelsTableSortKey } from '#lib/enums/index.ts'; - import { backendsModelsStore, modelsStore, settingsStore } from '#lib/stores/index.ts'; + import { modelsStore, serversModelsStore, settingsStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; import { groupModelFamilies, type ModelFamilyGroup } from '#lib/utils/model-families.ts'; import type { Snippet } from 'svelte'; @@ -65,10 +65,10 @@ onSelect: (option: ModelOption) => void; /** Modalities a model must support at least one of. */ modalities?: ModalityKey[]; - /** Backend ids to keep; empty keeps every provider. */ - providers?: string[]; - /** Repos each provider contributes to the current search, for the filter menu. */ - providerCounts?: Record; + /** Server ids to keep; empty keeps every server. */ + servers?: string[]; + /** Repos each server contributes to the current search, for the filter menu. */ + serverCounts?: Record; selectedId: string | null; /** Rendered at the toolbar's right end, past the filters. */ toolbarEnd?: Snippet; @@ -82,15 +82,15 @@ isFavorite, modalities = $bindable([]), onSelect, - providerCounts = {}, - providers = $bindable([]), selectedId, + serverCounts = {}, + servers = $bindable([]), toolbarEnd }: Props = $props(); let isEmpty = $derived(groups.every((group) => group.items.length === 0)); let hasFilters = $derived( - hasActiveFilters(contextLimit, modalities, capabilities) || providers.length > 0 + hasActiveFilters(contextLimit, modalities, capabilities) || servers.length > 0 ); /** Noun the show-more row counts in, per unit of the grouped list. */ @@ -253,7 +253,7 @@ {onSelect} option={quant} selected={selectedId === quant.id} - showProvider={entry.kind === 'providers'} + showServer={entry.kind === 'servers'} /> {/each} @@ -358,8 +358,8 @@ bind:contextLimit bind:filter bind:modalities - bind:providers - {providerCounts} + bind:servers + {serverCounts} {toolbarEnd} /> @@ -394,22 +394,22 @@ {/if} {/snippet} - {@const backendState = group.backendId ? backendsModelsStore.get(group.backendId) : null} + {@const serverState = group.serverId ? serversModelsStore.get(group.serverId) : null} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte index 87bdc65ae0cc..bb3e9b99cba0 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte @@ -10,7 +10,7 @@ import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '#lib/constants/index.ts'; import { modelsStore } from '#lib/stores/index.ts'; import type { ModelOption } from '#lib/types/models.js'; - import { getBackend } from '#lib/utils/api-base.ts'; + import { getServer } from '#lib/utils/api-base.ts'; import { isAuxSidecar } from '#lib/utils/index.ts'; interface Props { @@ -19,8 +19,8 @@ onDelete: (option: ModelOption) => void; onSelect: (option: ModelOption) => void; selected: boolean; - /** Badge the backend name instead of the quant, for a provider group. */ - showProvider?: boolean; + /** Badge the server name instead of the quant, for a server group. */ + showServer?: boolean; /** Left padding in px, from the nesting depth. */ indent?: number; } @@ -32,7 +32,7 @@ onSelect, option, selected, - showProvider = false + showServer = false }: Props = $props(); let favorite = $derived(isFavorite(option)); @@ -71,7 +71,7 @@ type="button" > - {showProvider ? (getBackend(option.backendId)?.name ?? quant) : quant} + {showServer ? (getServer(option.serverId)?.name ?? quant) : quant} option.backendId ?? '')).size); + let serverCount = $derived(new Set(entry.quants.map((option) => option.serverId ?? '')).size); let groupLabel = $derived( - entry.kind === ModelGroupKind.PROVIDERS - ? `${providerCount} provider${providerCount === 1 ? '' : 's'}` + entry.kind === ModelGroupKind.SERVERS + ? `${serverCount} server${serverCount === 1 ? '' : 's'}` : entry.kind === ModelGroupKind.VARIANTS ? `${entry.quants.length} variants` : `${entry.quants.length} quants available` diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte similarity index 55% rename from ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte rename to ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte index 637cab82a5a8..42186e7d6ee7 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelProviders.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerServers.svelte @@ -1,15 +1,15 @@ diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte index b8524c987600..f91263ab5f94 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte @@ -6,7 +6,7 @@ import { SearchInput } from '#lib/components/app/forms/index.ts'; import { type ModalityKey } from '#lib/constants/index.ts'; import { ModelCapability } from '#lib/enums/index.ts'; - import { backendsStore, deviceStore } from '#lib/stores/index.ts'; + import { deviceStore, serversStore } from '#lib/stores/index.ts'; import type { Snippet } from 'svelte'; interface Props { @@ -17,10 +17,10 @@ filter?: string; /** Modalities a model must support at least one of. */ modalities?: ModalityKey[]; - /** Repos each provider contributes to the current search, for the filter menu. */ - providerCounts?: Record; - /** Backend ids to keep; empty keeps every provider. */ - providers?: string[]; + /** Repos each server contributes to the current search, for the filter menu. */ + serverCounts?: Record; + /** Server ids to keep; empty keeps every server. */ + servers?: string[]; /** Rendered at the toolbar's right end, past the filters. */ toolbarEnd?: Snippet; } @@ -30,13 +30,13 @@ contextLimit = $bindable(0), filter = $bindable(''), modalities = $bindable([]), - providerCounts = {}, - providers = $bindable([]), + serverCounts = {}, + servers = $bindable([]), toolbarEnd }: Props = $props(); let hasFilters = $derived( - hasActiveFilters(contextLimit, modalities, capabilities) || providers.length > 0 + hasActiveFilters(contextLimit, modalities, capabilities) || servers.length > 0 ); let filterInput = $state(null); @@ -73,16 +73,16 @@ bind:capabilities bind:contextLimit bind:modalities - bind:providers - backends={backendsStore.enabled} - {providerCounts} + bind:servers + availableServers={serversStore.enabled} + {serverCounts} /> {#if hasFilters} {:else}

{displayUrl}

@@ -132,8 +132,8 @@
onToggle?.(value)} />
@@ -191,16 +191,16 @@
{#if isIdle} - {/if} {#if remote && tunnel === TunnelStatus.CONNECTED} - {/each} -
+ + onChange({ kind: value as ServerKind })} + value={server.kind} + > + + {#each [ServerKind.URL, ServerKind.REMOTE] as kind (kind)} + + {kind === ServerKind.REMOTE ? 'Remote server' : 'Server URL'} + + {/each} + + {#if isRemote}
-
{:else}
-
- onToggle?.(value)} - /> + {#if remote} + + + {#snippet child({ props })} + onToggle?.(value)} + /> + {/snippet} + + + +

Disabling also closes this server's tunnel.

+
+
+ {:else} + onToggle?.(value)} + /> + {/if} - {#if isOffline && errorText} - - -

{errorText}

-
+ {#if hostCode} +
+
+
+

Remote access: on

+ +

+ This server can be reached through its access code. +

+
+ +
+ + + +
+
+
+ {/if} - -

{errorText}

-
-
+ {#if status.detail} +

+ {status.detail} +

{/if}
- {#if isConnecting} + {#if isConnecting || isChecking} - - Connecting - {:else if isChecking} - - - Checking {:else if isOnline} - - {modelCount === 1 ? '1 model' : `${modelCount} models`} {:else if isOffline} - - {remote ? 'Disconnected' : 'Unreachable'} - - {:else if isIdle} - - Not connected {/if} {#if hasStatus} + {status.label} + + {#if isOffline} + + {/if} + · {/if} @@ -196,12 +251,15 @@ {/if} - {#if remote && tunnel === TunnelStatus.CONNECTED} + {#if remote && tunnel !== TunnelStatus.IDLE}
+ +
+ + {#if showHostQr && hostQr} +
+ Access code QR code + +

Scan it with the other device.

+
+ {/if} {/if} diff --git a/ui/apps/chat/src/lib/utils/index.ts b/ui/apps/chat/src/lib/utils/index.ts index 318ec09e5e70..2e3ff869ca0d 100644 --- a/ui/apps/chat/src/lib/utils/index.ts +++ b/ui/apps/chat/src/lib/utils/index.ts @@ -47,12 +47,13 @@ export { tunnelHostLabel, tunnelOrigin } from './servers'; +export { serverCardStatusOf } from './server-status'; +export type { ServerCardFacts, ServerCardStatus } from './server-status'; // Camera access code scanning (native BarcodeDetector where the browser has one) export { scanFrameForCode, supportsQrScan } from './qr-scan'; -// What a server card reports about its tunnel and its listing -export { serverCardStatusOf } from './server-status'; -export type { ServerCardFacts, ServerCardStatus } from './server-status'; +// Access code as a QR code, the other end of the scanner above +export { qrCodeDataUrl } from './qr-code'; // Attachment utilities export { getAttachmentDisplayItems, isMcpPrompt, isMcpResource } from './attachment-display'; diff --git a/ui/apps/chat/src/lib/utils/qr-code.ts b/ui/apps/chat/src/lib/utils/qr-code.ts new file mode 100644 index 000000000000..94d002490bed --- /dev/null +++ b/ui/apps/chat/src/lib/utils/qr-code.ts @@ -0,0 +1,39 @@ +/** + * QR code of an access code, for carrying it from the screen it is shown on to + * the camera of another device. + * + * The payload is the code itself, in the shape the scanner accepts: the scanner + * hands whatever it decodes to `normalizeShareCode`, so a code pasted by hand + * and a code scanned off a screen go through the same door. + */ + +import { normalizeShareCode } from './servers'; +import { renderSVG } from 'uqr'; + +/** Module size of the rendered code, in SVG units. The image is sized by CSS. */ +const PIXEL_SIZE = 4; +/** Quiet zone, in modules on each side; the spec asks for four, a camera needs them. */ +const BORDER = 4; + +/** + * Data URL of an SVG QR code carrying the access code, or null when the value + * is not a usable code: a QR a phone cannot pair with is worse than no QR. + * + * Black on white, pinned: a scanner reads dark modules on a light field, and + * the app's dark theme must not invert that. + */ +export function qrCodeDataUrl(code: string): string | null { + const normalized = normalizeShareCode(code); + + if (!normalized) return null; + + const svg = renderSVG(normalized, { + blackColor: '#000000', + border: BORDER, + ecc: 'M', + pixelSize: PIXEL_SIZE, + whiteColor: '#ffffff' + }); + + return `data:image/svg+xml;utf8,${encodeURIComponent(svg)}`; +} From c63bcc0c5008748690609f123c15c7aa0200bde5 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Tue, 20 Oct 2026 10:00:57 +0200 Subject: [PATCH 20/34] ui : give servers a dialog of their own The models dialog stays desktop-only, so a phone had no way into its server list; the sidebar now carries a Servers entry that opens the list on its own. Assisted-by: DeepSeek-V4.1-Flash --- .../SidebarNavigation.svelte | 7 +++ .../SidebarNavigationActions.svelte | 3 +- .../app/servers/DialogServers.svelte | 46 +++++++++++++++++++ .../src/lib/components/app/servers/index.ts | 1 + .../chat/src/lib/constants/ui.constants.ts | 7 ++- ui/apps/chat/src/lib/enums/ui.enums.ts | 1 + ui/apps/chat/src/lib/stores/ui.svelte.ts | 9 ++++ 7 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 ui/apps/chat/src/lib/components/app/servers/DialogServers.svelte diff --git a/ui/apps/chat/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte b/ui/apps/chat/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte index 0f7ba7e964bc..1f42f8d751c2 100644 --- a/ui/apps/chat/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte +++ b/ui/apps/chat/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte @@ -10,6 +10,7 @@ DialogProject, DialogProjectPicker, DialogSearch, + DialogServers, DialogSettingsChat, Logo, ModelsManagerModelDrawer, @@ -549,6 +550,10 @@ case SidebarAction.MANAGE_MODELS: openDialog(() => uiStore.openModelsManager()); + break; + case SidebarAction.SERVERS: + openDialog(() => uiStore.openServers()); + break; // a dialog on a desktop or tablet; on a phone, a page of its own case SidebarAction.SETTINGS: @@ -807,6 +812,8 @@ + +