From 70eecd59aed68483fafac14207b5d33988c04c0a Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 09:17:27 +0100 Subject: [PATCH 01/10] ui: let the dev proxy pass llama-server's cross-origin check llama-server refuses browser requests whose Origin does not match its own host unless --cors-origins allows it, and the Vite proxy forwarded the page's Origin (localhost:5173, or a LAN or Tailscale address). Loading a model, saving tools and /v1 calls failed with 403 under `vite dev`. The proxy now calls the server with its own Host and no Origin, WebSocket upgrades included. Sec-Fetch-Site still passes through, so requests from other sites stay refused. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/vite.config.ts | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/ui/apps/chat/vite.config.ts b/ui/apps/chat/vite.config.ts index cf88572a9c2f..5c69a43c078e 100644 --- a/ui/apps/chat/vite.config.ts +++ b/ui/apps/chat/vite.config.ts @@ -13,7 +13,7 @@ import { playwright } from '@vitest/browser-playwright'; import { mdsvex } from 'mdsvex'; import { dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; -import { defineConfig, loadEnv, searchForWorkspaceRoot } from 'vite'; +import { defineConfig, loadEnv, type ProxyOptions, searchForWorkspaceRoot } from 'vite'; const __dirname = dirname(fileURLToPath(import.meta.url)); // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -35,6 +35,18 @@ const outDir = process.env.LLAMA_UI_OUT_DIR ?? './dist'; export default defineConfig(({ mode }) => { const env = loadEnv(mode, process.cwd(), 'VITE_PUBLIC_'); const SERVER_ORIGIN = env.VITE_PUBLIC_SERVER_ORIGIN || 'http://localhost:9931'; + // llama-server takes cross-origin browser requests only from --cors-origins, so the proxy + // calls it as its own page: the Host of the server and no Origin. Sec-Fetch-Site still + // passes through, so the server keeps refusing requests from other sites. + const toServer = (ws = false): ProxyOptions => ({ + changeOrigin: true, + configure: (proxy) => { + proxy.on('proxyReq', (req) => req.removeHeader('origin')); + proxy.on('proxyReqWs', (req) => req.removeHeader('origin')); + }, + target: SERVER_ORIGIN, + ws + }); return { build: { @@ -90,12 +102,12 @@ export default defineConfig(({ mode }) => { 'Cross-Origin-Opener-Policy': 'same-origin' }, proxy: { - '/cors-proxy': SERVER_ORIGIN, - '/models': SERVER_ORIGIN, - '/props': SERVER_ORIGIN, - '/slots': SERVER_ORIGIN, - '/tools': SERVER_ORIGIN, - '/v1': SERVER_ORIGIN + '/cors-proxy': toServer(), + '/models': toServer(), + '/props': toServer(), + '/slots': toServer(), + '/tools': toServer(), + '/v1': toServer() } }, From 38845aa144a3fb94951ecca577b8018c9e80d739 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 15:13:18 +0100 Subject: [PATCH 02/10] ui: run Llama Chat with npx @ggml-org/llama-chat Makes @ggml-org/llama-chat publishable: drops private, adds bin, files, exports, repository and a prepack build. Its CLI serves the prebuilt dist on localhost (3000 by default) and forwards every other request to a llama-server (--server, default http://127.0.0.1:8080), with the headers llama-server sends for the UI. A thin unscoped llama-chat package in alias/ forwards to it, so npx llama-chat works. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/LICENSE | 21 ++ ui/apps/chat/README.md | 69 +++++ ui/apps/chat/alias/LICENSE | 21 ++ ui/apps/chat/alias/README.md | 9 + ui/apps/chat/alias/cli.js | 3 + ui/apps/chat/alias/package.json | 33 +++ ui/apps/chat/bin/cli.js | 5 + ui/apps/chat/cli/server.js | 386 ++++++++++++++++++++++++++++ ui/apps/chat/package-lock.json | 11 +- ui/apps/chat/package.json | 44 +++- ui/apps/chat/tests/unit/cli.test.ts | 192 ++++++++++++++ 11 files changed, 789 insertions(+), 5 deletions(-) create mode 100644 ui/apps/chat/LICENSE create mode 100644 ui/apps/chat/alias/LICENSE create mode 100644 ui/apps/chat/alias/README.md create mode 100755 ui/apps/chat/alias/cli.js create mode 100644 ui/apps/chat/alias/package.json create mode 100755 ui/apps/chat/bin/cli.js create mode 100644 ui/apps/chat/cli/server.js create mode 100644 ui/apps/chat/tests/unit/cli.test.ts diff --git a/ui/apps/chat/LICENSE b/ui/apps/chat/LICENSE new file mode 100644 index 000000000000..50c59c51c158 --- /dev/null +++ b/ui/apps/chat/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) ggml + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 1e3bf68f3d4d..4cd8c9881abe 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -11,6 +11,8 @@ Llama UI supports two server operation modes: ## Table of Contents +- [Run it with npx](#run-it-with-npx) +- [Self-hosting](#self-hosting) - [Features](#features) - [Getting Started](#getting-started) - [Tech Stack](#tech-stack) @@ -19,9 +21,52 @@ Llama UI supports two server operation modes: - [Data Flows](#data-flows) - [Architectural Patterns](#architectural-patterns) - [Testing](#testing) +- [Publishing to npm](#publishing-to-npm) --- +## Run it with npx + +Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app on localhost and connects it to a llama-server you run yourself; it brings no server and no models of its own. It needs Node.js 20 or later. + +```bash +# start llama-server, then serve Llama Chat on http://127.0.0.1:3000 and open the browser +npx @ggml-org/llama-chat +``` + +```bash +# the same, by its short name, against a llama-server elsewhere on the network +npx llama-chat --server http://192.168.1.20:8080 +``` + +```bash +# reachable from other devices, without opening a browser +npx llama-chat --host 0.0.0.0 --port 3000 --no-open +``` + +| Flag | Default | What it does | +| --------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------- | +| `--server ` | `http://127.0.0.1:8080` | The llama-server to connect to. A path is kept, for a server started with `--api-prefix` | +| `--port ` | `3000` | Fixed, so the origin, and the chats the browser stores under it, stay the same between runs | +| `--host ` | `127.0.0.1` | `0.0.0.0` to open it to the LAN | +| `--open` / `--no-open` | open | Opens the default browser once it listens | +| `-v`, `--version`, `-h`, `--help` | | As usual | + +The bare command is the documented form; `start` is accepted as an alias for it. + +How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server, without an `Origin` header, so the server needs no `--cors-origins`. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. + +An `http://` page can call a plain `http://` llama-server on the LAN, which an `https://` page cannot (browsers block it as mixed content). That is why the npx command matters next to a hosted copy. + +## Self-hosting + +The build in `dist` is static and uses relative paths and a hash router, so any static host can serve it at any path. Two ways to host it: + +- **Run the CLI behind your reverse proxy**: `npx @ggml-org/llama-chat --host 0.0.0.0 --no-open --server `, then proxy your domain to its port. Long chat responses stream as server-sent events, so turn off response buffering in the proxy. +- **Serve `dist` yourself**: `npm run build`, or take `dist` from the npm package (`node_modules/@ggml-org/llama-chat/dist`). Serve it from the same origin as the llama-server API: forward `/v1`, `/props`, `/models`, `/slots`, `/tools` and `/cors-proxy` to llama-server, and everything else from `dist`. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Remote Assets Under Cross-Origin Isolation](#9-remote-assets-under-cross-origin-isolation)). + +Until the app can add a server by URL, it reaches only the llama-server on its own origin: through the CLI, a reverse proxy, or llama-server itself (`--path dist`). + ## Features ### Chat Interface @@ -768,3 +813,27 @@ ui/apps/chat/ - [llama.cpp Server README](../../../tools/server/README.md) - Full server documentation - [Multimodal Documentation](../../../docs/multimodal.md) - Image and audio support - [Function Calling](../../../docs/function-calling.md) - Tool use capabilities + +--- + +## Publishing to npm + +Two packages go out with the same version: `@ggml-org/llama-chat`, which carries the app and the CLI, and `llama-chat` (in `alias/`), which depends on it and only forwards to its CLI so that `npx llama-chat` works. A unit test (`tests/unit/cli.test.ts`) fails when their versions drift apart. + +```bash +cd ui/apps/chat +npm version --no-git-tag-version +# then set the same version, and the same @ggml-org/llama-chat dependency, in alias/package.json +npm pack --dry-run +npm publish --otp +cd alias && npm publish --otp +``` + +`npm pack` and `npm publish` build `dist` first (`prepack`), so the package always carries a build of the sources it was packed from. Source maps stay out of the package. + +To try a package before publishing it: + +```bash +npm pack +npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:8080 +``` diff --git a/ui/apps/chat/alias/LICENSE b/ui/apps/chat/alias/LICENSE new file mode 100644 index 000000000000..50c59c51c158 --- /dev/null +++ b/ui/apps/chat/alias/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) ggml + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ui/apps/chat/alias/README.md b/ui/apps/chat/alias/README.md new file mode 100644 index 000000000000..a83be04d61d4 --- /dev/null +++ b/ui/apps/chat/alias/README.md @@ -0,0 +1,9 @@ +# llama-chat + +The short name for [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat): `npx llama-chat` runs Llama Chat, the chat app for llama.cpp, on localhost and connects it to a llama-server. + +```bash +npx llama-chat --server http://127.0.0.1:8080 +``` + +This package only forwards to `@ggml-org/llama-chat`, at the same version. See its [README](https://github.com/ggml-org/llama.cpp/tree/master/ui/apps/chat#readme) for the options. diff --git a/ui/apps/chat/alias/cli.js b/ui/apps/chat/alias/cli.js new file mode 100755 index 000000000000..9c24124034c4 --- /dev/null +++ b/ui/apps/chat/alias/cli.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node +// `npx llama-chat` runs the command of @ggml-org/llama-chat, which carries the app +import '@ggml-org/llama-chat/cli'; diff --git a/ui/apps/chat/alias/package.json b/ui/apps/chat/alias/package.json new file mode 100644 index 000000000000..8dd541ed4793 --- /dev/null +++ b/ui/apps/chat/alias/package.json @@ -0,0 +1,33 @@ +{ + "name": "llama-chat", + "version": "0.1.0", + "description": "npx llama-chat: runs Llama Chat, the chat app for llama.cpp. Same as npx @ggml-org/llama-chat.", + "keywords": [ + "llama.cpp", + "llama-server", + "llm", + "chat", + "local-ai" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/ggml-org/llama.cpp.git", + "directory": "ui/apps/chat/alias" + }, + "homepage": "https://github.com/ggml-org/llama.cpp/tree/master/ui/apps/chat#readme", + "bugs": "https://github.com/ggml-org/llama.cpp/issues", + "type": "module", + "bin": { + "llama-chat": "./cli.js" + }, + "files": [ + "cli.js" + ], + "engines": { + "node": ">=20" + }, + "dependencies": { + "@ggml-org/llama-chat": "0.1.0" + } +} diff --git a/ui/apps/chat/bin/cli.js b/ui/apps/chat/bin/cli.js new file mode 100755 index 000000000000..1e41ac29d74d --- /dev/null +++ b/ui/apps/chat/bin/cli.js @@ -0,0 +1,5 @@ +#!/usr/bin/env node +// `npx @ggml-org/llama-chat` and `llama-chat`, see cli/server.js +import { main } from '../cli/server.js'; + +await main(process.argv.slice(2)); diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js new file mode 100644 index 000000000000..aab4ad13f30b --- /dev/null +++ b/ui/apps/chat/cli/server.js @@ -0,0 +1,386 @@ +/** + * `npx @ggml-org/llama-chat`: serves the prebuilt Llama Chat (`dist`) on localhost and forwards + * its API calls to a llama-server. Node built-ins only, so the published package has no + * dependencies: no llama-server, no models, no Vite or SvelteKit. + * + * The app calls the server on its own origin (`./v1/...`, `./props`, ...), as it does when + * llama-server serves it. So a request is answered from `dist` when a file there matches its + * path, and forwarded to the llama-server otherwise. + */ + +import { spawn } from 'node:child_process'; +import { createReadStream, readFileSync } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import http from 'node:http'; +import https from 'node:https'; +import { extname, join, normalize, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +export const PACKAGE_ROOT = fileURLToPath(new URL('..', import.meta.url)); +export const DIST_DIR = join(PACKAGE_ROOT, 'dist'); + +export const DEFAULTS = Object.freeze({ + host: '127.0.0.1', + // fixed, so the origin, and the chats the browser stores under it, stay the same between runs + port: 3000, + // where llama-server listens when started without --host or --port + server: 'http://127.0.0.1:8080' +}); + +// The same caching llama-server applies to the UI it embeds (tools/server/server-http.cpp): +// index.html and the PWA's update checks are revalidated, hashed assets are cached for good. +const NO_CACHE = new Set([ + '/index.html', + '/sw.js', + '/manifest.webmanifest', + '/_app/version.json', + '/build.json' +]); +const CACHE_IMMUTABLE = 'public, max-age=31536000, immutable'; +const CACHE_REVALIDATE = 'no-cache'; +/** @type {Record} */ +const MIME_TYPES = { + '.css': 'text/css; charset=utf-8', + '.gif': 'image/gif', + '.html': 'text/html; charset=utf-8', + '.ico': 'image/x-icon', + '.jpg': 'image/jpeg', + '.js': 'text/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.map': 'application/json; charset=utf-8', + '.mjs': 'text/javascript; charset=utf-8', + '.png': 'image/png', + '.svg': 'image/svg+xml', + '.ttf': 'font/ttf', + '.txt': 'text/plain; charset=utf-8', + '.wasm': 'application/wasm', + '.webmanifest': 'application/manifest+json; charset=utf-8', + '.webp': 'image/webp', + '.woff': 'font/woff', + '.woff2': 'font/woff2' +}; +// hop-by-hop headers, which a proxy does not pass on +const HOP_BY_HOP = [ + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'proxy-connection', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade' +]; + +export const USAGE = `Usage: llama-chat [options] + +Serves Llama Chat on localhost and connects it to a llama-server. + +Options: + --server llama-server to connect to (default: ${DEFAULTS.server}) + --port port to listen on (default: ${DEFAULTS.port}) + --host address to listen on, 0.0.0.0 for the LAN (default: ${DEFAULTS.host}) + --open open the browser once it listens (default) + --no-open do not open the browser + -v, --version print the version + -h, --help print this help + +Examples: + npx @ggml-org/llama-chat + npx @ggml-org/llama-chat --server http://192.168.1.20:8080 + npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open`; + +/** The version in this package's package.json. */ +export function packageVersion() { + return JSON.parse(readFileSync(join(PACKAGE_ROOT, 'package.json'), 'utf8')).version; +} + +/** + * Reads the command line. `start` is accepted as a verb and means the same as none. + * @param {string[]} argv the arguments after the command + * @returns {{ help: boolean, host: string, open: boolean, port: number, server: URL, version: boolean }} + */ +export function parseCliArgs(argv) { + const { positionals, values } = parseArgs({ + allowNegative: true, + allowPositionals: true, + args: argv, + options: { + help: { short: 'h', type: 'boolean' }, + host: { type: 'string' }, + open: { type: 'boolean' }, + port: { type: 'string' }, + server: { type: 'string' }, + version: { short: 'v', type: 'boolean' } + }, + strict: true + }); + const [verb, ...rest] = positionals; + + if ((verb !== undefined && verb !== 'start') || rest.length > 0) { + throw new Error(`unknown command: ${positionals.join(' ')}`); + } + + const port = Number(values.port ?? DEFAULTS.port); + + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error(`invalid --port: ${values.port}`); + } + + return { + help: values.help ?? false, + host: values.host ?? DEFAULTS.host, + open: values.open ?? true, + port, + server: parseServerUrl(values.server ?? DEFAULTS.server), + version: values.version ?? false + }; +} + +/** + * @param {string} value a llama-server address, with or without `http://` + * @returns {URL} + */ +export function parseServerUrl(value) { + const url = new URL(/^[a-z][a-z\d+.-]*:\/\//i.test(value) ? value : `http://${value}`); + + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error(`invalid --server: ${value} (use an http:// or https:// address)`); + } + + return url; +} + +/** + * Maps a request path to a file under `root`, or null when it falls outside it. + * @param {string} root + * @param {string} pathname + */ +function resolveStatic(root, pathname) { + let decoded; + + try { + decoded = decodeURIComponent(pathname); + } catch { + return null; + } + + if (decoded.includes('\0')) return null; + + const file = normalize(join(root, decoded === '/' ? '/index.html' : decoded)); + + return file === root || file.startsWith(root.endsWith(sep) ? root : root + sep) ? file : null; +} + +/** + * Sends a file from `dist`, with llama-server's caching and cross-origin isolation headers. + * @param {http.IncomingMessage} req + * @param {http.ServerResponse} res + * @param {string} file + * @param {string} pathname + * @param {import('node:fs').Stats} stats + */ +function sendFile(req, res, file, pathname, stats) { + const name = pathname === '/' ? '/index.html' : pathname; + const etag = `W/"${stats.size.toString(16)}-${Math.floor(stats.mtimeMs).toString(16)}"`; + + res.setHeader('ETag', etag); + res.setHeader('Cache-Control', NO_CACHE.has(name) ? CACHE_REVALIDATE : CACHE_IMMUTABLE); + + if (name === '/index.html') { + res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp'); + res.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + } + + if (req.headers['if-none-match'] === etag) { + res.writeHead(304).end(); + + return; + } + + res.setHeader( + 'Content-Type', + MIME_TYPES[extname(file).toLowerCase()] ?? 'application/octet-stream' + ); + res.setHeader('Content-Length', stats.size); + res.writeHead(200); + + if (req.method === 'HEAD') { + res.end(); + + return; + } + + createReadStream(file).pipe(res); +} + +/** + * Forwards a request to the llama-server. llama-server takes cross-origin browser requests only + * from its --cors-origins, so, like the Vite dev proxy, this calls it as its own page: with the + * Host of the server and no Origin. + * @param {http.IncomingMessage} req + * @param {http.ServerResponse} res + * @param {URL} server + */ +function proxy(req, res, server) { + const prefix = server.pathname.replace(/\/+$/, ''); + const target = new URL(prefix + (req.url ?? '/'), server.origin); + /** @type {http.OutgoingHttpHeaders} */ + const headers = { ...req.headers, host: target.host }; + + delete headers.origin; + + for (const name of HOP_BY_HOP) delete headers[name]; + + const upstream = (target.protocol === 'https:' ? https : http).request( + target, + { headers, method: req.method }, + (response) => { + const responseHeaders = { ...response.headers }; + + for (const name of HOP_BY_HOP) delete responseHeaders[name]; + + res.writeHead(response.statusCode ?? 502, responseHeaders); + response.pipe(res); + } + ); + + upstream.on('error', (error) => { + if (res.headersSent) { + res.destroy(error); + + return; + } + + res.writeHead(502, { 'Content-Type': 'application/json; charset=utf-8' }); + res.end( + JSON.stringify({ + error: { + code: 502, + message: `llama-chat could not reach the llama-server at ${server.href}: ${error.message}`, + type: 'unavailable_error' + } + }) + ); + }); + + // a stream the browser stops reading (Stop, a closed tab) stops the generation on the server + res.on('close', () => { + if (!res.writableFinished) upstream.destroy(); + }); + + req.pipe(upstream); +} + +/** + * The HTTP server: files from `dist`, everything else from the llama-server. + * @param {{ distDir?: string, server: URL }} options + */ +export function createChatServer({ distDir = DIST_DIR, server }) { + const root = normalize(distDir); + + return http.createServer(async (req, res) => { + const { pathname } = new URL(req.url ?? '/', 'http://localhost'); + + if (req.method === 'GET' || req.method === 'HEAD') { + const file = resolveStatic(root, pathname); + const stats = file ? await stat(file).catch(() => null) : null; + + if (file && stats?.isFile()) { + sendFile(req, res, file, pathname, stats); + + return; + } + } + + proxy(req, res, server); + }); +} + +/** + * Opens a URL in the default browser, quietly doing nothing where that fails. + * @param {string} url + */ +function openBrowser(url) { + const [command, args] = + process.platform === 'darwin' + ? ['open', [url]] + : process.platform === 'win32' + ? ['cmd', ['/c', 'start', '""', url]] + : ['xdg-open', [url]]; + + try { + spawn(command, args, { detached: true, stdio: 'ignore' }) + .on('error', () => {}) + .unref(); + } catch { + // no browser to open, e.g. on a headless machine + } +} + +/** + * Runs the command. + * @param {string[]} argv the arguments after the command + * @returns {Promise} the server, once it listens + */ +export async function main(argv) { + let options; + + try { + options = parseCliArgs(argv); + } catch (error) { + console.error(`llama-chat: ${error instanceof Error ? error.message : error}\n\n${USAGE}`); + process.exitCode = 2; + + return; + } + + if (options.help) { + console.log(USAGE); + + return; + } + + if (options.version) { + console.log(packageVersion()); + + return; + } + + if (!(await stat(join(DIST_DIR, 'index.html')).catch(() => null))) { + console.error(`llama-chat: no build found in ${DIST_DIR}, run \`npm run build\` first`); + process.exitCode = 1; + + return; + } + + const server = createChatServer({ server: options.server }); + + return new Promise((resolve) => { + server.on('error', (error) => { + const busy = /** @type {NodeJS.ErrnoException} */ (error).code === 'EADDRINUSE'; + + console.error( + busy + ? `llama-chat: port ${options.port} is in use, pick another with --port` + : `llama-chat: ${error.message}` + ); + process.exitCode = 1; + resolve(undefined); + }); + + server.listen(options.port, options.host, () => { + const address = /** @type {import('node:net').AddressInfo} */ (server.address()); + const shown = ['0.0.0.0', '::'].includes(options.host) ? 'localhost' : options.host; + const url = `http://${shown.includes(':') ? `[${shown}]` : shown}:${address.port}/`; + + console.log(`Llama Chat ${packageVersion()} on ${url}`); + console.log(`Connected to the llama-server at ${options.server.href}`); + + if (options.open) openBrowser(url); + + resolve(server); + }); + }); +} diff --git a/ui/apps/chat/package-lock.json b/ui/apps/chat/package-lock.json index 44b5ffa32493..0e732ff776c8 100644 --- a/ui/apps/chat/package-lock.json +++ b/ui/apps/chat/package-lock.json @@ -1,12 +1,16 @@ { "name": "@ggml-org/llama-chat", - "version": "1.0.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ggml-org/llama-chat", - "version": "1.0.0", + "version": "0.1.0", + "license": "MIT", + "bin": { + "llama-chat": "bin/cli.js" + }, "devDependencies": { "@chromatic-com/storybook": "5.5.1", "@codemirror/commands": "6.11.1", @@ -91,6 +95,9 @@ "vite-plugin-devtools-json": "1.1.2", "vitest": "4.1.12", "vitest-browser-svelte": "2.1.1" + }, + "engines": { + "node": ">=20" } }, "node_modules/@adobe/css-tools": { diff --git a/ui/apps/chat/package.json b/ui/apps/chat/package.json index 1120f6f872e9..9b568c7c0b83 100644 --- a/ui/apps/chat/package.json +++ b/ui/apps/chat/package.json @@ -1,8 +1,44 @@ { "name": "@ggml-org/llama-chat", - "private": true, - "version": "1.0.0", + "version": "0.1.0", + "description": "Llama Chat, the chat app for llama.cpp. Run it with npx @ggml-org/llama-chat against any llama-server.", + "keywords": [ + "llama.cpp", + "llama-server", + "llm", + "chat", + "local-ai" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/ggml-org/llama.cpp.git", + "directory": "ui/apps/chat" + }, + "homepage": "https://github.com/ggml-org/llama.cpp/tree/master/ui/apps/chat#readme", + "bugs": "https://github.com/ggml-org/llama.cpp/issues", "type": "module", + "bin": { + "llama-chat": "./bin/cli.js" + }, + "exports": { + "./cli": "./bin/cli.js", + "./server": "./cli/server.js", + "./dist/*": "./dist/*", + "./package.json": "./package.json" + }, + "files": [ + "bin", + "cli", + "dist", + "!dist/**/*.map" + ], + "engines": { + "node": ">=20" + }, + "publishConfig": { + "access": "public" + }, "scripts": { "build": "npm run build-pwa-assets && vite build", "build-pwa-assets": "pwa-assets-generator --root . --config pwa-assets.config.ts && pwa-assets-generator --root . --config pwa-assets-dark.config.ts && node scripts/make-icons-circular.js", @@ -32,7 +68,9 @@ "test:ui": "vitest --project=ui", "storybook": "svelte-kit sync && storybook dev -p 6006", "build-storybook": "svelte-kit sync && storybook build", - "cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico" + "cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico", + "prepack": "npm run build", + "start": "node bin/cli.js" }, "devDependencies": { "@chromatic-com/storybook": "5.5.1", diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts new file mode 100644 index 000000000000..d04f0ff5884e --- /dev/null +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -0,0 +1,192 @@ +import { createChatServer, DEFAULTS, parseCliArgs, parseServerUrl } from '../../cli/server.js'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +function listen(server: http.Server): Promise { + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + resolve(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); + }); + }); +} + +function close(server: http.Server): Promise { + return new Promise((resolve) => server.close(() => resolve())); +} + +describe('llama-chat CLI arguments', () => { + it('starts on the defaults with no arguments', () => { + const options = parseCliArgs([]); + + expect(options.host).toBe(DEFAULTS.host); + expect(options.port).toBe(DEFAULTS.port); + expect(options.open).toBe(true); + expect(options.server.href).toBe(`${DEFAULTS.server}/`); + }); + + it('reads the documented flags', () => { + const options = parseCliArgs([ + '--server', + 'http://192.168.1.20:8080', + '--host', + '0.0.0.0', + '--port', + '3001', + '--no-open' + ]); + + expect(options).toMatchObject({ host: '0.0.0.0', open: false, port: 3001 }); + expect(options.server.href).toBe('http://192.168.1.20:8080/'); + }); + + it('accepts start as a verb that changes nothing', () => { + expect(parseCliArgs(['start', '--no-open']).open).toBe(false); + }); + + it('refuses unknown verbs, unknown flags and bad ports', () => { + expect(() => parseCliArgs(['serve'])).toThrow(/unknown command/); + expect(() => parseCliArgs(['--nope'])).toThrow(); + expect(() => parseCliArgs(['--port', 'abc'])).toThrow(/invalid --port/); + expect(() => parseCliArgs(['--port', '70000'])).toThrow(/invalid --port/); + }); + + it('takes a server address without a scheme as http', () => { + expect(parseServerUrl('localhost:8080').href).toBe('http://localhost:8080/'); + expect(() => parseServerUrl('ftp://localhost')).toThrow(/invalid --server/); + }); +}); + +describe('llama-chat packages', () => { + it('keeps the llama-chat alias on the version of @ggml-org/llama-chat', async () => { + const read = async (path: string) => + JSON.parse(await readFile(new URL(path, import.meta.url), 'utf8')); + const app = await read('../../package.json'); + const alias = await read('../../alias/package.json'); + + expect(app.private).toBeUndefined(); + expect(app.bin['llama-chat']).toBe('./bin/cli.js'); + expect(alias.version).toBe(app.version); + expect(alias.dependencies[app.name]).toBe(app.version); + }); +}); + +describe('llama-chat server', () => { + let dist: string; + let upstream: http.Server; + let upstreamUrl: string; + let chat: http.Server; + let chatUrl: string; + + const seen: { headers: http.IncomingHttpHeaders; method?: string; url?: string; body: string }[] = + []; + + beforeAll(async () => { + dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-')); + await mkdir(join(dist, '_app', 'immutable'), { recursive: true }); + await writeFile(join(dist, 'index.html'), 'Llama Chat'); + await writeFile(join(dist, 'sw.js'), 'self;'); + await writeFile(join(dist, '_app', 'immutable', 'app.abc123.js'), 'export {};'); + + upstream = http.createServer((req, res) => { + let body = ''; + + req.on('data', (chunk) => (body += chunk)); + req.on('end', () => { + seen.push({ body, headers: req.headers, method: req.method, url: req.url }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ path: req.url })); + }); + }); + upstreamUrl = await listen(upstream); + + chat = createChatServer({ distDir: dist, server: new URL(`${upstreamUrl}/prefix/`) }); + chatUrl = await listen(chat); + }); + + afterAll(async () => { + await close(chat); + await close(upstream); + await rm(dist, { force: true, recursive: true }); + }); + + it('serves index.html at the root, revalidated and cross-origin isolated', async () => { + const res = await fetch(`${chatUrl}/`); + + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toMatch(/^text\/html/); + expect(res.headers.get('cache-control')).toBe('no-cache'); + expect(res.headers.get('cross-origin-embedder-policy')).toBe('require-corp'); + expect(res.headers.get('cross-origin-opener-policy')).toBe('same-origin'); + expect(await res.text()).toContain('Llama Chat'); + + const again = await fetch(`${chatUrl}/`, { + headers: { 'If-None-Match': res.headers.get('etag') ?? '' } + }); + + expect(again.status).toBe(304); + }); + + it('caches hashed assets for good and revalidates the service worker', async () => { + const asset = await fetch(`${chatUrl}/_app/immutable/app.abc123.js`); + + expect(asset.headers.get('cache-control')).toContain('immutable'); + expect(asset.headers.get('content-type')).toMatch(/^text\/javascript/); + + const sw = await fetch(`${chatUrl}/sw.js`); + + expect(sw.headers.get('cache-control')).toBe('no-cache'); + }); + + it('forwards API calls to the llama-server as its own page', async () => { + const res = await fetch(`${chatUrl}/v1/chat/completions?x=1`, { + body: '{"messages":[]}', + headers: { 'Content-Type': 'application/json', Origin: 'http://localhost:3000' }, + method: 'POST' + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ path: '/prefix/v1/chat/completions?x=1' }); + + const request = seen.at(-1)!; + + expect(request.method).toBe('POST'); + expect(request.body).toBe('{"messages":[]}'); + expect(request.headers.origin).toBeUndefined(); + expect(request.headers.host).toBe(new URL(upstreamUrl).host); + }); + + it('does not serve files from outside dist', async () => { + // fetch() would resolve the dots itself, so the raw path goes out through http.get + const path = '/..%2f..%2f..%2fetc%2fpasswd'; + const body = await new Promise((resolve, reject) => { + http + .get(`${chatUrl}${path}`, (res) => { + let data = ''; + + res.on('data', (chunk) => (data += chunk)); + res.on('end', () => resolve(data)); + }) + .on('error', reject); + }); + + expect(JSON.parse(body)).toEqual({ path: `/prefix${path}` }); + }); + + it('answers 502 when the llama-server is down', async () => { + const down = createChatServer({ distDir: dist, server: new URL('http://127.0.0.1:9') }); + const url = await listen(down); + + try { + const res = await fetch(`${url}/props`); + + expect(res.status).toBe(502); + expect((await res.json()).error.message).toMatch(/could not reach the llama-server/); + } finally { + await close(down); + } + }); +}); From 8f43a91a83c02d4340b80e198b63c0228fb7d3ae Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Mon, 16 Nov 2026 17:20:33 +0100 Subject: [PATCH 03/10] ui: llama-chat revalidates service-worker.js and asks for a build before packing SvelteKit 3 adds service-worker.js next to sw.js, and llama-server revalidates it, so the CLI does too. The folder's .npmrc sets ignore-scripts, so prepack never ran: it goes, and the publishing steps build dist explicitly and check the packed file list. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 6 ++++-- ui/apps/chat/cli/server.js | 1 + ui/apps/chat/package.json | 1 - ui/apps/chat/tests/unit/cli.test.ts | 5 +++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 4cd8c9881abe..e8101f43a61a 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -54,7 +54,7 @@ npx llama-chat --host 0.0.0.0 --port 3000 --no-open The bare command is the documented form; `start` is accepted as an alias for it. -How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server, without an `Origin` header, so the server needs no `--cors-origins`. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. +How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server, without an `Origin` header, so the server needs no `--cors-origins`. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `service-worker.js`, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. An `http://` page can call a plain `http://` llama-server on the LAN, which an `https://` page cannot (browsers block it as mixed content). That is why the npx command matters next to a hosted copy. @@ -824,16 +824,18 @@ Two packages go out with the same version: `@ggml-org/llama-chat`, which carries cd ui/apps/chat npm version --no-git-tag-version # then set the same version, and the same @ggml-org/llama-chat dependency, in alias/package.json +npm run build npm pack --dry-run npm publish --otp cd alias && npm publish --otp ``` -`npm pack` and `npm publish` build `dist` first (`prepack`), so the package always carries a build of the sources it was packed from. Source maps stay out of the package. +Build right before packing: the package carries whatever is in `dist`. This folder's `.npmrc` sets `ignore-scripts`, so npm runs no lifecycle script such as `prepack` that could build it for you. Check that `npm pack --dry-run` lists `dist/index.html` and about 77 files. Source maps stay out of the package. To try a package before publishing it: ```bash +npm run build npm pack npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:8080 ``` diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js index aab4ad13f30b..23d12a3d4ef7 100644 --- a/ui/apps/chat/cli/server.js +++ b/ui/apps/chat/cli/server.js @@ -32,6 +32,7 @@ export const DEFAULTS = Object.freeze({ // index.html and the PWA's update checks are revalidated, hashed assets are cached for good. const NO_CACHE = new Set([ '/index.html', + '/service-worker.js', '/sw.js', '/manifest.webmanifest', '/_app/version.json', diff --git a/ui/apps/chat/package.json b/ui/apps/chat/package.json index 9b568c7c0b83..ee0e9aafa253 100644 --- a/ui/apps/chat/package.json +++ b/ui/apps/chat/package.json @@ -69,7 +69,6 @@ "storybook": "svelte-kit sync && storybook dev -p 6006", "build-storybook": "svelte-kit sync && storybook build", "cleanup": "rm -rf .svelte-kit build node_modules test-results dist dev-dist debug-storybook.log static/pwa-*.png static/maskable-icon-*.png static/apple-touch-icon-*.png static/apple-splash-*.png static/favicon*.ico", - "prepack": "npm run build", "start": "node bin/cli.js" }, "devDependencies": { diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts index d04f0ff5884e..300411a056cc 100644 --- a/ui/apps/chat/tests/unit/cli.test.ts +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -89,6 +89,7 @@ describe('llama-chat server', () => { await mkdir(join(dist, '_app', 'immutable'), { recursive: true }); await writeFile(join(dist, 'index.html'), 'Llama Chat'); await writeFile(join(dist, 'sw.js'), 'self;'); + await writeFile(join(dist, 'service-worker.js'), 'self;'); await writeFile(join(dist, '_app', 'immutable', 'app.abc123.js'), 'export {};'); upstream = http.createServer((req, res) => { @@ -139,6 +140,10 @@ describe('llama-chat server', () => { const sw = await fetch(`${chatUrl}/sw.js`); expect(sw.headers.get('cache-control')).toBe('no-cache'); + + const worker = await fetch(`${chatUrl}/service-worker.js`); + + expect(worker.headers.get('cache-control')).toBe('no-cache'); }); it('forwards API calls to the llama-server as its own page', async () => { From f4326fb332af2c1233c4d44860dcad3081acd509 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 08:36:58 +0100 Subject: [PATCH 04/10] ui: llama-chat keeps llama-server's DNS rebinding and cross-origin guards The CLI called llama-server with its own Host and dropped every Origin, so a rebound host name or another page got past the checks llama-server makes when --cors-origins is not *. It now drops only its own page's Origin, passes other Origins and Sec-Fetch-Site through, and, when it listens on loopback, answers only to localhost and loopback addresses. `start` on Windows gets its empty title verbatim. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 2 +- ui/apps/chat/cli/server.js | 54 +++++++++++++++++++++++++++-- ui/apps/chat/tests/unit/cli.test.ts | 33 +++++++++++++++++- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index e8101f43a61a..fd4a6965cf51 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -54,7 +54,7 @@ npx llama-chat --host 0.0.0.0 --port 3000 --no-open The bare command is the documented form; `start` is accepted as an alias for it. -How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server, without an `Origin` header, so the server needs no `--cors-origins`. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `service-worker.js`, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. +How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server as if from the server's own page (its `Host`, and without the CLI page's `Origin`), so the server needs no `--cors-origins`. Requests from other pages keep their `Origin` and `Sec-Fetch-Site`, so llama-server refuses them as it would directly, and a CLI listening on loopback answers only to `localhost` and loopback addresses, which stops DNS rebinding. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `service-worker.js`, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. An `http://` page can call a plain `http://` llama-server on the LAN, which an `https://` page cannot (browsers block it as mixed content). That is why the npx command matters next to a hosted copy. diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js index 23d12a3d4ef7..35b6d9afd8b1 100644 --- a/ui/apps/chat/cli/server.js +++ b/ui/apps/chat/cli/server.js @@ -216,10 +216,39 @@ function sendFile(req, res, file, pathname, stats) { createReadStream(file).pipe(res); } +/** + * The host name in a Host header, lower case and without the port or IPv6 brackets. + * @param {string} host + */ +function hostName(host) { + const name = host.startsWith('[') ? host.slice(1, host.indexOf(']')) : host.split(':')[0]; + + return name.toLowerCase(); +} + +/** @param {string} address */ +function isLoopback(address) { + return /^(127\.|::1$|::ffff:127\.)/.test(address); +} + +/** + * DNS rebinding guard, as llama-server has: a page whose own name resolves to 127.0.0.1 reaches + * a loopback listener with that name in Host, so there Host must name loopback itself. + * @param {http.IncomingMessage} req + */ +function hostIsAllowed(req) { + if (!isLoopback(req.socket.localAddress ?? '')) return true; + + const name = hostName(req.headers.host ?? ''); + + return name === 'localhost' || name === '0.0.0.0' || name === '::' || isLoopback(name); +} + /** * Forwards a request to the llama-server. llama-server takes cross-origin browser requests only * from its --cors-origins, so, like the Vite dev proxy, this calls it as its own page: with the - * Host of the server and no Origin. + * Host of the server, and without the Origin of this page. Another page's Origin and + * Sec-Fetch-Site pass through, so llama-server still refuses what it would refuse directly. * @param {http.IncomingMessage} req * @param {http.ServerResponse} res * @param {URL} server @@ -229,8 +258,11 @@ function proxy(req, res, server) { const target = new URL(prefix + (req.url ?? '/'), server.origin); /** @type {http.OutgoingHttpHeaders} */ const headers = { ...req.headers, host: target.host }; + const origin = req.headers.origin ?? ''; - delete headers.origin; + if (origin.slice(origin.indexOf('://') + 3).toLowerCase() === req.headers.host?.toLowerCase()) { + delete headers.origin; + } for (const name of HOP_BY_HOP) delete headers[name]; @@ -284,6 +316,21 @@ export function createChatServer({ distDir = DIST_DIR, server }) { return http.createServer(async (req, res) => { const { pathname } = new URL(req.url ?? '/', 'http://localhost'); + if (!hostIsAllowed(req)) { + res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' }); + res.end( + JSON.stringify({ + error: { + code: 403, + message: `llama-chat refused the host name ${req.headers.host}, open it on localhost or 127.0.0.1`, + type: 'permission_error' + } + }) + ); + + return; + } + if (req.method === 'GET' || req.method === 'HEAD') { const file = resolveStatic(root, pathname); const stats = file ? await stat(file).catch(() => null) : null; @@ -312,7 +359,8 @@ function openBrowser(url) { : ['xdg-open', [url]]; try { - spawn(command, args, { detached: true, stdio: 'ignore' }) + // verbatim, or Node quotes the empty title of `start` into one cmd.exe misreads + spawn(command, args, { detached: true, stdio: 'ignore', windowsVerbatimArguments: true }) .on('error', () => {}) .unref(); } catch { diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts index 300411a056cc..fd9e3b73d0c6 100644 --- a/ui/apps/chat/tests/unit/cli.test.ts +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -149,7 +149,7 @@ describe('llama-chat server', () => { it('forwards API calls to the llama-server as its own page', async () => { const res = await fetch(`${chatUrl}/v1/chat/completions?x=1`, { body: '{"messages":[]}', - headers: { 'Content-Type': 'application/json', Origin: 'http://localhost:3000' }, + headers: { 'Content-Type': 'application/json', Origin: chatUrl }, method: 'POST' }); @@ -164,6 +164,37 @@ describe('llama-chat server', () => { expect(request.headers.host).toBe(new URL(upstreamUrl).host); }); + it("passes another page's Origin through, so llama-server can refuse it", async () => { + await fetch(`${chatUrl}/props`, { + headers: { Origin: 'http://evil.example', 'Sec-Fetch-Site': 'cross-site' } + }); + + const request = seen.at(-1)!; + + expect(request.headers.origin).toBe('http://evil.example'); + expect(request.headers['sec-fetch-site']).toBe('cross-site'); + }); + + it('refuses other host names on loopback, against DNS rebinding', async () => { + // fetch() sets Host itself, so the request goes out through http.request + const status = (host: string) => + new Promise((resolve, reject) => { + const { port } = new URL(chatUrl); + + http + .request({ headers: { host }, host: '127.0.0.1', path: '/props', port }, (res) => { + res.resume(); + resolve(res.statusCode ?? 0); + }) + .on('error', reject) + .end(); + }); + + expect(await status('rebind.example:3000')).toBe(403); + expect(await status('localhost:3000')).toBe(200); + expect(await status('[::1]:3000')).toBe(200); + }); + it('does not serve files from outside dist', async () => { // fetch() would resolve the dots itself, so the raw path goes out through http.get const path = '/..%2f..%2f..%2fetc%2fpasswd'; From 258c7f604135b72ec45fbe5ea259b4da1d9546fa Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 08:47:40 +0100 Subject: [PATCH 05/10] ui: llama-chat build writes the app to a folder for any static server `npx @ggml-org/llama-chat build [--out ] [--force]` copies the prebuilt app, without source maps, to ./llama-chat or the given folder, so a plain file server can host it next to the CLI serving it. It refuses a folder that is not empty unless forced, and the serving flags and build flags refuse each other's command. The README shows serving the folder with http-server forwarding the API to llama-server. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 20 +++++-- ui/apps/chat/cli/server.js | 90 ++++++++++++++++++++++++++--- ui/apps/chat/tests/unit/cli.test.ts | 76 +++++++++++++++++++++++- 3 files changed, 171 insertions(+), 15 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index fd4a6965cf51..5d120b520e61 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -27,7 +27,7 @@ Llama UI supports two server operation modes: ## Run it with npx -Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app on localhost and connects it to a llama-server you run yourself; it brings no server and no models of its own. It needs Node.js 20 or later. +Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app on localhost and connects it to a llama-server you run yourself, or writes it to a folder for any static file server (see [Self-hosting](#self-hosting)); it brings no server and no models of its own. It needs Node.js 20 or later. ```bash # start llama-server, then serve Llama Chat on http://127.0.0.1:3000 and open the browser @@ -44,6 +44,11 @@ npx llama-chat --server http://192.168.1.20:8080 npx llama-chat --host 0.0.0.0 --port 3000 --no-open ``` +```bash +# write the static app to ./llama-chat instead of serving it +npx llama-chat build +``` + | Flag | Default | What it does | | --------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------- | | `--server ` | `http://127.0.0.1:8080` | The llama-server to connect to. A path is kept, for a server started with `--api-prefix` | @@ -52,7 +57,7 @@ npx llama-chat --host 0.0.0.0 --port 3000 --no-open | `--open` / `--no-open` | open | Opens the default browser once it listens | | `-v`, `--version`, `-h`, `--help` | | As usual | -The bare command is the documented form; `start` is accepted as an alias for it. +The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`) and `--force`, which lets it write into a folder that is not empty. How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server as if from the server's own page (its `Host`, and without the CLI page's `Origin`), so the server needs no `--cors-origins`. Requests from other pages keep their `Origin` and `Sec-Fetch-Site`, so llama-server refuses them as it would directly, and a CLI listening on loopback answers only to `localhost` and loopback addresses, which stops DNS rebinding. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `service-worker.js`, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. @@ -60,10 +65,17 @@ An `http://` page can call a plain `http://` llama-server on the LAN, which an ` ## Self-hosting -The build in `dist` is static and uses relative paths and a hash router, so any static host can serve it at any path. Two ways to host it: +The app is static and uses relative paths and a hash router, so any static host can serve it at any path. Two ways to host it: - **Run the CLI behind your reverse proxy**: `npx @ggml-org/llama-chat --host 0.0.0.0 --no-open --server `, then proxy your domain to its port. Long chat responses stream as server-sent events, so turn off response buffering in the proxy. -- **Serve `dist` yourself**: `npm run build`, or take `dist` from the npm package (`node_modules/@ggml-org/llama-chat/dist`). Serve it from the same origin as the llama-server API: forward `/v1`, `/props`, `/models`, `/slots`, `/tools` and `/cors-proxy` to llama-server, and everything else from `dist`. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Remote Assets Under Cross-Origin Isolation](#9-remote-assets-under-cross-origin-isolation)). +- **Serve the static files yourself**: `npx @ggml-org/llama-chat build --out ` writes them (as does `npm run build` here, to `dist`). Serve them from the same origin as the llama-server API: a path with a file answers from the folder, and every other path goes to llama-server. For example, with [http-server](https://www.npmjs.com/package/http-server), which forwards what it has no file for: + + ```bash + npx llama-chat build + npx http-server llama-chat -p 3000 -P http://127.0.0.1:8080 + ``` + + A plain file server sends the browser's `Origin` on to llama-server, so start llama-server with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Remote Assets Under Cross-Origin Isolation](#9-remote-assets-under-cross-origin-isolation)). Until the app can add a server by URL, it reaches only the llama-server on its own origin: through the CLI, a reverse proxy, or llama-server itself (`--path dist`). diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js index 35b6d9afd8b1..7dd90345e7f5 100644 --- a/ui/apps/chat/cli/server.js +++ b/ui/apps/chat/cli/server.js @@ -1,7 +1,8 @@ /** * `npx @ggml-org/llama-chat`: serves the prebuilt Llama Chat (`dist`) on localhost and forwards - * its API calls to a llama-server. Node built-ins only, so the published package has no - * dependencies: no llama-server, no models, no Vite or SvelteKit. + * its API calls to a llama-server. `npx @ggml-org/llama-chat build` writes the same files to a + * folder instead, for any static file server. Node built-ins only, so the published package has + * no dependencies: no llama-server, no models, no Vite or SvelteKit. * * The app calls the server on its own origin (`./v1/...`, `./props`, ...), as it does when * llama-server serves it. So a request is answered from `dist` when a file there matches its @@ -10,10 +11,10 @@ import { spawn } from 'node:child_process'; import { createReadStream, readFileSync } from 'node:fs'; -import { stat } from 'node:fs/promises'; +import { cp, readdir, stat } from 'node:fs/promises'; import http from 'node:http'; import https from 'node:https'; -import { extname, join, normalize, sep } from 'node:path'; +import { extname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { parseArgs } from 'node:util'; @@ -22,6 +23,8 @@ export const DIST_DIR = join(PACKAGE_ROOT, 'dist'); export const DEFAULTS = Object.freeze({ host: '127.0.0.1', + // the folder `build` writes to, in the current directory + out: 'llama-chat', // fixed, so the origin, and the chats the browser stores under it, stay the same between runs port: 3000, // where llama-server listens when started without --host or --port @@ -73,10 +76,14 @@ const HOP_BY_HOP = [ 'transfer-encoding', 'upgrade' ]; +const SERVE_OPTIONS = ['host', 'open', 'port', 'server']; +const BUILD_OPTIONS = ['force', 'out']; export const USAGE = `Usage: llama-chat [options] + llama-chat build [--out ] [--force] -Serves Llama Chat on localhost and connects it to a llama-server. +Serves Llama Chat on localhost and connects it to a llama-server, or, with +build, writes it to a folder as static files for any HTTP server. Options: --server llama-server to connect to (default: ${DEFAULTS.server}) @@ -87,10 +94,15 @@ Options: -v, --version print the version -h, --help print this help +Build options: + --out folder to write to (default: ./${DEFAULTS.out}) + --force write into a folder that is not empty + Examples: npx @ggml-org/llama-chat npx @ggml-org/llama-chat --server http://192.168.1.20:8080 - npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open`; + npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open + npx @ggml-org/llama-chat build --out ./public`; /** The version in this package's package.json. */ export function packageVersion() { @@ -98,9 +110,10 @@ export function packageVersion() { } /** - * Reads the command line. `start` is accepted as a verb and means the same as none. + * Reads the command line. No verb serves the app, `start` means the same, and `build` writes it + * to a folder. * @param {string[]} argv the arguments after the command - * @returns {{ help: boolean, host: string, open: boolean, port: number, server: URL, version: boolean }} + * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, server: URL, version: boolean }} */ export function parseCliArgs(argv) { const { positionals, values } = parseArgs({ @@ -108,9 +121,11 @@ export function parseCliArgs(argv) { allowPositionals: true, args: argv, options: { + force: { type: 'boolean' }, help: { short: 'h', type: 'boolean' }, host: { type: 'string' }, open: { type: 'boolean' }, + out: { type: 'string' }, port: { type: 'string' }, server: { type: 'string' }, version: { short: 'v', type: 'boolean' } @@ -119,10 +134,21 @@ export function parseCliArgs(argv) { }); const [verb, ...rest] = positionals; - if ((verb !== undefined && verb !== 'start') || rest.length > 0) { + if ((verb !== undefined && verb !== 'start' && verb !== 'build') || rest.length > 0) { throw new Error(`unknown command: ${positionals.join(' ')}`); } + const command = verb === 'build' ? 'build' : 'serve'; + const misplaced = (command === 'build' ? SERVE_OPTIONS : BUILD_OPTIONS).find( + (name) => name in values + ); + + if (misplaced) { + throw new Error( + `--${misplaced} does not apply to ${command === 'build' ? 'build' : 'serving'}` + ); + } + const port = Number(values.port ?? DEFAULTS.port); if (!Number.isInteger(port) || port < 0 || port > 65535) { @@ -130,9 +156,12 @@ export function parseCliArgs(argv) { } return { + command, + force: values.force ?? false, help: values.help ?? false, host: values.host ?? DEFAULTS.host, open: values.open ?? true, + out: values.out ?? DEFAULTS.out, port, server: parseServerUrl(values.server ?? DEFAULTS.server), version: values.version ?? false @@ -346,6 +375,32 @@ export function createChatServer({ distDir = DIST_DIR, server }) { }); } +/** + * Writes the app to a folder: the files of `dist` without source maps, ready for any static + * file server. Refuses a folder that is not empty unless `force` is set, and a folder in `dist`. + * @param {{ distDir?: string, force?: boolean, out: string }} options + * @returns {Promise} the absolute path of the folder + */ +export async function buildStatic({ distDir = DIST_DIR, force = false, out }) { + const source = resolve(distDir); + const target = resolve(out); + const inside = relative(source, target); + + if (inside === '' || !(inside.startsWith('..') || isAbsolute(inside))) { + throw new Error(`cannot build into ${target}, it is inside the app's own files`); + } + + const entries = await readdir(target).catch(() => []); + + if (entries.length > 0 && !force) { + throw new Error(`${target} is not empty, pick another --out or add --force`); + } + + await cp(source, target, { filter: (file) => !file.endsWith('.map'), recursive: true }); + + return target; +} + /** * Opens a URL in the default browser, quietly doing nothing where that fails. * @param {string} url @@ -404,6 +459,23 @@ export async function main(argv) { return; } + if (options.command === 'build') { + try { + const target = await buildStatic({ force: options.force, out: options.out }); + const shown = relative(process.cwd(), target) || '.'; + + console.log(`Llama Chat ${packageVersion()} written to ${target}`); + console.log( + `Serve it on the same origin as the llama-server API, e.g.: npx http-server ${shown} -P ${DEFAULTS.server}` + ); + } catch (error) { + console.error(`llama-chat: ${error instanceof Error ? error.message : error}`); + process.exitCode = 1; + } + + return; + } + const server = createChatServer({ server: options.server }); return new Promise((resolve) => { diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts index fd9e3b73d0c6..d6fcd515466f 100644 --- a/ui/apps/chat/tests/unit/cli.test.ts +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -1,5 +1,11 @@ -import { createChatServer, DEFAULTS, parseCliArgs, parseServerUrl } from '../../cli/server.js'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { + buildStatic, + createChatServer, + DEFAULTS, + parseCliArgs, + parseServerUrl +} from '../../cli/server.js'; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; @@ -54,6 +60,22 @@ describe('llama-chat CLI arguments', () => { expect(() => parseCliArgs(['--port', '70000'])).toThrow(/invalid --port/); }); + it('reads build with its own flags, and keeps serving flags out of it', () => { + expect(parseCliArgs(['build'])).toMatchObject({ + command: 'build', + force: false, + out: DEFAULTS.out + }); + expect(parseCliArgs(['build', '--out', 'public', '--force'])).toMatchObject({ + command: 'build', + force: true, + out: 'public' + }); + expect(parseCliArgs([]).command).toBe('serve'); + expect(() => parseCliArgs(['build', '--port', '3001'])).toThrow(/does not apply to build/); + expect(() => parseCliArgs(['--out', 'public'])).toThrow(/does not apply to serving/); + }); + it('takes a server address without a scheme as http', () => { expect(parseServerUrl('localhost:8080').href).toBe('http://localhost:8080/'); expect(() => parseServerUrl('ftp://localhost')).toThrow(/invalid --server/); @@ -74,6 +96,56 @@ describe('llama-chat packages', () => { }); }); +describe('llama-chat build', () => { + let dist: string; + let work: string; + + beforeAll(async () => { + dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-')); + work = await mkdtemp(join(tmpdir(), 'llama-chat-out-')); + await mkdir(join(dist, '_app'), { recursive: true }); + await writeFile(join(dist, 'index.html'), ''); + await writeFile(join(dist, '_app', 'app.js'), 'export {};'); + await writeFile(join(dist, '_app', 'app.js.map'), '{}'); + }); + + afterAll(async () => { + await rm(dist, { force: true, recursive: true }); + await rm(work, { force: true, recursive: true }); + }); + + it('writes dist to the folder without source maps', async () => { + const out = await buildStatic({ distDir: dist, out: join(work, 'site') }); + + expect((await readdir(out, { recursive: true })).sort()).toEqual([ + '_app', + join('_app', 'app.js'), + 'index.html' + ]); + }); + + it('refuses a folder that is not empty unless forced', async () => { + const out = join(work, 'busy'); + + await mkdir(out); + await writeFile(join(out, 'keep.txt'), 'mine'); + + await expect(buildStatic({ distDir: dist, out })).rejects.toThrow(/not empty/); + await buildStatic({ distDir: dist, force: true, out }); + expect(await readFile(join(out, 'keep.txt'), 'utf8')).toBe('mine'); + expect(await readFile(join(out, 'index.html'), 'utf8')).toBe(''); + }); + + it('refuses to write into the app files themselves', async () => { + await expect(buildStatic({ distDir: dist, force: true, out: dist })).rejects.toThrow( + /inside the app's own files/ + ); + await expect( + buildStatic({ distDir: dist, force: true, out: join(dist, '_app', 'copy') }) + ).rejects.toThrow(/inside the app's own files/); + }); +}); + describe('llama-chat server', () => { let dist: string; let upstream: http.Server; From 79bbd89886968e9188989597e35daa13e808f7ba Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 09:42:53 +0100 Subject: [PATCH 06/10] ui: publish Llama Chat to npm from CI when its version is bumped ui-chat-publish.yml publishes @ggml-org/llama-chat and its llama-chat alias the way ui-lib-publish.yml publishes @ggml-org/llama-ui: the version is set by hand, and a commit that reaches master with a version npm does not have builds dist, checks the packed files and publishes both with trusted publishing. PRs that change the packages get a dry run. npm-version-check.mjs takes the package folder, so both workflows share it. ui/README.md says how releases work, and Chat's AGENTS.md says not to bump versions unless asked to release. Assisted-by: Claude (Opus 5.5 Medium) --- .github/workflows/ui-chat-publish.yml | 202 ++++++++++++++++++++++++ ui/README.md | 19 ++- ui/apps/chat/AGENTS.md | 1 + ui/apps/chat/README.md | 12 +- ui/lib/ui/scripts/npm-version-check.mjs | 15 +- 5 files changed, 237 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ui-chat-publish.yml diff --git a/.github/workflows/ui-chat-publish.yml b/.github/workflows/ui-chat-publish.yml new file mode 100644 index 000000000000..2de0907b95c9 --- /dev/null +++ b/.github/workflows/ui-chat-publish.yml @@ -0,0 +1,202 @@ +name: UI Chat Publish + +# Publishes Llama Chat to npm: @ggml-org/llama-chat (ui/apps/chat, the app and its CLI) and +# llama-chat (ui/apps/chat/alias, the short name for npx), always at the same version. +# +# The version in ui/apps/chat/package.json is the release, set by hand, as for @ggml-org/llama-ui +# in ui-lib-publish.yml. A commit that reaches master with a version npm does not have builds the +# app and publishes both packages; any other commit publishes nothing. Bump both package.json +# files in the PR that should ship. +# +# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted +# publishing) and signs the tarballs with provenance. One-time setup, before this can publish: on +# each of the two packages, Publishing settings, Trusted Publisher, GitHub Actions, with +# organization ggml-org, repository llama.cpp, workflow file ui-chat-publish.yml and Environment +# left blank. Until then the publish step fails on master; re-running the job publishes it. + +on: + workflow_dispatch: + inputs: + dry_run: + description: 'Build and pack, but do not publish' + required: false + type: boolean + default: false + push: + branches: + - master + paths: [ + '.github/workflows/ui-chat-publish.yml', + 'ui/apps/chat/package.json', + 'ui/apps/chat/alias/package.json' + ] + # a dry run for PRs that change what goes into the packages + pull_request: + types: [opened, synchronize, reopened] + paths: [ + '.github/workflows/ui-chat-publish.yml', + 'ui/apps/chat/package.json', + 'ui/apps/chat/alias/**', + 'ui/apps/chat/bin/**', + 'ui/apps/chat/cli/**' + ] + +cache-mode: none +permissions: + contents: read + id-token: write + +concurrency: + # never cancel a run mid-publish: a version half-published cannot be taken back + group: ${{ github.workflow }} + cancel-in-progress: false + +env: + APP_DIR: ui/apps/chat + ALIAS_DIR: ui/apps/chat/alias + VERSION_CHECK: ui/lib/ui/scripts/npm-version-check.mjs + +jobs: + publish: + name: Publish @ggml-org/llama-chat and llama-chat + if: github.repository == 'ggml-org/llama.cpp' + runs-on: ubuntu-26.04-arm + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + registry-url: 'https://registry.npmjs.org' + package-manager-cache: false + + - name: Check npm can publish with an OIDC token + run: | + node -e ' + const v = process.argv[1]; + const [major, minor] = v.split(".").map(Number); + if (major < 11 || (major === 11 && minor < 5)) { + console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`); + process.exit(1); + } + ' "$(npm -v)" + + - name: Install dependencies + run: npm ci + working-directory: ${{ env.APP_DIR }} + + - name: Build the app + # the same build.json version as the UI that llama-server embeds, for the app's update check. + # This folder's .npmrc sets ignore-scripts, so npm never builds it: this step does. + run: LLAMA_BUILD_NUMBER="b$(git rev-list --count HEAD)" npm run build + working-directory: ${{ env.APP_DIR }} + + - name: Test the CLI and the alias version + run: npx vitest --project=unit --run tests/unit/cli.test.ts + working-directory: ${{ env.APP_DIR }} + + - name: Check what goes into the package + run: | + npm pack --dry-run --json > "$RUNNER_TEMP/pack.json" + node -e ' + const [{ files }] = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); + const paths = files.map((file) => file.path); + const maps = paths.filter((path) => path.endsWith(".map")); + if (!paths.includes("dist/index.html")) { + console.error("dist/index.html is not in the package: nothing would be served"); + process.exit(1); + } + if (maps.length > 0) { + console.error(`source maps in the package: ${maps.join(", ")}`); + process.exit(1); + } + console.log(`${paths.length} files, no source maps`); + ' "$RUNNER_TEMP/pack.json" + working-directory: ${{ env.APP_DIR }} + + - name: Compare @ggml-org/llama-chat with npm + id: app + continue-on-error: ${{ github.event_name == 'pull_request' }} + run: node "$VERSION_CHECK" "$APP_DIR" + + - name: Compare llama-chat with npm + id: alias + continue-on-error: ${{ github.event_name == 'pull_request' }} + run: node "$VERSION_CHECK" "$ALIAS_DIR" + + # master only: npm's trusted publisher matches the repository and the workflow file, not the + # branch, so this condition is what keeps a manual run on a PR branch from publishing. + # The app goes first, since the alias depends on it at the same version. + - name: Publish @ggml-org/llama-chat + if: github.ref == 'refs/heads/master' && steps.app.outputs.should_publish == 'true' && inputs.dry_run != true + env: + TAG: ${{ steps.app.outputs.tag }} + run: npm publish --access public --provenance --ignore-scripts --tag "$TAG" + working-directory: ${{ env.APP_DIR }} + + - name: Publish llama-chat + if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true + env: + TAG: ${{ steps.alias.outputs.tag }} + run: npm publish --access public --provenance --ignore-scripts --tag "$TAG" + working-directory: ${{ env.ALIAS_DIR }} + + - name: Verify they landed + if: github.ref == 'refs/heads/master' && (steps.app.outputs.should_publish == 'true' || steps.alias.outputs.should_publish == 'true') && inputs.dry_run != true + env: + VERSION: ${{ steps.app.outputs.version }} + run: | + # the registry can serve a new version some time after publish returns + for name in @ggml-org/llama-chat llama-chat; do + for try in 1 2 3 4 5 6; do + if [ "$(npm view "$name@$VERSION" version 2>/dev/null || true)" = "$VERSION" ]; then + echo "$name@$VERSION is on npm" + continue 2 + fi + echo "$name@$VERSION not visible yet (attempt $try of 6)" + sleep 10 + done + echo "::error::$name@$VERSION published without error but is not visible on the registry. Check npm before publishing again." + exit 1 + done + + - name: Report + if: always() + # values reach the shell through env, so a name from a PR's package.json cannot inject here + env: + APP: ${{ steps.app.outputs.name }}@${{ steps.app.outputs.version }} + APP_VERDICT: ${{ steps.app.outputs.verdict || 'not reached' }} + APP_NOTE: ${{ steps.app.outputs.note }} + ALIAS: ${{ steps.alias.outputs.name }}@${{ steps.alias.outputs.version }} + ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }} + ALIAS_NOTE: ${{ steps.alias.outputs.note }} + DRY: ${{ inputs.dry_run }} + EVENT: ${{ github.event_name }} + run: | + { + echo "### Llama Chat on npm" + echo + echo "| package | verdict | note |" + echo "| --- | --- | --- |" + echo "| \`$APP\` | \`$APP_VERDICT\` | $APP_NOTE |" + echo "| \`$ALIAS\` | \`$ALIAS_VERDICT\` | $ALIAS_NOTE |" + echo + if [ "$EVENT" = "pull_request" ] || [ "$DRY" = "true" ]; then echo "Dry run: nothing was published."; fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Mind the verdict + # a version going backwards is a mistake; an unchanged version is a package.json edit that + # is not a release, such as a dependency update + if: always() && github.event_name != 'pull_request' + env: + VERDICTS: ${{ steps.app.outputs.verdict }} ${{ steps.alias.outputs.verdict }} + run: | + case " $VERDICTS " in + *" stale "*) + echo "::error::a version in package.json is older than the one on npm" + exit 1 ;; + esac diff --git a/ui/README.md b/ui/README.md index 97e6a35e72c5..b70228d0de7c 100644 --- a/ui/README.md +++ b/ui/README.md @@ -14,7 +14,24 @@ ui/ `@sveltejs/package` and published to npm as `@ggml-org/llama-ui`. Apps in this repo skip the package: they compile its sources as part of their own build, so they hold no copy of a component, a token or a font. - [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop. CMake - builds it into llama-server (see `scripts/ui-assets.cmake`). + builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as `@ggml-org/llama-chat` + (with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server. Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building, checking and testing `apps/chat` needs no install in `lib/ui`. + +## Releases + +The npm packages are versioned by hand, with semver, and published by CI: + +| Package | Version in | Published by | +| --- | --- | --- | +| `@ggml-org/llama-ui` | `lib/ui/package.json` | [`ui-lib-publish.yml`](../.github/workflows/ui-lib-publish.yml) | +| `@ggml-org/llama-chat` and `llama-chat` | `apps/chat/package.json` and `apps/chat/alias/package.json`, always the same | [`ui-chat-publish.yml`](../.github/workflows/ui-chat-publish.yml) | + +To release, bump the version in the PR that should ship the change: a patch for fixes, a minor for new or changed +features or API while the version is 0.x. When that PR reaches `master` with a version npm does not have yet, the +workflow builds the package and publishes it; any other commit publishes nothing. A version lower than the one on npm +fails the run. + +Llama Chat as bundled in llama-server is not versioned this way: it ships in every llama.cpp release, as `b`. diff --git a/ui/apps/chat/AGENTS.md b/ui/apps/chat/AGENTS.md index f585ec54fec8..32e6a009a2c9 100644 --- a/ui/apps/chat/AGENTS.md +++ b/ui/apps/chat/AGENTS.md @@ -4,6 +4,7 @@ These add to the repository's [AGENTS.md](../../../AGENTS.md) for work in `ui/ap - **Design:** [DESIGN.md](DESIGN.md) holds the tokens, roles and rules, written for agents. Follow it for anything visual. Its token sections (`colors`, `typography`, `rounded`) are generated from the CSS with `npm run design:tokens`, and CI checks they are current; the rest is written by hand, and when it and the CSS disagree, the CSS wins and DESIGN.md is updated. - **Stories:** a story lives with the code it shows. Llama UI's (its foundations, base components, app components and Model UI) are in `ui/lib/ui/tests/stories`, titled `Llama UI/...`, and run as the library's tests; this app's Storybook shows them too. Llama Chat's are in `tests/stories/`, one folder per `src/lib/components/app/` (`chat/` has one per Chat folder), titled `Llama Chat//`. An accessibility story sits next to its component's, as `.a11y.stories.svelte`, titled `...//Accessibility`. The story helpers both Storybooks use (the Editorial the prose pages are set in, `StateGallery`) live with the library's stories and are imported here as `$llama-ui-stories/...`; reuse them rather than copying them here. +- **Releases:** `@ggml-org/llama-chat` and `@ggml-org/llama-ui` are versioned by hand and published by CI when a new version reaches `master` (see [Releases](../../README.md#releases)). Bump a version only when asked to release, never as a side effect of a change; Chat's version and its alias's (`alias/package.json`) always move together. - **Components:** the `SKILL.md` next to a component folder (such as [src/lib/components/app/SKILL.md](src/lib/components/app/SKILL.md)) holds that folder's conventions. ## Content is Markdown, custom pieces are Svelte: .svx diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 5d120b520e61..371998b988e2 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -832,22 +832,22 @@ ui/apps/chat/ Two packages go out with the same version: `@ggml-org/llama-chat`, which carries the app and the CLI, and `llama-chat` (in `alias/`), which depends on it and only forwards to its CLI so that `npx llama-chat` works. A unit test (`tests/unit/cli.test.ts`) fails when their versions drift apart. +The version is set by hand and CI publishes it, as for `@ggml-org/llama-ui` (see [Releases](../../README.md#releases)). To release, bump the version in the PR that should ship it: + ```bash cd ui/apps/chat npm version --no-git-tag-version # then set the same version, and the same @ggml-org/llama-chat dependency, in alias/package.json -npm run build -npm pack --dry-run -npm publish --otp -cd alias && npm publish --otp ``` -Build right before packing: the package carries whatever is in `dist`. This folder's `.npmrc` sets `ignore-scripts`, so npm runs no lifecycle script such as `prepack` that could build it for you. Check that `npm pack --dry-run` lists `dist/index.html` and about 77 files. Source maps stay out of the package. +Once that reaches `master`, [`ui-chat-publish.yml`](../../../.github/workflows/ui-chat-publish.yml) builds `dist`, checks that the package holds `dist/index.html` and no source maps, and publishes both packages with npm trusted publishing. A version npm already has publishes nothing, and a PR that changes the packages gets a dry run. -To try a package before publishing it: +To try a package before it is published: ```bash npm run build npm pack npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:8080 ``` + +Build right before packing: the package carries whatever is in `dist`, and this folder's `.npmrc` sets `ignore-scripts`, so npm runs no lifecycle script such as `prepack` that could build it for you. diff --git a/ui/lib/ui/scripts/npm-version-check.mjs b/ui/lib/ui/scripts/npm-version-check.mjs index e992c2e96c58..51280594e662 100644 --- a/ui/lib/ui/scripts/npm-version-check.mjs +++ b/ui/lib/ui/scripts/npm-version-check.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node -// Decides whether this library is a version npm does not have yet, for -// .github/workflows/ui-lib-publish.yml. It reads package.json next to its own folder, so it -// runs from anywhere. +// Decides whether a package is a version npm does not have yet, for +// .github/workflows/ui-lib-publish.yml and ui-chat-publish.yml. It reads the package.json in the +// folder given as its argument, or this library's own without one, so it runs from anywhere. // // The version in package.json is the release: this takes it, asks the registry what it holds, // and says whether to publish, under which dist-tag, and why not when not. The version already @@ -12,10 +12,15 @@ // registry could not be read: the workflow decides between warning and failing. import { readFileSync, appendFileSync } from 'node:fs'; +import { join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; const REGISTRY = process.env.NPM_REGISTRY_URL ?? 'https://registry.npmjs.org'; -const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const REPO_ROOT = fileURLToPath(new URL('../../../../', import.meta.url)); +const pkgDir = process.argv[2] ? resolve(process.argv[2]) : fileURLToPath(new URL('..', import.meta.url)); +const pkgPath = relative(REPO_ROOT, join(pkgDir, 'package.json')); +const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')); const { name, version } = pkg; if (!name || !version) { @@ -53,7 +58,7 @@ let note = `${name}@${version} is not on npm${latest ? `, which holds ${latest}` if (publishedHere) { verdict = 'published'; shouldPublish = false; - note = `${name}@${version} is already on npm. Bump the version in ui/lib/ui/package.json to release these changes.`; + note = `${name}@${version} is already on npm. Bump the version in ${pkgPath} to release these changes.`; } else if (latest && compare(version, latest) < 0) { // covers the prerelease of an already-released core: 0.4.0-beta.1 behind 0.4.0 verdict = 'stale'; From 13620842ef47a13343651c36b704fa840adcb4bf Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 10:53:42 +0100 Subject: [PATCH 07/10] ui: call the app Llama Chat in its README The README's title and a few lines still called the app llama-ui or Llama UI, the name of the library it is built on. They now say Llama Chat, and the intro points to Llama UI in ui/lib/ui. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 371998b988e2..9054286fc0d7 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -1,8 +1,8 @@ -# llama-ui +# Llama Chat -A modern, feature-rich web interface for llama-server built with SvelteKit. This UI provides an intuitive chat interface with advanced file handling, conversation management, and comprehensive model interaction capabilities. +Llama Chat is the chat app for llama.cpp: a web interface for llama-server built with SvelteKit, with file handling, conversation management and model interaction. It is built on [Llama UI](../../lib/ui/README.md) (`@ggml-org/llama-ui`), the design system and component library in `ui/lib/ui`. -Llama UI supports two server operation modes: +Llama Chat supports two server operation modes: - **MODEL mode** - Single model operation (standard llama-server) - **ROUTER mode** - Multi-model operation with dynamic model loading/unloading @@ -280,19 +280,19 @@ output: { ### Integration with llama-server -llama-ui is embedded directly into the llama-server binary: +Llama Chat is embedded directly into the llama-server binary: 1. `npm run build` outputs `index.html` to `build/ui/apps/chat/dist/` 2. llama-server compiles this into the binary at build time 3. When accessing `/`, llama-server serves the bundled HTML -This results in a **single portable binary** with the full Llama UI included. +This results in a **single portable binary** with Llama Chat included. --- ## Architecture -Llama UI follows a layered architecture with unidirectional data flow: +Llama Chat follows a layered architecture with unidirectional data flow: ```text Routes → Components → Hooks → Stores → Services → Storage/API From be285127084997f23e2d4f97d187a2f05f7548c2 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 10:55:38 +0100 Subject: [PATCH 08/10] ui: llama-chat connects to llama-server's default port, 9931 The CLI's --server default and its examples used 8080, which is not where llama-server listens. They now use 9931, its default port, so npx llama-chat reaches a llama-server started without --port. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 8 ++++---- ui/apps/chat/alias/README.md | 2 +- ui/apps/chat/cli/server.js | 4 ++-- ui/apps/chat/tests/unit/cli.test.ts | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 9054286fc0d7..6300d5c995d3 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -36,7 +36,7 @@ npx @ggml-org/llama-chat ```bash # the same, by its short name, against a llama-server elsewhere on the network -npx llama-chat --server http://192.168.1.20:8080 +npx llama-chat --server http://192.168.1.20:9931 ``` ```bash @@ -51,7 +51,7 @@ npx llama-chat build | Flag | Default | What it does | | --------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------- | -| `--server ` | `http://127.0.0.1:8080` | The llama-server to connect to. A path is kept, for a server started with `--api-prefix` | +| `--server ` | `http://127.0.0.1:9931` | The llama-server to connect to. A path is kept, for a server started with `--api-prefix` | | `--port ` | `3000` | Fixed, so the origin, and the chats the browser stores under it, stay the same between runs | | `--host ` | `127.0.0.1` | `0.0.0.0` to open it to the LAN | | `--open` / `--no-open` | open | Opens the default browser once it listens | @@ -72,7 +72,7 @@ The app is static and uses relative paths and a hash router, so any static host ```bash npx llama-chat build - npx http-server llama-chat -p 3000 -P http://127.0.0.1:8080 + npx http-server llama-chat -p 3000 -P http://127.0.0.1:9931 ``` A plain file server sends the browser's `Origin` on to llama-server, so start llama-server with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Remote Assets Under Cross-Origin Isolation](#9-remote-assets-under-cross-origin-isolation)). @@ -847,7 +847,7 @@ To try a package before it is published: ```bash npm run build npm pack -npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:8080 +npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:9931 ``` Build right before packing: the package carries whatever is in `dist`, and this folder's `.npmrc` sets `ignore-scripts`, so npm runs no lifecycle script such as `prepack` that could build it for you. diff --git a/ui/apps/chat/alias/README.md b/ui/apps/chat/alias/README.md index a83be04d61d4..9ac31f5ad10a 100644 --- a/ui/apps/chat/alias/README.md +++ b/ui/apps/chat/alias/README.md @@ -3,7 +3,7 @@ The short name for [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat): `npx llama-chat` runs Llama Chat, the chat app for llama.cpp, on localhost and connects it to a llama-server. ```bash -npx llama-chat --server http://127.0.0.1:8080 +npx llama-chat --server http://127.0.0.1:9931 ``` This package only forwards to `@ggml-org/llama-chat`, at the same version. See its [README](https://github.com/ggml-org/llama.cpp/tree/master/ui/apps/chat#readme) for the options. diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js index 7dd90345e7f5..5b04852e1b3c 100644 --- a/ui/apps/chat/cli/server.js +++ b/ui/apps/chat/cli/server.js @@ -28,7 +28,7 @@ export const DEFAULTS = Object.freeze({ // fixed, so the origin, and the chats the browser stores under it, stay the same between runs port: 3000, // where llama-server listens when started without --host or --port - server: 'http://127.0.0.1:8080' + server: 'http://127.0.0.1:9931' }); // The same caching llama-server applies to the UI it embeds (tools/server/server-http.cpp): @@ -100,7 +100,7 @@ Build options: Examples: npx @ggml-org/llama-chat - npx @ggml-org/llama-chat --server http://192.168.1.20:8080 + npx @ggml-org/llama-chat --server http://192.168.1.20:9931 npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open npx @ggml-org/llama-chat build --out ./public`; diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts index d6fcd515466f..1c8be0977cd8 100644 --- a/ui/apps/chat/tests/unit/cli.test.ts +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -37,7 +37,7 @@ describe('llama-chat CLI arguments', () => { it('reads the documented flags', () => { const options = parseCliArgs([ '--server', - 'http://192.168.1.20:8080', + 'http://192.168.1.20:9931', '--host', '0.0.0.0', '--port', @@ -46,7 +46,7 @@ describe('llama-chat CLI arguments', () => { ]); expect(options).toMatchObject({ host: '0.0.0.0', open: false, port: 3001 }); - expect(options.server.href).toBe('http://192.168.1.20:8080/'); + expect(options.server.href).toBe('http://192.168.1.20:9931/'); }); it('accepts start as a verb that changes nothing', () => { @@ -77,7 +77,7 @@ describe('llama-chat CLI arguments', () => { }); it('takes a server address without a scheme as http', () => { - expect(parseServerUrl('localhost:8080').href).toBe('http://localhost:8080/'); + expect(parseServerUrl('localhost:9931').href).toBe('http://localhost:9931/'); expect(() => parseServerUrl('ftp://localhost')).toThrow(/invalid --server/); }); }); From bd1a2fd9b4758084c26e320b732011c24e361184 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 11:01:39 +0100 Subject: [PATCH 09/10] ui: a shorter Llama Chat README that matches the code The README grew to nearly 900 lines, and its tables of components, hooks, stores and services had drifted from the code. It now keeps what a reader needs: running it with npx and self-hosting, a short feature list, development commands, the architecture (the line between Llama UI and the app, a table of the layers and where they live, a flowchart and one sequence diagram for sending a message), a few patterns worth knowing, cross-origin isolation, how the build is embedded in llama-server, the project tree and publishing. The folders now carry the detail the tables used to. ASCII replaces the unicode arrows. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 832 +++++------------------------------------ 1 file changed, 94 insertions(+), 738 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 6300d5c995d3..80aa8c3da97f 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -1,52 +1,28 @@ # Llama Chat -Llama Chat is the chat app for llama.cpp: a web interface for llama-server built with SvelteKit, with file handling, conversation management and model interaction. It is built on [Llama UI](../../lib/ui/README.md) (`@ggml-org/llama-ui`), the design system and component library in `ui/lib/ui`. +Llama Chat is the chat app for llama.cpp: a web interface for llama-server, built with SvelteKit on [Llama UI](../../lib/ui/README.md) (`@ggml-org/llama-ui`), the design system in `ui/lib/ui`. llama-server embeds it, and `npx llama-chat` runs it anywhere. -Llama Chat supports two server operation modes: +It works with a llama-server in either mode: **MODEL** (one model) or **ROUTER** (many models, loaded and unloaded on demand). -- **MODEL mode** - Single model operation (standard llama-server) -- **ROUTER mode** - Multi-model operation with dynamic model loading/unloading - ---- - -## Table of Contents +## Contents - [Run it with npx](#run-it-with-npx) - [Self-hosting](#self-hosting) - [Features](#features) -- [Getting Started](#getting-started) -- [Tech Stack](#tech-stack) -- [Build Pipeline](#build-pipeline) +- [Development](#development) - [Architecture](#architecture) -- [Data Flows](#data-flows) -- [Architectural Patterns](#architectural-patterns) -- [Testing](#testing) +- [Build and embedding](#build-and-embedding) - [Publishing to npm](#publishing-to-npm) ---- - ## Run it with npx Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app on localhost and connects it to a llama-server you run yourself, or writes it to a folder for any static file server (see [Self-hosting](#self-hosting)); it brings no server and no models of its own. It needs Node.js 20 or later. ```bash -# start llama-server, then serve Llama Chat on http://127.0.0.1:3000 and open the browser -npx @ggml-org/llama-chat -``` - -```bash -# the same, by its short name, against a llama-server elsewhere on the network -npx llama-chat --server http://192.168.1.20:9931 -``` - -```bash -# reachable from other devices, without opening a browser -npx llama-chat --host 0.0.0.0 --port 3000 --no-open -``` - -```bash -# write the static app to ./llama-chat instead of serving it -npx llama-chat build +npx @ggml-org/llama-chat # serve it on http://127.0.0.1:3000 and open the browser +npx llama-chat --server http://192.168.1.20:9931 # the same, by its short name, against another llama-server +npx llama-chat --host 0.0.0.0 --no-open # reachable from other devices +npx llama-chat build # write the static app to ./llama-chat instead ``` | Flag | Default | What it does | @@ -59,9 +35,7 @@ npx llama-chat build The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`) and `--force`, which lets it write into a folder that is not empty. -How it works: the app calls its server on its own origin (`./v1/...`, `./props`, ...), as it does when llama-server serves it. The CLI answers a request from `dist` when a file there matches its path and forwards everything else to the llama-server as if from the server's own page (its `Host`, and without the CLI page's `Origin`), so the server needs no `--cors-origins`. Requests from other pages keep their `Origin` and `Sec-Fetch-Site`, so llama-server refuses them as it would directly, and a CLI listening on loopback answers only to `localhost` and loopback addresses, which stops DNS rebinding. It sends the response headers llama-server sends for the UI: `index.html` is cross-origin isolated (`Cross-Origin-Opener-Policy` and `Cross-Origin-Embedder-Policy`) and revalidated, `service-worker.js`, `sw.js`, `manifest.webmanifest`, `build.json` and `_app/version.json` are revalidated, and every other file is cached for good. It is plain Node.js (`node:http`) with no dependencies. - -An `http://` page can call a plain `http://` llama-server on the LAN, which an `https://` page cannot (browsers block it as mixed content). That is why the npx command matters next to a hosted copy. +How it works: the app calls its server on its own origin (`./v1/...`, `./props`), as it does under llama-server. The CLI serves the files of `dist` and forwards every other request to llama-server as the server's own page, so the server needs no `--cors-origins`; requests from other pages keep their `Origin` and are refused as before, and on loopback it answers only to `localhost` names, against DNS rebinding. It sends llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good) and is plain Node.js with no dependencies. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot. ## Self-hosting @@ -75,759 +49,134 @@ The app is static and uses relative paths and a hash router, so any static host npx http-server llama-chat -p 3000 -P http://127.0.0.1:9931 ``` - A plain file server sends the browser's `Origin` on to llama-server, so start llama-server with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Remote Assets Under Cross-Origin Isolation](#9-remote-assets-under-cross-origin-isolation)). + A plain file server sends the browser's `Origin` on to llama-server, so start llama-server with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Cross-origin isolation](#cross-origin-isolation)). Until the app can add a server by URL, it reaches only the llama-server on its own origin: through the CLI, a reverse proxy, or llama-server itself (`--path dist`). ## Features -### Chat Interface - -- **Streaming responses** with real-time updates -- **Reasoning content** - Support for models with thinking/reasoning blocks -- **Dark/light theme** with system preference detection -- **Responsive design** for desktop and mobile - -### File Attachments - -- **Images** - JPEG, PNG, GIF, WebP, SVG (with PNG conversion) -- **Documents** - PDF (text extraction or image conversion for vision models) -- **Audio** - MP3, WAV for audio-capable models -- **Text files** - Source code, markdown, and other text formats -- **Drag-and-drop** and paste support with rich previews - -### Conversation Management - -- **Branching** - Branch messages conversations at any point by editing messages or regenerating responses, navigate between branches -- **Regeneration** - Regenerate responses with optional model switching (ROUTER mode) -- **Import/Export** - JSON format for backup and sharing -- **Search** - Find conversations by title or content - -### Advanced Rendering - -- **Syntax highlighting** - Code blocks with language detection -- **Math formulas** - KaTeX rendering for LaTeX expressions -- **Markdown** - Full GFM support with tables, lists, and more - -### Multi-Model Support (ROUTER mode) - -- **Models manager** - one table for every model the server can serve, split into loaded, downloading, favorites, local and hidden sections, with search, context/capability/modality filters, sorting, load/unload, delete from disk and hide. It opens from the sidebar, from a model row or from the selector. -- **Model selector** with Loaded/Available groups -- **Automatic loading** - Models load on selection -- **Modality validation** - Prevents sending images to non-vision models -- **LRU unloading** - Server auto-manages model cache -- **Downloads** - In-flight downloads stay visible above the selector and can be paused, resumed or cancelled from the manager -- **Hub metadata** - Avatars, context length and chat template come from the Hugging Face Hub only while the `Use Hugging Face Hub API for models metadata` setting is on. It is off by default, and the UI then shows what the server reports for `/v1/models` - -### Keyboard Shortcuts +- **Chat** - streaming replies, reasoning blocks, branching by edit or regenerate, tabs, search, import and export +- **Attachments** - images, PDFs (as text or as pages for vision models), audio and text files, by drag and drop or paste +- **Rendering** - Markdown (GFM), code highlighting, KaTeX math and Mermaid diagrams +- **Models** - in ROUTER mode, a selector and a models manager: load and unload, downloads, Discover from the Hugging Face Hub, favorites and hidden models +- **Tools** - built-in browser tools, llama-server's tools and MCP servers, each behind a permission prompt +- **Everywhere** - light and dark themes, phone layouts, installable as a PWA, keyboard shortcuts (`Ctrl/Cmd+K` search, `Shift+Ctrl/Cmd+O` new chat, `Ctrl/Cmd+B` sidebar) -| Shortcut | Action | -| ------------------ | -------------------- | -| `Shift+Ctrl/Cmd+O` | New chat | -| `Shift+Ctrl/Cmd+E` | Edit conversation | -| `Shift+Ctrl/Cmd+D` | Delete conversation | -| `Ctrl/Cmd+K` | Search conversations | -| `Ctrl/Cmd+B` | Toggle sidebar | +## Development -### Developer Experience - -- **Request tracking** - Monitor token generation with `/slots` endpoint -- **Storybook** - Component library with visual testing -- **Hot reload** - Instant updates during development - ---- - -## Getting Started - -### Prerequisites - -- **Node.js** 18+ (20+ recommended) -- **npm** 9+ -- **llama-server** running locally (for API access) - -### 1. Install Dependencies +You need Node.js 20 or later and a llama-server to talk to. ```bash cd ui/apps/chat npm ci +npm run dev # the app on http://localhost:5173, Storybook on http://localhost:6006 ``` -### 2. Start llama-server - -In a separate terminal, start the backend server: - -```bash -# Single model (MODEL mode) -./llama-server -m model.gguf - -# Multi-model (ROUTER mode) -./llama-server --models-dir /path/to/models -``` - -### 3. Start Development Servers - -```bash -npm run dev -``` - -This starts: - -- **Vite dev server** at `http://localhost:5173` - The main UI frontend app -- **Storybook** at `http://localhost:6006` - Component documentation - -The Vite dev server proxies API requests to `SERVER_ORIGIN` (with fallback to default llama-server `9931` port): - -```typescript -// vite.config.ts proxy configuration -proxy: { - '/v1': SERVER_ORIGIN, - '/props': SERVER_ORIGIN, - '/models': SERVER_ORIGIN, - '/tools': SERVER_ORIGIN, - '/slots': SERVER_ORIGIN, - '/cors-proxy': SERVER_ORIGIN -}, -``` - -### Development Workflow - -1. Open `http://localhost:5173` in your browser -2. Make changes to `.svelte`, `.ts`, or `.css` files -3. Changes hot-reload instantly -4. Use Storybook at `http://localhost:6006` for isolated component development - ---- - -## Tech Stack - -| Layer | Technology | Purpose | -| ----------------- | ------------------------------- | -------------------------------------------------------- | -| **Framework** | SvelteKit + Svelte 5 | Reactive UI with runes (`$state`, `$derived`, `$effect`) | -| **UI Components** | shadcn-svelte + bits-ui | Accessible, customizable component library | -| **Drawers** | vaul-svelte | Draggable drawer overlays (mobile pickers, model pane) | -| **Styling** | TailwindCSS 4 | Utility-first CSS with design tokens | -| **Database** | IndexedDB (Dexie) | Client-side storage for conversations and messages | -| **Build** | Vite | Fast bundling with static adapter | -| **Testing** | Playwright + Vitest + Storybook | E2E, unit, and visual testing | -| **Markdown** | remark + rehype | Markdown processing with KaTeX and syntax highlighting | - -### Key Dependencies - -```json -{ - "svelte": "^5.0.0", - "bits-ui": "^2.8.11", - "dexie": "^4.0.11", - "pdfjs-dist": "^5.4.54", - "highlight.js": "^11.11.1", - "rehype-katex": "^7.0.1" -} -``` +The dev server forwards the API (`/v1`, `/props`, `/models`, `/tools`, `/slots`, `/cors-proxy`) to llama-server on `http://localhost:9931`, or to `VITE_PUBLIC_SERVER_ORIGIN` when it is set. ---- +| Command | What it does | +| --------------------- | --------------------------------------------------------------- | +| `npm run build` | The production build, in `dist/` | +| `npm run check` | Type checking with svelte-check | +| `npm run lint` | Prettier and ESLint; `npm run format` fixes what they can | +| `npm run test:unit` | Unit tests (Vitest, `tests/unit`) | +| `npm run test:client` | Component tests in the browser (Vitest, `tests/client`) | +| `npm run test:ui` | Storybook stories run as tests (`tests/stories`) | +| `npm run test:e2e` | End-to-end tests (Playwright, `tests/e2e`), with a llama-server | +| `npm run storybook` | Storybook on its own | -## Build Pipeline - -### Development Build - -```bash -npm run dev -``` - -Runs Vite in development mode with: - -- Hot Module Replacement (HMR) -- Source maps -- Proxy to llama-server - -### Production Build - -```bash -npm run build -``` - -The build process: - -1. **Vite Build** - Bundles all TypeScript, Svelte, and CSS -2. **Static Adapter** - Outputs to `../../../build/ui/apps/chat/dist` (llama-server's static file directory) -3. **Post-Build Script** - Cleans up intermediate files -4. **Custom Plugin** - Creates `index.html` with: - - Inlined favicon as base64 - - GZIP compression (level 9) - - Deterministic output (zeroed timestamps) - -```text -ui/apps/chat/ → build → build/ui/apps/chat/dist/ -├── src/ ├── index.html (served by llama-server) -├── static/ └── (favicon inlined) -└── ... -``` - -### SvelteKit Configuration - -```javascript -// vite.config.ts - options passed to sveltekit({ ... }) -adapter: adapter({ - pages: '../../../build/ui/apps/chat/dist', // Output directory - assets: '../../../build/ui/apps/chat/dist', // Static assets - fallback: 'index.html', // SPA fallback - strict: true -}), -output: { - bundleStrategy: 'inline' // Single-file bundle -} -``` - -### Integration with llama-server - -Llama Chat is embedded directly into the llama-server binary: - -1. `npm run build` outputs `index.html` to `build/ui/apps/chat/dist/` -2. llama-server compiles this into the binary at build time -3. When accessing `/`, llama-server serves the bundled HTML - -This results in a **single portable binary** with Llama Chat included. - ---- +Locally the browser tests run in your installed Google Chrome; CI uses Playwright's bundled Chromium. ## Architecture -Llama Chat follows a layered architecture with unidirectional data flow: - -```text -Routes → Components → Hooks → Stores → Services → Storage/API -``` - -### High-Level Architecture - -```mermaid -flowchart TB - subgraph Routes["📍 Routes"] - R1["/ (Welcome)"] - R2["/chat/[id]"] - R3["/mcp-servers"] - R4["/search"] - R5["/settings"] - RL["+layout.svelte"] - end - - subgraph Components["🧩 Components"] - C_Screen["ChatScreen"] - C_Form["ChatForm"] - C_Messages["ChatMessages"] - C_Sidebar["ChatSidebar"] - C_Models["ModelsSelector"] - C_Settings["ChatSettings"] - C_Mcp["McpServers"] - end - - subgraph Hooks["🔌 Hooks"] - H1["use-chat-screen-active-model"] - H2["use-processing-state"] - H3["use-context-gauge"] - H4["use-models-selector"] - H5["use-tools-panel"] - end - - subgraph Stores["🗄️ Stores"] - S1["chatStore"] - S2["conversationsStore"] - S3["modelsStore"] - S4["mcpStore"] - S5["agenticStore"] - S6["serverStore"] - S7["settingsStore"] - S8["toolsStore"] - end +Llama Chat is the first app on Llama UI, and the reference for the line between library and app: **Llama UI draws and handles interaction, the app decides what happens.** - subgraph Services["⚙️ Services"] - SV1["ChatService"] - SV2["ModelsService"] - SV3["PropsService"] - SV4["DatabaseService"] - SV5["MCPService"] - SV6["ToolsService"] - SV7["SandboxService"] - end +- **In Llama UI:** components, design tokens, fonts and surfaces, and the logic inside them (keyboard navigation, validation as you type, formatting a file size). Components take data in and report what the user did out. +- **In Llama Chat:** talking to llama-server, storing conversations, the agentic loop and its tool permissions, MCP servers, managing models, settings. - subgraph Storage["💾 Storage"] - ST1["IndexedDB"] - ST2["LocalStorage"] - end +The app imports the library as `@ggml-org/llama-ui/ui/` and compiles its sources in its own build, with no install of the library (see `scripts/vite-plugin-llama-ui.ts`). Data flows one way, through these layers: - subgraph APIs["🌐 llama-server"] - API1["/v1/chat/completions"] - API2["/props"] - API3["/models/*"] - API4["/tools"] - end - - R1 & R2 --> C_Screen - RL --> C_Sidebar - C_Screen --> C_Form & C_Messages & C_Settings - C_Screen --> H1 & H2 & H3 - C_Models --> H4 - C_Mcp --> S4 - C_Screen --> S1 & S2 & S3 - C_Models --> S3 - H1 --> S3 - S1 --> SV1 & SV4 - S2 --> SV4 - S3 --> SV2 & SV3 - S4 --> SV5 - S5 --> SV1 & SV5 & SV6 & SV7 - SV4 --> ST1 - SV1 --> API1 - SV2 --> API3 - SV3 --> API2 - SV6 --> API4 -``` - -### Layer Breakdown - -#### Routes (`src/routes/`) - -- **`/`** - Welcome screen, creates new conversation -- **`/chat/[id]`** - Active chat interface -- **`/mcp-servers`** - MCP server management -- **`/search`** - Conversation search -- **`/settings`** - Settings (optional `[[section]]`) -- **`+layout.svelte`** - Sidebar, navigation, global initialization - -#### Components (`src/lib/components/`) - -Application components live in `app/`. The base components (the shadcn-svelte primitives: button, dialog, tooltip…), the design tokens, the fonts and the surfaces are Llama UI's, in [`ui/lib/ui`](../../lib/ui/README.md): the app imports them as `@ggml-org/llama-ui/ui/` and compiles them from source as part of its own build, with no install or build of the library (see `scripts/llama-ui.js`). - -**Chat Components** (`app/chat/`): - -| Component | Responsibility | -| ------------------ | --------------------------------------------------------------------------- | -| `ChatScreen/` | Main chat container, coordinates message list, input form, and attachments | -| `ChatForm/` | Message input textarea with file upload, paste handling, keyboard shortcuts | -| `ChatMessages/` | Message list with branch navigation, regenerate/continue/edit actions | -| `ChatAttachments/` | File attachment previews, drag-and-drop, PDF/image/audio handling | -| `ChatSettings/` | Parameter sliders (temperature, top-p, etc.) with server default sync | -| `ChatSidebar/` | Conversation list, search, import/export, navigation | - -**Dialog Components** (`app/dialogs/`): - -| Component | Responsibility | -| ------------------------------ | -------------------------------------------------------- | -| `DialogChatSettings` | Full-screen settings configuration | -| `DialogManageModels` | Models manager: browse, load, download and delete models | -| `DialogModelInformation` | Model details (context size, modalities, parallel slots) | -| `DialogChatAttachmentsPreview` | Full preview for images, PDFs (text or page view), code | -| `DialogConfirmation` | Generic confirmation for destructive actions | -| `DialogConversationRename` | Edit conversation title | - -**Server/Model Components** (`app/servers/`, `app/models/`): - -| Component | Responsibility | -| ----------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| `ServerErrorSplash` | Error display when server is unreachable | -| `ModelsSelector/` | Model dropdown and mobile sheet: favorites, loaded and local sections, rows windowed and folded into families | -| `ModelsManager/` | Models manager table and model pane: sections, filters, sorting, quants folded per repo, downloads | -| `ModelAvatar` | Org avatar of a model, with the quantizer badge; follows the family grouping of the list | -| `ModelId`, `ModelBadge` | Model name, aliases, tags, quantization and draft sidecar badges | -| `ModelContext`, `ModelCapabilities` | Context window and capability icons, filled from the Hub when it is enabled | - -**Shared UI Components** (`app/misc/`): - -| Component | Responsibility | -| ----------------------------------------- | ----------------------------------------------------------------------- | -| `MarkdownContent` | Markdown rendering with KaTeX, syntax highlighting, copy buttons | -| `SyntaxHighlightedCode` | Code blocks with language detection and highlighting | -| `ActionIcon`, `DropdownMenuActions` | Reusable action button and action menu | -| `BadgesModality`, `BadgeInfo` | Status and capability badges | -| `CollapsibleSection`, `CollapsibleRegion` | Section header with a trigger that expands in place | -| `GroupedList` | Grouped rows with foldable groups, per-group windows and show-more rows | -| `ScrollCarousel`, `TruncatedText` | Horizontal scroll strip and single-line truncation with tooltip | - -#### Hooks (`src/lib/hooks/`) - -Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state. - -| Hook | Responsibility | -| ------------------------------- | ----------------------------------------------------------------------------------------------- | -| `use-chat-screen-active-model` | Active model resolution + modality capability detection | -| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens | -| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge | -| `use-models-selector` | Model selector state: hidden models, family grouping, windowed rows, selection and load on pick | -| `use-tools-panel` | Tools panel state | -| `use-reasoning-menu` | Reasoning-effort menu state | -| `use-attachment-menu` | Attachment menu + modality flags | -| `use-draft-messages` | Per-chat draft message/files persistence | -| `use-chat-form-pickers` | Chat form pickers (commands, mentions) | -| `use-debounced-search` | Shared debounced async search for pickers | -| `use-picker-navigation` | Picker keyboard navigation | -| `use-chat-message-edit-context` | Message edit context (content + extras) | -| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | -| `use-chat-screen-file-upload` | File upload queue + capability validation | -| `use-stick-to-bottom` | Follows new content while the reader sits at the bottom of a scroller | -| `use-marquee-selection` | Shift+click / marquee range selection | -| `use-keyboard-shortcuts` | Global keyboard shortcuts | -| `use-settings-navigation` | Settings section navigation | -| `use-pwa` | PWA install/update + version mismatch detection | - -#### Stores (`src/lib/stores/`) - -Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns). - -| Store | Responsibility | -| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` | -| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` | -| `modelsStore` | Model list, selection, loading/unloading (ROUTER), recent picks, favorites, hidden models and list open state; composes `props`, `status` | -| `mcpStore` | MCP host role: multi-server lifecycle, tool routing, server icons; composes `health`, `resources` | -| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` | -| `serverStore` | Server connection state, `/props`, role detection, modalities | -| `settingsStore` | User preferences, theme, parameter sync with server defaults | -| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM | -| `permissionsStore` | Persisted tool permission grants | -| `contextStatsStore` | Context window usage for the active conversation | -| `draftMessagesStore` | Per-chat draft message/files | -| `deviceStore` | Browser environment signals (mobile, OS, theme) | -| `versionStore` | Build version information | - -#### Services (`src/lib/services/`) - -Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access. - -| Service | Responsibility | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion | -| `ModelsService` | `/models`, `/models/load`, `/models/unload` | -| `PropsService` | `/props`, `/props?model=` | -| `HuggingFaceService` | Hugging Face Hub metadata: model details, avatars, README and file tree, cached per repo and off unless the Hub setting is on | -| `DatabaseService` | IndexedDB operations via Dexie | -| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources | -| `ToolsService` | Server tool list/execute/stream (`/tools`) | -| `SandboxService` | Browser JS execution in a sandboxed worker | -| `ParameterSyncService` | Syncs settings with server defaults | -| `ConversationTransferService` | Conversation import/export JSONL + ZIP format | -| `MigrationService` | Non-destructive localStorage/IndexedDB migrations | -| `RouterService` | Dynamic route URL construction | - ---- - -## Data Flows - -### MODEL Mode (Single Model) +| Layer | Lives in | What it holds | +| -------------- | ------------------------- | --------------------------------------------------------------------------------------------------- | +| Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` | +| App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area | +| UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI | +| Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard | +| Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings | +| Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox | +| Storage + API | the browser, llama-server | IndexedDB and localStorage; llama-server's `/v1`, `/props`, `/models`, `/tools`, `/slots` | ```mermaid -sequenceDiagram - participant User - participant UI - participant Stores - participant DB as IndexedDB - participant API as llama-server - - Note over User,API: Initialization - UI->>Stores: initStores() (awaited by route loads) - Stores->>Stores: run migrations - Stores->>DB: load conversations (background) - Stores->>API: GET /props - API-->>Stores: server config - Stores->>API: GET /v1/models - API-->>Stores: single model (auto-selected) - - Note over User,API: Chat Flow - User->>UI: send message - Stores->>DB: save user message - Stores->>API: POST /v1/chat/completions (stream) - loop streaming - API-->>Stores: SSE chunks - Stores-->>UI: reactive update - end - Stores->>DB: save assistant message +flowchart LR + Routes --> Components["App components"] + Components --> Hooks + Components --> Stores + Hooks --> Stores + Stores --> Services + Services --> DB[("IndexedDB, localStorage")] + Services --> API["llama-server"] + Components -. "imports" .-> UI["Llama UI"] + Stores -. "reactive state" .-> Components ``` -### ROUTER Mode (Multi-Model) +### Sending a message ```mermaid sequenceDiagram participant User - participant UI participant Stores + participant DB as IndexedDB participant API as llama-server - Note over User,API: Initialization - Stores->>API: GET /props - API-->>Stores: {role: "router"} - Stores->>API: GET /models - API-->>Stores: models[] with status - - Note over User,API: Model Selection - User->>UI: select model - alt model not loaded - Stores->>API: POST /models/load - loop poll status - Stores->>API: GET /models - end - Stores->>API: GET /props?model=X + Note over Stores,API: on start + Stores->>API: GET /props, then the model list + User->>Stores: send a message + opt ROUTER mode, model not loaded + Stores->>API: POST /models/load, then poll GET /models end - Stores->>Stores: validate modalities - - Note over User,API: Chat Flow - Stores->>API: POST /v1/chat/completions {model: X} + Stores->>DB: save the user message + Stores->>API: POST /v1/chat/completions (stream) loop streaming - API-->>Stores: SSE chunks + model info + API-->>Stores: SSE chunks, shown as they arrive end -``` - ---- - -## Architectural Patterns - -### 1. Reactive State with Svelte 5 Runes - -All stores use Svelte 5's fine-grained reactivity: - -```typescript -// Store with reactive state -class ChatStore { - #isLoading = $state(false); - #currentResponse = $state(''); - - // Derived values auto-update - get isStreaming() { - return $derived(this.#isLoading && this.#currentResponse.length > 0); - } -} - -// Exported reactive accessors -export const isLoading = () => chatStore.isLoading; -export const currentResponse = () => chatStore.currentResponse; -``` - -### 2. Unidirectional Data Flow - -Data flows in one direction, making state predictable: - -```mermaid -flowchart LR - subgraph UI["UI Layer"] - A[User Action] --> B[Component] - end - - subgraph State["State Layer"] - B --> C[Store Method] - C --> D[State Update] - end - - subgraph IO["I/O Layer"] - C --> E[Service] - E --> F[API / IndexedDB] - F -.->|Response| D + opt the model called tools + Stores->>Stores: run them (after permission), send the results, stream again end - - D -->|Reactive| B -``` - -Components dispatch actions to stores, stores coordinate with services for I/O, and state updates reactively propagate back to the UI. - -### 3. Per-Conversation State - -Enables concurrent streaming across multiple conversations. Loading is tracked -per conversation by the activity ledger (`chatStore.activity`), while streaming -state and abort controllers live in per-conversation maps: - -```typescript -class ChatStore { - chatStreamingStates = new SvelteMap(); - abortControllers = new SvelteMap(); -} + Stores->>DB: save the assistant message ``` -### 4. Message Branching with Tree Structure +### Patterns worth knowing -Conversations are stored as a tree, not a linear list: +- **Stores** hold state as Svelte 5 runes (`$state`, `$derived`); larger ones live in folders and compose focused sub-stores. **Services** are stateless and do all I/O. +- **Per-conversation state**: streaming, loading and abort controllers are kept per conversation, so several chats can stream at once. +- **Messages form a tree**: each message knows its parent and children, and a conversation points at the branch tip it shows, so editing or regenerating branches without losing history. +- **Server role**: one codebase serves MODEL and ROUTER mode; `serverStore` reads the role from `/props`, and ROUTER-only UI shows only there. +- **Modalities**: attachments a model cannot read (images for a text-only model, say) are refused before sending, from the model's `/props`. +- **Storage**: conversations and messages in IndexedDB (Dexie), settings and model preferences in localStorage, server state and Hub details in memory only. -```typescript -interface DatabaseMessage { - id: string; - parent: string | null; // Points to parent message - children: string[]; // List of child message IDs - // ... -} +### Cross-origin isolation -interface DatabaseConversation { - currentNode: string; // Currently viewed branch tip - // ... -} -``` +llama-server serves the UI with `Cross-Origin-Embedder-Policy: require-corp` and `Cross-Origin-Opener-Policy: same-origin`, so every cross-origin asset needs the other origin's consent. Hugging Face Hub avatars and metadata are requested in CORS mode, which the Hub allows. MCP server icons and search-result favicons go through llama-server's `/cors-proxy` when it runs with `--ui-mcp-proxy`, and fall back to a bundled icon otherwise. -Navigation between branches updates `currentNode` without losing history. +## Build and embedding -### 5. Layered Service Architecture +`npm run build` generates the PWA icons, bundles the app with Vite and writes it to `dist/` (or to `LLAMA_UI_OUT_DIR`) as static files: `index.html`, one JS and one CSS bundle under `_app/`, the service worker, the web manifest and `build.json` with the version (`LLAMA_BUILD_NUMBER`, `b0000` locally). Paths are relative and routing uses the URL hash, so `dist` works from any path. -Stores handle state; services handle I/O: - -```text -┌─────────────────┐ -│ Stores │ Business logic, state management -├─────────────────┤ -│ Services │ API calls, database operations -├─────────────────┤ -│ Storage/API │ IndexedDB, LocalStorage, HTTP -└─────────────────┘ -``` - -### 6. Server Role Abstraction - -Single codebase handles both MODEL and ROUTER modes: - -```typescript -// serverStore.ts -get isRouterMode() { - return this.role === ServerRole.ROUTER; -} - -// Components conditionally render based on mode -{#if isRouterMode()} - -{/if} -``` - -### 7. Modality Validation - -Prevents sending attachments to incompatible models. The -`use-chat-screen-active-model` hook derives the active model's capabilities -from `modelsStore.props`: - -```typescript -// use-chat-screen-active-model hook -const hasVisionModality = $derived.by(() => modelsStore.props.modelSupportsVision(activeModelId)); -const hasAudioModality = $derived.by(() => modelsStore.props.modelSupportsAudio(activeModelId)); -``` - -### 8. Persistent Storage Strategy - -Data is persisted across sessions using two storage mechanisms: - -```mermaid -flowchart TB - subgraph Browser["Browser Storage"] - subgraph IDB["IndexedDB (Dexie)"] - C[Conversations] - M[Messages] - end - subgraph LS["LocalStorage"] - S[Settings Config] - O[User Overrides] - T[Theme Preference] - end - end - - subgraph Stores["Svelte Stores"] - CS[conversationsStore] --> C - CS --> M - SS[settingsStore] --> S - SS --> O - SS --> T - end -``` - -- **IndexedDB**: Conversations and messages (large, structured data) -- **LocalStorage**: Settings, user parameter overrides, theme, model favorites, recent picks, hidden models and model list open state (small key-value data) -- **Memory only**: Server props, model list (fetched fresh on each session), Hugging Face Hub details (cached per repo for the session) - -### 9. Remote Assets Under Cross-Origin Isolation - -llama-server serves the UI with `Cross-Origin-Embedder-Policy: require-corp` and `Cross-Origin-Opener-Policy: same-origin`, so a cross-origin subresource has to be allowed by the target origin. The UI handles that per asset kind: - -- **Hub avatars** are requested in CORS mode (`crossorigin="anonymous"`), which the Hub allows -- **Hub metadata** is fetched with `fetch()`, which is a CORS request by default, and only while the Hub setting is on -- **MCP server icons and web-search result favicons** have no CORS guarantee, so they go through the server's `/cors-proxy` when the user started llama-server with `--ui-mcp-proxy`; without it they fall back to the bundled glyph -- **Per-row Hub lookups** wait until the row is near the viewport (the `nearViewport` action), so a long model list does not fire one request per row on mount - -### 10. Nested Surfaces - -The sidebar stacks four surfaces, outermost first: - -| Layer | Example | Corners | -| ----- | ---------------------------------- | -------------------------------- | -| Page | the body background | none | -| Panel | the sidebar | `rounded-2xl` | -| Group | the sidebar's footer, inset in it | `inset-group-surface` | -| Row | Models, MCP or Settings in a group | `rounded-(--surface-row-radius)` | - -A row keeps its corners concentric with its group's: its radius is the group's less the padding between their edges, so the curves run parallel instead of pinching at the corners. `app.css` holds the three tokens (`--surface-group-radius`, which follows the design system's `--inset-group-radius`, `--surface-group-padding` and the derived `--surface-row-radius`); change the group's radius or padding and the rows follow. A group pads its rows with `p-(--surface-group-padding)`. `SidebarNavigationActions` rounds every row from the token, so the actions at the top of the sidebar (New chat, Search) match the footer's rows; a phone sizes its rows for a thumb instead. A selected row takes `selected-row-surface`, the group's edge and shade around a tint, where a hovered one is only tinted, so the two never look alike. The group's hairline border is left out of the sum. `tests/client/nested-surfaces.svelte.test.ts` checks the rule on the rendered footer. - ---- - -## Testing - -### Test Types - -| Type | Tool | Location | Command | -| ------------- | ------------------ | ---------------- | ------------------- | -| **Unit** | Vitest | `tests/unit/` | `npm run test:unit` | -| **UI/Visual** | Storybook + Vitest | `tests/stories/` | `npm run test:ui` | -| **E2E** | Playwright | `tests/e2e/` | `npm run test:e2e` | -| **Client** | Vitest | `tests/client/`. | `npm run test:unit` | - -### Running Tests - -```bash -# All tests -npm run test - -# Individual test suites -npm run test:e2e # End-to-end (requires llama-server) -npm run test:client # Client-side unit tests -npm run test:server # Server-side unit tests -npm run test:ui # Storybook visual tests -``` - -Locally, the browser tests (e2e, client and Storybook) run in your installed Google Chrome with its sandbox on, since Chrome updates itself and Playwright's bundled Chromium lags behind security fixes. CI (`CI` set) uses the bundled Chromium from `npx playwright install`. - -### Storybook Development - -```bash -npm run storybook # Start Storybook dev server on :6006 -npm run build-storybook # Build static Storybook -``` - -### Linting and Formatting - -```bash -npm run lint # Check code style -npm run format # Auto-format with Prettier -npm run check # TypeScript type checking -``` - ---- - -## Project Structure +llama-server embeds it: when llama-server is built, `scripts/ui-assets.cmake` takes `ui/apps/chat/dist` if it exists, runs `npm run build` if `LLAMA_BUILD_UI` is on, or downloads the release's prebuilt `dist.tar.gz` from Hugging Face, and compiles every file into the binary. The same `dist` is what `npx @ggml-org/llama-chat` serves. ```text ui/apps/chat/ ├── src/ -│ ├── lib/ -│ │ ├── components/ # Application components (base ones come from ui/lib/ui) -│ │ ├── hooks/ # Svelte hooks -│ │ ├── stores/ # State management -│ │ ├── services/ # API and database services -│ │ ├── types/ # TypeScript interfaces -│ │ └── utils/ # Utility functions +│ ├── lib/ # components, hooks, stores, services, types, utils, constants, enums │ ├── routes/ # SvelteKit routes -│ └── styles/ # Global styles -├── static/ # Static assets -├── tests/ # Test files -└── .storybook/ # Storybook configuration +│ └── service-worker/ # the PWA's service worker +├── bin/, cli/ # the npx llama-chat command and its server +├── alias/ # the llama-chat npm package, the short name +├── scripts/ # Vite plugins and build scripts +├── static/ # static assets +└── tests/ # unit, client and end-to-end tests, and the Storybook stories ``` ---- - -## Related Documentation - -- [llama.cpp Server README](../../../tools/server/README.md) - Full server documentation -- [Multimodal Documentation](../../../docs/multimodal.md) - Image and audio support -- [Function Calling](../../../docs/function-calling.md) - Tool use capabilities - ---- - ## Publishing to npm Two packages go out with the same version: `@ggml-org/llama-chat`, which carries the app and the CLI, and `llama-chat` (in `alias/`), which depends on it and only forwards to its CLI so that `npx llama-chat` works. A unit test (`tests/unit/cli.test.ts`) fails when their versions drift apart. @@ -851,3 +200,10 @@ npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:9931 ``` Build right before packing: the package carries whatever is in `dist`, and this folder's `.npmrc` sets `ignore-scripts`, so npm runs no lifecycle script such as `prepack` that could build it for you. + +## Related documentation + +- [llama.cpp server](../../../tools/server/README.md) +- [Multimodal input](../../../docs/multimodal.md) +- [Function calling](../../../docs/function-calling.md) +- [`DESIGN.md`](DESIGN.md), the design rules, and [`AGENTS.md`](AGENTS.md), the conventions for agents From 44193237ee1a5132069ce16a4b53bddb493be7dd Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 18 Nov 2026 12:18:34 +0100 Subject: [PATCH 10/10] ui: llama-chat serves only static files, without the API proxy The CLI forwarded every request it had no file for to llama-server, from loopback. With --host 0.0.0.0 that let any LAN client reach a llama-server started on 127.0.0.1 as a local request, tools included, which llama-server on its own refuses without an API key. The browser now talks to each llama-server directly, so llama-server's own rules apply. --server is gone, and anything that is not a file in dist is a 404, or a 405 for other methods. Assisted-by: Claude (Opus 5.5 Medium) --- ui/apps/chat/README.md | 40 ++---- ui/apps/chat/alias/README.md | 4 +- ui/apps/chat/cli/server.js | 192 +++++----------------------- ui/apps/chat/tests/unit/cli.test.ts | 126 +++--------------- 4 files changed, 70 insertions(+), 292 deletions(-) diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 80aa8c3da97f..5fa37286aadf 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -16,42 +16,30 @@ It works with a llama-server in either mode: **MODEL** (one model) or **ROUTER** ## Run it with npx -Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app on localhost and connects it to a llama-server you run yourself, or writes it to a folder for any static file server (see [Self-hosting](#self-hosting)); it brings no server and no models of its own. It needs Node.js 20 or later. +Llama Chat is published to npm as [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat), with [`llama-chat`](https://www.npmjs.com/package/llama-chat) as its short name. It serves the prebuilt app as static files on localhost, or writes them to a folder for any static file server (see [Self-hosting](#self-hosting)); it brings no server and no models of its own. It needs Node.js 20 or later. ```bash -npx @ggml-org/llama-chat # serve it on http://127.0.0.1:3000 and open the browser -npx llama-chat --server http://192.168.1.20:9931 # the same, by its short name, against another llama-server -npx llama-chat --host 0.0.0.0 --no-open # reachable from other devices -npx llama-chat build # write the static app to ./llama-chat instead +npx @ggml-org/llama-chat # serve it on http://127.0.0.1:3000 and open the browser +npx llama-chat --host 0.0.0.0 --no-open # the same, by its short name, reachable from other devices +npx llama-chat build # write the static app to ./llama-chat instead ``` -| Flag | Default | What it does | -| --------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------- | -| `--server ` | `http://127.0.0.1:9931` | The llama-server to connect to. A path is kept, for a server started with `--api-prefix` | -| `--port ` | `3000` | Fixed, so the origin, and the chats the browser stores under it, stay the same between runs | -| `--host ` | `127.0.0.1` | `0.0.0.0` to open it to the LAN | -| `--open` / `--no-open` | open | Opens the default browser once it listens | -| `-v`, `--version`, `-h`, `--help` | | As usual | +| Flag | Default | What it does | +| --------------------------------- | ----------- | ------------------------------------------------------------------------------------------- | +| `--port ` | `3000` | Fixed, so the origin, and the chats the browser stores under it, stay the same between runs | +| `--host ` | `127.0.0.1` | `0.0.0.0` to open it to the LAN | +| `--open` / `--no-open` | open | Opens the default browser once it listens | +| `-v`, `--version`, `-h`, `--help` | | As usual | The bare command is the documented form; `start` is accepted as an alias for it. `build` takes `--out ` (default `./llama-chat`) and `--force`, which lets it write into a folder that is not empty. -How it works: the app calls its server on its own origin (`./v1/...`, `./props`), as it does under llama-server. The CLI serves the files of `dist` and forwards every other request to llama-server as the server's own page, so the server needs no `--cors-origins`; requests from other pages keep their `Origin` and are refused as before, and on loopback it answers only to `localhost` names, against DNS rebinding. It sends llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good) and is plain Node.js with no dependencies. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot. +How it works: the CLI only serves the files of `dist`, with llama-server's headers for the UI (`index.html` cross-origin isolated, the PWA files revalidated, hashed assets cached for good), and forwards nothing. The browser talks to each llama-server you add in the app by URL directly, so llama-server's own rules apply: start it with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled, and with `--api-key` when it is reachable from the LAN. An `http://` page like this one can reach a plain `http://` llama-server on the LAN, which an `https://` page cannot. ## Self-hosting -The app is static and uses relative paths and a hash router, so any static host can serve it at any path. Two ways to host it: +The app is static and uses relative paths and a hash router, so any static host can serve it at any path. `npx @ggml-org/llama-chat build --out ` writes the files (as does `npm run build` here, to `dist`); serve them with any static file server, for example `npx http-server llama-chat -p 3000`. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Cross-origin isolation](#cross-origin-isolation)). -- **Run the CLI behind your reverse proxy**: `npx @ggml-org/llama-chat --host 0.0.0.0 --no-open --server `, then proxy your domain to its port. Long chat responses stream as server-sent events, so turn off response buffering in the proxy. -- **Serve the static files yourself**: `npx @ggml-org/llama-chat build --out ` writes them (as does `npm run build` here, to `dist`). Serve them from the same origin as the llama-server API: a path with a file answers from the folder, and every other path goes to llama-server. For example, with [http-server](https://www.npmjs.com/package/http-server), which forwards what it has no file for: - - ```bash - npx llama-chat build - npx http-server llama-chat -p 3000 -P http://127.0.0.1:9931 - ``` - - A plain file server sends the browser's `Origin` on to llama-server, so start llama-server with `--cors-origins http://127.0.0.1:3000` (the address you open) when it restricts origins, as it does with tools or MCP enabled. Send the headers listed above with `index.html` and the PWA files, so the app behaves as it does under llama-server (see [Cross-origin isolation](#cross-origin-isolation)). - -Until the app can add a server by URL, it reaches only the llama-server on its own origin: through the CLI, a reverse proxy, or llama-server itself (`--path dist`). +The app reaches the llama-server on its own origin, if there is one (llama-server serves it with `--path`, or your reverse proxy puts the API next to the files), and any other llama-server you add by URL, which needs `--cors-origins` for the app's address. ## Features @@ -196,7 +184,7 @@ To try a package before it is published: ```bash npm run build npm pack -npx ./ggml-org-llama-chat-.tgz --server http://127.0.0.1:9931 +npx ./ggml-org-llama-chat-.tgz ``` Build right before packing: the package carries whatever is in `dist`, and this folder's `.npmrc` sets `ignore-scripts`, so npm runs no lifecycle script such as `prepack` that could build it for you. diff --git a/ui/apps/chat/alias/README.md b/ui/apps/chat/alias/README.md index 9ac31f5ad10a..6c2e5c23b010 100644 --- a/ui/apps/chat/alias/README.md +++ b/ui/apps/chat/alias/README.md @@ -1,9 +1,9 @@ # llama-chat -The short name for [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat): `npx llama-chat` runs Llama Chat, the chat app for llama.cpp, on localhost and connects it to a llama-server. +The short name for [`@ggml-org/llama-chat`](https://www.npmjs.com/package/@ggml-org/llama-chat): `npx llama-chat` runs Llama Chat, the chat app for llama.cpp, on localhost; the browser talks to your llama-server directly. ```bash -npx llama-chat --server http://127.0.0.1:9931 +npx llama-chat ``` This package only forwards to `@ggml-org/llama-chat`, at the same version. See its [README](https://github.com/ggml-org/llama.cpp/tree/master/ui/apps/chat#readme) for the options. diff --git a/ui/apps/chat/cli/server.js b/ui/apps/chat/cli/server.js index 5b04852e1b3c..8ef0f557c3a4 100644 --- a/ui/apps/chat/cli/server.js +++ b/ui/apps/chat/cli/server.js @@ -1,19 +1,18 @@ /** - * `npx @ggml-org/llama-chat`: serves the prebuilt Llama Chat (`dist`) on localhost and forwards - * its API calls to a llama-server. `npx @ggml-org/llama-chat build` writes the same files to a - * folder instead, for any static file server. Node built-ins only, so the published package has - * no dependencies: no llama-server, no models, no Vite or SvelteKit. + * `npx @ggml-org/llama-chat`: serves the prebuilt Llama Chat (`dist`) as static files on + * localhost. `npx @ggml-org/llama-chat build` writes the same files to a folder instead, for any + * static file server. Node built-ins only, so the published package has no dependencies: no + * llama-server, no models, no Vite or SvelteKit. * - * The app calls the server on its own origin (`./v1/...`, `./props`, ...), as it does when - * llama-server serves it. So a request is answered from `dist` when a file there matches its - * path, and forwarded to the llama-server otherwise. + * It only serves files and forwards nothing: the browser calls each llama-server the app is + * pointed at directly, so llama-server's own rules (--cors-origins, --api-key) apply as they do + * to any other page. */ import { spawn } from 'node:child_process'; import { createReadStream, readFileSync } from 'node:fs'; import { cp, readdir, stat } from 'node:fs/promises'; import http from 'node:http'; -import https from 'node:https'; import { extname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { parseArgs } from 'node:util'; @@ -26,9 +25,7 @@ export const DEFAULTS = Object.freeze({ // the folder `build` writes to, in the current directory out: 'llama-chat', // fixed, so the origin, and the chats the browser stores under it, stay the same between runs - port: 3000, - // where llama-server listens when started without --host or --port - server: 'http://127.0.0.1:9931' + port: 3000 }); // The same caching llama-server applies to the UI it embeds (tools/server/server-http.cpp): @@ -64,29 +61,17 @@ const MIME_TYPES = { '.woff': 'font/woff', '.woff2': 'font/woff2' }; -// hop-by-hop headers, which a proxy does not pass on -const HOP_BY_HOP = [ - 'connection', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'proxy-connection', - 'te', - 'trailer', - 'transfer-encoding', - 'upgrade' -]; -const SERVE_OPTIONS = ['host', 'open', 'port', 'server']; +const SERVE_OPTIONS = ['host', 'open', 'port']; const BUILD_OPTIONS = ['force', 'out']; export const USAGE = `Usage: llama-chat [options] llama-chat build [--out ] [--force] -Serves Llama Chat on localhost and connects it to a llama-server, or, with -build, writes it to a folder as static files for any HTTP server. +Serves Llama Chat on localhost as static files, or, with build, writes them to +a folder for any HTTP server. The browser talks to llama-server directly: start +it with --cors-origins set to this page's address when it restricts origins. Options: - --server llama-server to connect to (default: ${DEFAULTS.server}) --port port to listen on (default: ${DEFAULTS.port}) --host address to listen on, 0.0.0.0 for the LAN (default: ${DEFAULTS.host}) --open open the browser once it listens (default) @@ -100,7 +85,6 @@ Build options: Examples: npx @ggml-org/llama-chat - npx @ggml-org/llama-chat --server http://192.168.1.20:9931 npx @ggml-org/llama-chat --host 0.0.0.0 --port 3000 --no-open npx @ggml-org/llama-chat build --out ./public`; @@ -113,7 +97,7 @@ export function packageVersion() { * Reads the command line. No verb serves the app, `start` means the same, and `build` writes it * to a folder. * @param {string[]} argv the arguments after the command - * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, server: URL, version: boolean }} + * @returns {{ command: 'build' | 'serve', force: boolean, help: boolean, host: string, open: boolean, out: string, port: number, version: boolean }} */ export function parseCliArgs(argv) { const { positionals, values } = parseArgs({ @@ -127,7 +111,6 @@ export function parseCliArgs(argv) { open: { type: 'boolean' }, out: { type: 'string' }, port: { type: 'string' }, - server: { type: 'string' }, version: { short: 'v', type: 'boolean' } }, strict: true @@ -163,25 +146,10 @@ export function parseCliArgs(argv) { open: values.open ?? true, out: values.out ?? DEFAULTS.out, port, - server: parseServerUrl(values.server ?? DEFAULTS.server), version: values.version ?? false }; } -/** - * @param {string} value a llama-server address, with or without `http://` - * @returns {URL} - */ -export function parseServerUrl(value) { - const url = new URL(/^[a-z][a-z\d+.-]*:\/\//i.test(value) ? value : `http://${value}`); - - if (url.protocol !== 'http:' && url.protocol !== 'https:') { - throw new Error(`invalid --server: ${value} (use an http:// or https:// address)`); - } - - return url; -} - /** * Maps a request path to a file under `root`, or null when it falls outside it. * @param {string} root @@ -246,132 +214,42 @@ function sendFile(req, res, file, pathname, stats) { } /** - * The host name in a Host header, lower case and without the port or IPv6 brackets. - * @param {string} host - */ -function hostName(host) { - const name = host.startsWith('[') ? host.slice(1, host.indexOf(']')) : host.split(':')[0]; - - return name.toLowerCase(); -} - -/** @param {string} address */ -function isLoopback(address) { - return /^(127\.|::1$|::ffff:127\.)/.test(address); -} - -/** - * DNS rebinding guard, as llama-server has: a page whose own name resolves to 127.0.0.1 reaches - * a loopback listener with that name in Host, so there Host must name loopback itself. - * @param {http.IncomingMessage} req - */ -function hostIsAllowed(req) { - if (!isLoopback(req.socket.localAddress ?? '')) return true; - - const name = hostName(req.headers.host ?? ''); - - return name === 'localhost' || name === '0.0.0.0' || name === '::' || isLoopback(name); -} - -/** - * Forwards a request to the llama-server. llama-server takes cross-origin browser requests only - * from its --cors-origins, so, like the Vite dev proxy, this calls it as its own page: with the - * Host of the server, and without the Origin of this page. Another page's Origin and - * Sec-Fetch-Site pass through, so llama-server still refuses what it would refuse directly. - * @param {http.IncomingMessage} req * @param {http.ServerResponse} res - * @param {URL} server + * @param {number} status + * @param {string} message */ -function proxy(req, res, server) { - const prefix = server.pathname.replace(/\/+$/, ''); - const target = new URL(prefix + (req.url ?? '/'), server.origin); - /** @type {http.OutgoingHttpHeaders} */ - const headers = { ...req.headers, host: target.host }; - const origin = req.headers.origin ?? ''; - - if (origin.slice(origin.indexOf('://') + 3).toLowerCase() === req.headers.host?.toLowerCase()) { - delete headers.origin; - } - - for (const name of HOP_BY_HOP) delete headers[name]; - - const upstream = (target.protocol === 'https:' ? https : http).request( - target, - { headers, method: req.method }, - (response) => { - const responseHeaders = { ...response.headers }; - - for (const name of HOP_BY_HOP) delete responseHeaders[name]; - - res.writeHead(response.statusCode ?? 502, responseHeaders); - response.pipe(res); - } - ); - - upstream.on('error', (error) => { - if (res.headersSent) { - res.destroy(error); - - return; - } - - res.writeHead(502, { 'Content-Type': 'application/json; charset=utf-8' }); - res.end( - JSON.stringify({ - error: { - code: 502, - message: `llama-chat could not reach the llama-server at ${server.href}: ${error.message}`, - type: 'unavailable_error' - } - }) - ); - }); - - // a stream the browser stops reading (Stop, a closed tab) stops the generation on the server - res.on('close', () => { - if (!res.writableFinished) upstream.destroy(); - }); - - req.pipe(upstream); +function sendError(res, status, message) { + res.writeHead(status, { 'Content-Type': 'text/plain; charset=utf-8' }); + res.end(`${message}\n`); } /** - * The HTTP server: files from `dist`, everything else from the llama-server. - * @param {{ distDir?: string, server: URL }} options + * The HTTP server: the files of `dist`, read-only. + * @param {{ distDir?: string }} [options] */ -export function createChatServer({ distDir = DIST_DIR, server }) { +export function createChatServer({ distDir = DIST_DIR } = {}) { const root = normalize(distDir); return http.createServer(async (req, res) => { const { pathname } = new URL(req.url ?? '/', 'http://localhost'); - if (!hostIsAllowed(req)) { - res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' }); - res.end( - JSON.stringify({ - error: { - code: 403, - message: `llama-chat refused the host name ${req.headers.host}, open it on localhost or 127.0.0.1`, - type: 'permission_error' - } - }) - ); + if (req.method !== 'GET' && req.method !== 'HEAD') { + res.setHeader('Allow', 'GET, HEAD'); + sendError(res, 405, 'Method Not Allowed'); return; } - if (req.method === 'GET' || req.method === 'HEAD') { - const file = resolveStatic(root, pathname); - const stats = file ? await stat(file).catch(() => null) : null; + const file = resolveStatic(root, pathname); + const stats = file ? await stat(file).catch(() => null) : null; - if (file && stats?.isFile()) { - sendFile(req, res, file, pathname, stats); + if (file && stats?.isFile()) { + sendFile(req, res, file, pathname, stats); - return; - } + return; } - proxy(req, res, server); + sendError(res, 404, 'Not Found'); }); } @@ -465,9 +343,7 @@ export async function main(argv) { const shown = relative(process.cwd(), target) || '.'; console.log(`Llama Chat ${packageVersion()} written to ${target}`); - console.log( - `Serve it on the same origin as the llama-server API, e.g.: npx http-server ${shown} -P ${DEFAULTS.server}` - ); + console.log(`Serve it with any static file server, e.g.: npx http-server ${shown}`); } catch (error) { console.error(`llama-chat: ${error instanceof Error ? error.message : error}`); process.exitCode = 1; @@ -476,7 +352,7 @@ export async function main(argv) { return; } - const server = createChatServer({ server: options.server }); + const server = createChatServer(); return new Promise((resolve) => { server.on('error', (error) => { @@ -497,7 +373,9 @@ export async function main(argv) { const url = `http://${shown.includes(':') ? `[${shown}]` : shown}:${address.port}/`; console.log(`Llama Chat ${packageVersion()} on ${url}`); - console.log(`Connected to the llama-server at ${options.server.href}`); + console.log( + `If llama-server restricts origins (tools or MCP on), start it with --cors-origins ${url.slice(0, -1)}` + ); if (options.open) openBrowser(url); diff --git a/ui/apps/chat/tests/unit/cli.test.ts b/ui/apps/chat/tests/unit/cli.test.ts index 1c8be0977cd8..fc573064ccbf 100644 --- a/ui/apps/chat/tests/unit/cli.test.ts +++ b/ui/apps/chat/tests/unit/cli.test.ts @@ -1,10 +1,4 @@ -import { - buildStatic, - createChatServer, - DEFAULTS, - parseCliArgs, - parseServerUrl -} from '../../cli/server.js'; +import { buildStatic, createChatServer, DEFAULTS, parseCliArgs } from '../../cli/server.js'; import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import http from 'node:http'; import type { AddressInfo } from 'node:net'; @@ -31,22 +25,12 @@ describe('llama-chat CLI arguments', () => { expect(options.host).toBe(DEFAULTS.host); expect(options.port).toBe(DEFAULTS.port); expect(options.open).toBe(true); - expect(options.server.href).toBe(`${DEFAULTS.server}/`); }); it('reads the documented flags', () => { - const options = parseCliArgs([ - '--server', - 'http://192.168.1.20:9931', - '--host', - '0.0.0.0', - '--port', - '3001', - '--no-open' - ]); + const options = parseCliArgs(['--host', '0.0.0.0', '--port', '3001', '--no-open']); expect(options).toMatchObject({ host: '0.0.0.0', open: false, port: 3001 }); - expect(options.server.href).toBe('http://192.168.1.20:9931/'); }); it('accepts start as a verb that changes nothing', () => { @@ -58,6 +42,8 @@ describe('llama-chat CLI arguments', () => { expect(() => parseCliArgs(['--nope'])).toThrow(); expect(() => parseCliArgs(['--port', 'abc'])).toThrow(/invalid --port/); expect(() => parseCliArgs(['--port', '70000'])).toThrow(/invalid --port/); + // it forwards nothing, so there is no llama-server to name + expect(() => parseCliArgs(['--server', 'http://127.0.0.1:9931'])).toThrow(); }); it('reads build with its own flags, and keeps serving flags out of it', () => { @@ -75,11 +61,6 @@ describe('llama-chat CLI arguments', () => { expect(() => parseCliArgs(['build', '--port', '3001'])).toThrow(/does not apply to build/); expect(() => parseCliArgs(['--out', 'public'])).toThrow(/does not apply to serving/); }); - - it('takes a server address without a scheme as http', () => { - expect(parseServerUrl('localhost:9931').href).toBe('http://localhost:9931/'); - expect(() => parseServerUrl('ftp://localhost')).toThrow(/invalid --server/); - }); }); describe('llama-chat packages', () => { @@ -148,14 +129,9 @@ describe('llama-chat build', () => { describe('llama-chat server', () => { let dist: string; - let upstream: http.Server; - let upstreamUrl: string; let chat: http.Server; let chatUrl: string; - const seen: { headers: http.IncomingHttpHeaders; method?: string; url?: string; body: string }[] = - []; - beforeAll(async () => { dist = await mkdtemp(join(tmpdir(), 'llama-chat-dist-')); await mkdir(join(dist, '_app', 'immutable'), { recursive: true }); @@ -164,25 +140,12 @@ describe('llama-chat server', () => { await writeFile(join(dist, 'service-worker.js'), 'self;'); await writeFile(join(dist, '_app', 'immutable', 'app.abc123.js'), 'export {};'); - upstream = http.createServer((req, res) => { - let body = ''; - - req.on('data', (chunk) => (body += chunk)); - req.on('end', () => { - seen.push({ body, headers: req.headers, method: req.method, url: req.url }); - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ path: req.url })); - }); - }); - upstreamUrl = await listen(upstream); - - chat = createChatServer({ distDir: dist, server: new URL(`${upstreamUrl}/prefix/`) }); + chat = createChatServer({ distDir: dist }); chatUrl = await listen(chat); }); afterAll(async () => { await close(chat); - await close(upstream); await rm(dist, { force: true, recursive: true }); }); @@ -218,83 +181,32 @@ describe('llama-chat server', () => { expect(worker.headers.get('cache-control')).toBe('no-cache'); }); - it('forwards API calls to the llama-server as its own page', async () => { - const res = await fetch(`${chatUrl}/v1/chat/completions?x=1`, { - body: '{"messages":[]}', - headers: { 'Content-Type': 'application/json', Origin: chatUrl }, - method: 'POST' - }); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ path: '/prefix/v1/chat/completions?x=1' }); - - const request = seen.at(-1)!; + it('forwards nothing: API paths are not found and other methods are refused', async () => { + const props = await fetch(`${chatUrl}/props`); - expect(request.method).toBe('POST'); - expect(request.body).toBe('{"messages":[]}'); - expect(request.headers.origin).toBeUndefined(); - expect(request.headers.host).toBe(new URL(upstreamUrl).host); - }); + expect(props.status).toBe(404); - it("passes another page's Origin through, so llama-server can refuse it", async () => { - await fetch(`${chatUrl}/props`, { - headers: { Origin: 'http://evil.example', 'Sec-Fetch-Site': 'cross-site' } + const post = await fetch(`${chatUrl}/v1/chat/completions`, { + body: '{"messages":[]}', + headers: { 'Content-Type': 'application/json' }, + method: 'POST' }); - const request = seen.at(-1)!; - - expect(request.headers.origin).toBe('http://evil.example'); - expect(request.headers['sec-fetch-site']).toBe('cross-site'); - }); - - it('refuses other host names on loopback, against DNS rebinding', async () => { - // fetch() sets Host itself, so the request goes out through http.request - const status = (host: string) => - new Promise((resolve, reject) => { - const { port } = new URL(chatUrl); - - http - .request({ headers: { host }, host: '127.0.0.1', path: '/props', port }, (res) => { - res.resume(); - resolve(res.statusCode ?? 0); - }) - .on('error', reject) - .end(); - }); - - expect(await status('rebind.example:3000')).toBe(403); - expect(await status('localhost:3000')).toBe(200); - expect(await status('[::1]:3000')).toBe(200); + expect(post.status).toBe(405); + expect(post.headers.get('allow')).toBe('GET, HEAD'); }); it('does not serve files from outside dist', async () => { // fetch() would resolve the dots itself, so the raw path goes out through http.get - const path = '/..%2f..%2f..%2fetc%2fpasswd'; - const body = await new Promise((resolve, reject) => { + const status = await new Promise((resolve, reject) => { http - .get(`${chatUrl}${path}`, (res) => { - let data = ''; - - res.on('data', (chunk) => (data += chunk)); - res.on('end', () => resolve(data)); + .get(`${chatUrl}/..%2f..%2f..%2fetc%2fpasswd`, (res) => { + res.resume(); + resolve(res.statusCode ?? 0); }) .on('error', reject); }); - expect(JSON.parse(body)).toEqual({ path: `/prefix${path}` }); - }); - - it('answers 502 when the llama-server is down', async () => { - const down = createChatServer({ distDir: dist, server: new URL('http://127.0.0.1:9') }); - const url = await listen(down); - - try { - const res = await fetch(`${url}/props`); - - expect(res.status).toBe(502); - expect((await res.json()).error.message).toMatch(/could not reach the llama-server/); - } finally { - await close(down); - } + expect(status).toBe(404); }); });