Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions common/arg.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3422,6 +3422,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.ui_mcp_proxy = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY"));
add_opt(common_arg(
{"--ui-storage"},
{"--no-ui-storage"},
"experimental: keep UI conversations and settings on disk instead of only in the browser - anyone who can reach the server can read them (default: disabled)",
[](common_params & params, bool value) {
params.ui_storage = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_STORAGE"));
add_opt(common_arg(
{"--ui-storage-dir"}, "PATH",
"directory for UI storage, implies --ui-storage (default: llama.cpp/ui in the user data directory)",
[](common_params & params, const std::string & value) {
params.ui_storage = true;
params.ui_storage_dir = value;
}
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_STORAGE_DIR"));
add_opt(common_arg(
{"--tools"}, "TOOL1,TOOL2,...",
"experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
Expand Down
18 changes: 18 additions & 0 deletions common/common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1014,6 +1014,24 @@ std::filesystem::path fs_get_config_directory() {
return config_directory / "llama.cpp";
}

std::filesystem::path fs_get_data_directory() {
std::filesystem::path data_directory;
#if defined(_WIN32)
data_directory = common_get_path_from_env("APPDATA");
if (data_directory.empty()) {
throw std::runtime_error("Failed to find %APPDATA% directory");
}
#elif defined(__APPLE__)
data_directory = get_home_directory() / "Library/Application Support";
#else
data_directory = common_get_path_from_env("XDG_DATA_HOME");
if (data_directory.empty()) {
data_directory = get_home_directory() / ".local/share";
}
#endif
return data_directory / "llama.cpp";
}

std::filesystem::path fs_get_cache_file(const std::string & filename) {
GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos);
const std::filesystem::path cache_directory = fs_get_cache_directory();
Expand Down
3 changes: 3 additions & 0 deletions common/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,8 @@ struct common_params {
bool ui = true;
bool ui_mcp_proxy = false;
std::string ui_config_json;
bool ui_storage = false; // keep UI conversations and settings on disk
std::string ui_storage_dir; // empty: fs_get_data_directory() / "ui"

// "advanced" endpoints are disabled by default for better security
bool endpoint_slots = true;
Expand Down Expand Up @@ -960,6 +962,7 @@ inline bool common_create_directories(const std::filesystem::path & path, std::e
std::filesystem::path fs_get_cache_directory();
std::filesystem::path fs_get_cache_file(const std::string & filename);
std::filesystem::path fs_get_config_directory();
std::filesystem::path fs_get_data_directory();

void fs_write_atomic(const std::filesystem::path & path, const std::string & data);

Expand Down
2 changes: 2 additions & 0 deletions tools/server/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ add_library(${TARGET} STATIC
server-stream.h
server-tools.cpp
server-tools.h
server-storage.cpp
server-storage.h
server-mcp.cpp
server-mcp.h
server-schema.cpp
Expand Down
2 changes: 2 additions & 0 deletions tools/server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
| `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
| `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
| `--ui-storage, --no-ui-storage` | experimental: keep UI conversations and settings on disk instead of only in the browser - anyone who can reach the server can read them (default: disabled)<br/>(env: LLAMA_ARG_UI_STORAGE) |
| `--ui-storage-dir PATH` | directory for UI storage, implies --ui-storage (default: llama.cpp/ui in the user data directory)<br/>(env: LLAMA_ARG_UI_STORAGE_DIR) |
| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
| `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/> 'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/> 'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/> 'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
| `--allow-unauthenticated-tools` | allow server tools, MCP servers or MCP proxy on a non-loopback address without an API key - only for trusted networks (default: disabled)<br/>(env: LLAMA_ARG_ALLOW_UNAUTHENTICATED_TOOLS) |
Expand Down
1 change: 1 addition & 0 deletions tools/server/server-context.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5630,6 +5630,7 @@ static json get_res_props(const server_context_meta & meta, const common_params
{ "build_info", meta.build_info },
{ "is_sleeping", is_sleeping },
{ "cors_proxy_enabled", params.ui_mcp_proxy },
{ "ui_storage", params.ui_storage },
};

if (!params.server_connect_code.empty()) {
Expand Down
4 changes: 4 additions & 0 deletions tools/server/server-models.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,9 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
preset.unset_option("LLAMA_ARG_CONNECT_CODE");
preset.unset_option("LLAMA_ARG_MCP_SERVERS_CONFIG");
preset.unset_option("LLAMA_ARG_MCP_SERVERS_JSON");
// UI storage is served by the router only
preset.unset_option("LLAMA_ARG_UI_STORAGE");
preset.unset_option("LLAMA_ARG_UI_STORAGE_DIR");
if (unset_model_args) {
preset.unset_option("LLAMA_ARG_MODEL");
preset.unset_option("LLAMA_ARG_SHARD");
Expand Down Expand Up @@ -2073,6 +2076,7 @@ void server_models_routes::init_routes() {
{"ui_settings", ui_settings},
{"build_info", std::string(llama_build_info())},
{"cors_proxy_enabled", params.ui_mcp_proxy},
{"ui_storage", params.ui_storage},
};
if (!params.server_connect_code.empty()) {
props["connect_code"] = params.server_connect_code;
Expand Down
227 changes: 227 additions & 0 deletions tools/server/server-storage.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
#include "server-storage.h"

#include <fstream>
#include <sstream>

namespace fs = std::filesystem;

static json read_json_file(const fs::path & path) {
std::ifstream f(path, std::ios::binary);
if (!f) {
throw std::runtime_error("failed to open file: " + path.string());
}
std::stringstream ss;
ss << f.rdbuf();
return json::parse(ss.str());
}

// ids and record kinds come from the UI (uuids, table names), anything else could escape the storage directory
static void validate_id(const std::string & id, const char * what = "conversation id") {
if (id.empty() || id.size() > 64) {
throw std::invalid_argument(std::string("invalid ") + what);
}
for (char c : id) {
if (!std::isalnum((unsigned char) c) && c != '-' && c != '_') {
throw std::invalid_argument(std::string("invalid ") + what);
}
}
}

static server_http_res_ptr res_json(int status, const json & data) {
auto res = std::make_unique<server_http_res>();
res->status = status;
res->data = safe_json_to_str(data);
return res;
}

fs::path server_storage::conversation_path(const std::string & id) const {
return dir / "conversations" / fs::u8path(id + ".json");
}

fs::path server_storage::records_path(const std::string & kind) const {
return dir / "records" / fs::u8path(kind + ".json");
}

void server_storage::load_index() {
const fs::path index_path = dir / "index.json";

if (fs::exists(index_path)) {
try {
index = read_json_file(index_path);
if (index.is_object()) {
return;
}
} catch (const std::exception & e) {
SRV_WRN("failed to read %s, rebuilding it: %s\n", index_path.string().c_str(), e.what());
}
}

index = json::object();
for (const auto & entry : fs::directory_iterator(dir / "conversations")) {
if (!entry.is_regular_file() || entry.path().extension() != ".json") {
continue;
}
try {
const json doc = read_json_file(entry.path());
const std::string id = doc.at("conv").at("id").get<std::string>();
index[id] = { { "rev", doc.value("rev", 0) }, { "conv", doc.at("conv") } };
} catch (const std::exception & e) {
SRV_WRN("skipping unreadable conversation %s: %s\n", entry.path().string().c_str(), e.what());
}
}
save_index();
}

void server_storage::save_index() const {
fs_write_atomic(dir / "index.json", safe_json_to_str(index));
}

void server_storage::setup(const fs::path & storage_dir) {
dir = storage_dir;

std::error_code ec;
common_create_directories(dir / "conversations", ec);
if (!ec) {
common_create_directories(dir / "records", ec);
}
if (ec) {
throw std::runtime_error("failed to create UI storage directory: " + dir.string());
}
load_index();

SRV_INF("UI storage in %s (%zu conversations)\n", dir.string().c_str(), index.size());

handle_list = [this](const server_http_req &) {
std::lock_guard<std::mutex> lock(mtx);
json list = json::array();
for (const auto & [id, entry] : index.items()) {
list.push_back(entry);
}
return res_json(200, list);
};

handle_get = [this](const server_http_req & req) {
const std::string id = req.get_param("id");
validate_id(id);

std::lock_guard<std::mutex> lock(mtx);
if (!index.contains(id)) {
return res_json(404, { { "error", format_error_response("conversation not found", ERROR_TYPE_NOT_FOUND) } });
}
return res_json(200, read_json_file(conversation_path(id)));
};

// the body carries the revision it was based on, 0 for a new conversation
handle_put = [this](const server_http_req & req) {
const std::string id = req.get_param("id");
validate_id(id);

const json body = json::parse(req.body);
const int rev = body.at("rev").get<int>();
const json & conv = body.at("conv");
const json & messages = body.at("messages");
if (!conv.is_object() || conv.value("id", "") != id || !messages.is_array()) {
throw std::invalid_argument("expected { rev, conv, messages } with conv.id matching the path");
}

std::lock_guard<std::mutex> lock(mtx);
const int current = index.contains(id) ? index[id].value("rev", 0) : 0;
if (rev != current) {
json error = { { "code", 409 }, { "message", "conversation was modified by another client" }, { "type", "conflict_error" } };
return res_json(409, { { "error", error }, { "rev", current } });
}

const json doc = { { "rev", current + 1 }, { "conv", conv }, { "messages", messages } };
fs_write_atomic(conversation_path(id), safe_json_to_str(doc));
index[id] = { { "rev", current + 1 }, { "conv", conv } };
save_index();

return res_json(200, { { "rev", current + 1 } });
};

handle_delete = [this](const server_http_req & req) {
const std::string id = req.get_param("id");
validate_id(id);

std::lock_guard<std::mutex> lock(mtx);
std::error_code ec;
fs::remove(conversation_path(id), ec);
if (index.contains(id)) {
index.erase(id);
save_index();
}
return res_json(200, json::object());
};

handle_get_settings = [this](const server_http_req &) {
std::lock_guard<std::mutex> lock(mtx);
const fs::path path = dir / "settings.json";
return res_json(200, fs::exists(path) ? read_json_file(path) : json::object());
};

// merges per key, so two clients changing different settings do not overwrite each other
handle_put_settings = [this](const server_http_req & req) {
const json body = json::parse(req.body);
const json set = body.value("set", json::object());
const json remove = body.value("remove", json::array());
if (!set.is_object() || !remove.is_array()) {
throw std::invalid_argument("expected { set: { key: value }, remove: [key] }");
}

std::lock_guard<std::mutex> lock(mtx);
const fs::path path = dir / "settings.json";
json settings = fs::exists(path) ? read_json_file(path) : json::object();
for (const auto & [key, value] : set.items()) {
if (!value.is_string()) {
throw std::invalid_argument("setting values must be strings");
}
settings[key] = value;
}
for (const auto & key : remove) {
settings.erase(key.get<std::string>());
}
fs_write_atomic(path, safe_json_to_str(settings));

return res_json(200, settings);
};

handle_get_records = [this](const server_http_req & req) {
const std::string kind = req.get_param("kind");
validate_id(kind, "record kind");

std::lock_guard<std::mutex> lock(mtx);
const fs::path path = records_path(kind);
return res_json(200, fs::exists(path) ? read_json_file(path) : json::object());
};

// records are small and written whole, merged per id like settings
handle_put_records = [this](const server_http_req & req) {
const std::string kind = req.get_param("kind");
validate_id(kind, "record kind");

const json body = json::parse(req.body);
const json set = body.value("set", json::object());
const json remove = body.value("remove", json::array());
if (!set.is_object() || !remove.is_array()) {
throw std::invalid_argument("expected { set: { id: record }, remove: [id] }");
}
for (const auto & [id, record] : set.items()) {
if (!record.is_object() || record.value("id", "") != id) {
throw std::invalid_argument("records must be objects whose id matches their key");
}
}

std::lock_guard<std::mutex> lock(mtx);
const fs::path path = records_path(kind);
json records = fs::exists(path) ? read_json_file(path) : json::object();
for (const auto & [id, record] : set.items()) {
records[id] = record;
}
for (const auto & id : remove) {
records.erase(id.get<std::string>());
}
fs_write_atomic(path, safe_json_to_str(records));

return res_json(200, json::object());
};
}
39 changes: 39 additions & 0 deletions tools/server/server-storage.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#pragma once

#include "server-common.h"
#include "server-http.h"

#include <filesystem>
#include <mutex>

// UI storage (--ui-storage): the Web UI's conversations and settings kept as JSON files on disk
// the server does not interpret the documents, it only tracks a revision per conversation so
// concurrent writers (tabs, browsers) get a 409 instead of silently overwriting each other
//
// layout:
// <dir>/conversations/<id>.json { "rev": n, "conv": {...}, "messages": [...] }
// <dir>/index.json { "<id>": { "rev": n, "conv": {...} } }, rebuilt when missing
// <dir>/settings.json { "<key>": "<value>" }
// <dir>/records/<kind>.json { "<id>": {...} }, groups, projects, prompts, ...
struct server_storage {
void setup(const std::filesystem::path & dir);

server_http_context::handler_t handle_list; // GET /storage/conversations
server_http_context::handler_t handle_get; // GET /storage/conversations/:id
server_http_context::handler_t handle_put; // POST /storage/conversations/:id
server_http_context::handler_t handle_delete; // DELETE /storage/conversations/:id
server_http_context::handler_t handle_get_settings; // GET /storage/settings
server_http_context::handler_t handle_put_settings; // POST /storage/settings
server_http_context::handler_t handle_get_records; // GET /storage/records/:kind
server_http_context::handler_t handle_put_records; // POST /storage/records/:kind

private:
std::filesystem::path dir;
std::mutex mtx;
json index = json::object();

std::filesystem::path conversation_path(const std::string & id) const;
std::filesystem::path records_path(const std::string & kind) const;
void load_index();
void save_index() const;
};
Loading
Loading