From 61374a7e6866a605251e202d630f7432b4b11e8d Mon Sep 17 00:00:00 2001 From: imneov Date: Tue, 13 Oct 2026 19:46:28 -0700 Subject: [PATCH 1/2] feat(web): load deploy route options from the gateway The deploy form's bundled catalog and independent placeholders drifted from the running engine. Proxy the public route catalog, render its host defaults and separate runtime state from editable overrides, and keep legacy metadata without stale defaults. Cover missing-gateway SPA fallthrough, old-plan route identity and live desktop picker behavior. Keep copied Swiss code and mobile sizing unchanged. Fixes #52 --- docs/console-design.md | 24 ++- docs/development/route-options-catalog.md | 61 ++++++ hack/check-route-catalog-live.mjs | 74 +++++++ internal/gateway/gateway.go | 4 +- internal/server/proxy.go | 16 ++ internal/server/route_options.go | 110 +++++++++++ internal/server/route_options_test.go | 186 ++++++++++++++++++ .../inferences/components/DeployForm.test.tsx | 73 +++++++ .../inferences/components/DeployForm.tsx | 181 ++++++++++------- .../inferences/components/DeploySheet.tsx | 14 ++ web/src/modules/inferences/locales/en-US.json | 68 ++++++- web/src/modules/inferences/locales/zh-CN.json | 68 ++++++- .../modules/inferences/route-options.test.ts | 64 ++++++ web/src/modules/inferences/route-options.ts | 87 ++++++++ 14 files changed, 950 insertions(+), 80 deletions(-) create mode 100644 docs/development/route-options-catalog.md create mode 100644 hack/check-route-catalog-live.mjs create mode 100644 internal/server/route_options.go create mode 100644 internal/server/route_options_test.go create mode 100644 web/src/modules/inferences/components/DeployForm.test.tsx create mode 100644 web/src/modules/inferences/route-options.test.ts create mode 100644 web/src/modules/inferences/route-options.ts diff --git a/docs/console-design.md b/docs/console-design.md index f442650..d79c5fd 100644 --- a/docs/console-design.md +++ b/docs/console-design.md @@ -307,14 +307,34 @@ Not there yet: - **Search on the services list.** swissd pages its list (at most 100 a page) and takes no query; filtering one page on the client would miss the others. It needs `?q=` on swissd's `/api/deployments`. -- **swiss's panels and the route directive texts are English.** They are swiss's, - and swiss's pages have no Chinese yet. +- **swiss's panels are English.** They are swiss's, and swiss's pages have no Chinese yet. Route option labels are translated by inferences; untranslated help and new options use the engine's text. - **Cluster resources needs a recent swissd.** An older one (0.5.6, for one) has no `/objects`: the tab says so, and the install track reports the route and scaler as unreadable rather than absent. - **Upgrade picks chart versions through `/api/catalog/:name/chart-versions`**, which swissd has from 0.6.2 (swiss #13). +### Deploy route options + +The deploy and upgrade form reads the running openresty's catalog instead of treating bundled defaults as deployment defaults (console #52; [agreed design](development/route-options-catalog.md)). + +```text +DeploySheet -> GET /api/llm/route-options[?route=] + -> configured gateway -> //_route_opts + -> common grouped rows + advanced picker + runtime state +``` + +| Concern | Decision | +|---|---| +| Authorization | Console login and `get backends` for `llm`, the same grant as the inference proxy | +| New deployment | Probe existing ConfigMap routes for a catalog; never show their per-route values as the new deployment's | +| Upgrade | Query the recorded route, display `value` and `set` separately from saved overrides | +| Defaults | Live `fallback` supplies placeholders; no copied values are submitted automatically | +| Legacy or empty gateway | 404 uses bundled metadata with all fallback values stripped | +| Other failures | Surface a catalog warning; do not claim a deployment default | +| Text | Per-key locale translations where present; new options retain engine text | +| Ownership | Inferences adapts the Swiss metadata without editing the copied upstream module | + ### Workload reads (`/api/k8s/...`) Events and logs are Kubernetes data, not swiss's, so console serves them itself diff --git a/docs/development/route-options-catalog.md b/docs/development/route-options-catalog.md new file mode 100644 index 0000000..71e6d24 --- /dev/null +++ b/docs/development/route-options-catalog.md @@ -0,0 +1,61 @@ +# Route Options Catalog + +Design agreed from console issue #52 on 2026-10-12. + +```text +DeploySheet -> GET /api/llm/route-options[?route=] + -> configured gateway -> //_route_opts + -> common grouped rows + advanced picker + runtime state +``` + +| Concern | Decision | +|---|---| +| API | Each configured backend exposes GET `/route-options`; gateway backends select resolved ConfigMap routes, while static URLs already name a route | +| Authorization | Console login and `get backends` on the configured backend name | +| Request | Optional existing route; reject invalid names and unknown routes; without one try available routes until a catalog answers | +| Credentials | Do not forward the browser session, cookies or gateway API key to the public engine endpoint | +| Compatibility | 404 for an unsupported engine, missing backend, empty gateway or unavailable route; bundled metadata has no fallback values | +| Failures | Permission, transport and malformed response failures are surfaced, not silently treated as legacy engines | +| Form state | Fetching metadata never changes stored overrides; `value` and `set` describe the current runtime separately | +| Runtime identity | Only show runtime values for the canonical requested upgrade route, including the release-name fallback for older plans | +| Named controls | Preserve chart and SLO semantics, use live placeholders, and avoid duplicate rows | +| Advanced | Engine internals remain behind the add-setting picker, which respects available viewport height | +| i18n | Translate known keys; unknown keys use engine text | +| Swiss | Adapt bundled metadata in inferences without editing the copied Swiss module | + +No CRD, chart value or configuration schema changes. Validation remains in the engine. + +## Regression Coverage + +| Scenario | Signal | +|---|---| +| SPA with no gateway backend | Catalog API returns 404 JSON instead of 200 HTML | +| Host defaults differ from bundled values | Render the live defaults, never copied placeholders | +| Existing route differs from a new deployment | Do not leak the probed route's runtime state into a new form | +| Upgrade without explicit route/serviceId overrides | Use the same canonical route for fetching and display | +| Unknown engine option | Render engine metadata without a console release | +| Optional boolean override | Preserve unset, true and false distinctly | +| Empty advanced setting | Omit the setting rather than sending its displayed default | +| Catalog response | Bound response size, reject malformed data and do not follow redirects | + +## Real Engine Verification + +Verification used a released `4pdosc/llm-openresty:0.1.26` image with an isolated catalog-only route and the existing preview application services. The route is not a working inference pool or a business deployment. + +| Check | Result | +|---|---| +| Engine catalog | 16 common directives and 24 advanced directives | +| Different host defaults | TPS fallback 73 and floor fraction 0.61 appear as placeholders | +| Separate route state | TPS value 81 with set=true is distinct from fallback 73 | +| Proxy | Preserves the engine response; known routes return 200 and unknown routes 404 | +| Browser compose | Returns 200; live defaults and empty advanced settings remain omitted | +| Native mouse interaction | Advanced option is pointer-reachable; selection keeps the form open and the value unset | +| Local checks | Go suite, frontend suite, typecheck and production build pass | + +The standalone browser check requires an auth-disabled preview, a real configured engine and the GLM sample model: + +```sh +CONSOLE_URL=http://127.0.0.1:8080 node hack/check-route-catalog-live.mjs +``` + +It checks live placeholders, viewport containment, pointer hit testing and native selection. Acceptance is desktop-only. No install/apply operation was performed, and existing stored release history was not rewritten. diff --git a/hack/check-route-catalog-live.mjs b/hack/check-route-catalog-live.mjs new file mode 100644 index 0000000..e7094a6 --- /dev/null +++ b/hack/check-route-catalog-live.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; + +const base = process.env.CONSOLE_URL; +assert.ok(base, "Set CONSOLE_URL to an auth-disabled preview with a real engine catalog"); +const session = "route-catalog-live-check"; +function browser(...args) { + const result = JSON.parse(execFileSync("agent-browser", ["--session", session, "--json", ...args], { encoding: "utf8", timeout: 40_000 })); + assert.equal(result.success, true, result.error); + return result.data; +} +const evaluate = (script) => browser("eval", script).result; + +const response = await fetch(`${base}/api/llm/route-options`); +assert.equal(response.status, 200, "A real engine catalog must be configured"); +const catalog = await response.json(); +const options = new Map([...catalog.directives, ...catalog.advanced].map((d) => [d.key, d])); + +try { + browser("open", base); + browser("set", "viewport", process.env.VIEWPORT_WIDTH ?? "1280", process.env.VIEWPORT_HEIGHT ?? "720"); + evaluate('localStorage.setItem("rise-locale", "en-US")'); + browser("open", `${base}/inferences/catalog/glm5.1?catalog=default`); + browser("wait", "--text", "Deploy"); + const defaults = evaluate(`(async () => { + const frame = () => new Promise(requestAnimationFrame); + [...document.querySelectorAll("button")].filter((x) => x.textContent.trim() === "Deploy").at(-1).click(); + for (let i = 0; i < 300 && !document.querySelector("#deploy-sec-routing-section"); i++) await frame(); + const section = document.querySelector("#deploy-sec-routing-section"); + if (!section) throw new Error("Deploy routing section did not render"); + const sheet = section.closest('[data-slot="sheet-content"]'); + const sheetRect = sheet.getBoundingClientRect(); + if (sheetRect.left < 0 || sheetRect.right > innerWidth) throw new Error("Deploy sheet exceeds viewport width"); + section.querySelector("button").click(); + await frame(); + const tpsLabel = [...document.querySelectorAll("label")].find((x) => x.textContent.includes("TPS fallback")); + const tps = document.getElementById(tpsLabel.htmlFor); + const floor = document.querySelector('input[aria-label="Adaptive floor fraction"]'); + const trigger = section.querySelector('button[aria-haspopup="menu"]'); + trigger.scrollIntoView({ block: "center" }); + for (let i = 0; i < 12; i++) await frame(); + trigger.click(); + for (let i = 0; i < 12; i++) await frame(); + const item = [...document.querySelectorAll('[role="menuitem"]')].find((x) => x.textContent.includes("rt_limit_factor")); + const menu = item.closest('[role="menu"]'); + const menuRect = menu.getBoundingClientRect(); + if (menuRect.top < 0 || menuRect.bottom > innerHeight) throw new Error("Advanced menu exceeds available viewport height"); + const initial = item.getBoundingClientRect(); + menu.scrollTop += initial.top - menuRect.top - (menu.clientHeight - initial.height) / 2; + for (let i = 0; i < 12; i++) await frame(); + const rect = item.getBoundingClientRect(); + const x = rect.left + rect.width / 2; + const y = rect.top + rect.height / 2; + if (document.elementFromPoint(x, y)?.closest('[role="menuitem"]') !== item) throw new Error("Advanced item is not pointer reachable"); + return { tps: tps.placeholder, floor: floor.placeholder, tpsValue: tps.value, floorValue: floor.value, x, y }; + })()`); + assert.equal(defaults.tps, options.get("tps_limit_tps").fallback); + assert.equal(defaults.floor, options.get("adaptive_cc_min_frac").fallback); + assert.equal(defaults.tpsValue, ""); + assert.equal(defaults.floorValue, ""); + browser("mouse", "move", String(Math.round(defaults.x)), String(Math.round(defaults.y))); + browser("mouse", "down"); + browser("mouse", "up"); + const selected = evaluate(`(() => { + const input = document.querySelector('input[id$="-rt_limit_factor"]'); + if (!input) throw new Error("Native pointer selection did not add the option"); + return { value: input.value, placeholder: input.placeholder }; + })()`); + assert.equal(selected.value, ""); + assert.equal(selected.placeholder, options.get("rt_limit_factor").fallback); + console.log(JSON.stringify({ realCatalog: true, route: catalog.route, hostDefaults: { tps: defaults.tps, floor: defaults.floor }, viewportAndNativePointer: true, unsetAdvancedPreserved: true })); +} finally { + browser("close"); +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 487ebea..a99669c 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -52,6 +52,8 @@ type Entry struct { KeySource string } +var ErrNoRoutes = fmt.Errorf("no session_route_.conf entry") + // refresh is how long a resolved entrypoint is reused. Long enough that a busy // Playground does not read the same two ConfigMaps per request, short enough that // a deploy shows up without touching the console. @@ -203,7 +205,7 @@ func (r *Resolver) routes(ctx context.Context, p profile, profileRef string) (ro } } if len(aggregates)+len(plain) == 0 { - return nil, "", fmt.Errorf("route configmap %s has no session_route_.conf entry; is this the entrypoint's shared configmap?", ref) + return nil, "", fmt.Errorf("route configmap %s has %w; is this the entrypoint's shared configmap?", ref, ErrNoRoutes) } sort.Strings(aggregates) sort.Strings(plain) diff --git a/internal/server/proxy.go b/internal/server/proxy.go index e986885..a5607bb 100644 --- a/internal/server/proxy.go +++ b/internal/server/proxy.go @@ -56,12 +56,18 @@ var unresolved = &backendTarget{url: url.URL{Scheme: "http", Host: "backend-with // is skipped and logged rather than taking the server down. func (s *Server) mountBackends(mux *http.ServeMux) { targets := s.backendTargets() + catalogMounted := false for _, b := range s.cfg.Backends { resolve, ok := targets[b.Name] if !ok { continue } proxy := s.backendProxy(b) + prefix := strings.TrimSuffix(b.Prefix, "/") + catalogMounted = catalogMounted || prefix == "/api/llm" + mux.HandleFunc(prefix+"/route-options", func(w http.ResponseWriter, r *http.Request) { + s.handleRouteOptions(w, r, b, resolve) + }) mux.HandleFunc(strings.TrimSuffix(b.Prefix, "/")+"/", func(w http.ResponseWriter, r *http.Request) { if !s.authorize(w, r, backendVerb(r.Method), backendResource, b.Name) { return @@ -82,6 +88,16 @@ func (s *Server) mountBackends(mux *http.ServeMux) { proxy.ServeHTTP(w, r.WithContext(withTarget(r.Context(), target))) }) } + if !catalogMounted { + mux.HandleFunc("/api/llm/route-options", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", "GET") + writeError(w, http.StatusMethodNotAllowed, "GET required") + return + } + writeError(w, http.StatusNotFound, "no gateway backend configured") + }) + } } // backendTargets resolves each backend once for the process: a gateway resolver diff --git a/internal/server/route_options.go b/internal/server/route_options.go new file mode 100644 index 0000000..6b074a7 --- /dev/null +++ b/internal/server/route_options.go @@ -0,0 +1,110 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "slices" + "strings" + + "github.com/modelsphere/console/internal/config" + "github.com/modelsphere/console/internal/gateway" +) + +var routeOptionName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) + +func (s *Server) handleRouteOptions(w http.ResponseWriter, r *http.Request, b config.Backend, resolve targetResolver) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", "GET") + writeError(w, http.StatusMethodNotAllowed, "GET required") + return + } + if !s.authorize(w, r, "get", backendResource, b.Name) { + return + } + route := r.URL.Query().Get("route") + if route != "" && !routeOptionName.MatchString(route) { + writeError(w, http.StatusBadRequest, "invalid route name") + return + } + ctx, cancel := context.WithTimeout(r.Context(), probeTimeout) + defer cancel() + target, err := resolve(ctx) + if err != nil { + if errors.Is(err, gateway.ErrNoRoutes) { + writeError(w, http.StatusNotFound, "no route catalog available") + return + } + writeError(w, http.StatusBadGateway, fmt.Sprintf("backend %s: %v", b.Name, err)) + return + } + routes := target.routes + if b.Gateway == nil { + if route != "" && route != strings.TrimPrefix(target.url.Path, "/") { + writeError(w, http.StatusNotFound, "route not served by backend") + return + } + routes = []string{""} + } else if route != "" { + if !slices.Contains(routes, route) { + writeError(w, http.StatusNotFound, "route not served by backend") + return + } + routes = []string{route} + } + client := &http.Client{Timeout: probeTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + failed := false + for _, name := range routes { + body, status, err := readRouteOptions(ctx, client, routeURL(target, name)+"/_route_opts") + if status == http.StatusNotFound { + continue + } + if err != nil || status != http.StatusOK { + failed = true + continue + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(body) + return + } + if failed { + writeError(w, http.StatusBadGateway, "gateway route catalog unavailable or invalid") + return + } + writeError(w, http.StatusNotFound, "no route catalog available") +} + +func readRouteOptions(ctx context.Context, client *http.Client, endpoint string) ([]byte, int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, 0, err + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return nil, 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, nil + } + const max = 1 << 20 + body, err := io.ReadAll(io.LimitReader(resp.Body, max+1)) + if err != nil { + return nil, resp.StatusCode, err + } + var catalog struct { + Route string `json:"route"` + Directives []json.RawMessage `json:"directives"` + Advanced []json.RawMessage `json:"advanced"` + } + if len(body) > max || json.Unmarshal(body, &catalog) != nil || catalog.Route == "" || catalog.Directives == nil || catalog.Advanced == nil { + return nil, resp.StatusCode, fmt.Errorf("invalid route catalog") + } + return body, resp.StatusCode, nil +} diff --git a/internal/server/route_options_test.go b/internal/server/route_options_test.go new file mode 100644 index 0000000..a1bdeb2 --- /dev/null +++ b/internal/server/route_options_test.go @@ -0,0 +1,186 @@ +package server + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "testing/fstest" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/modelsphere/console/internal/config" +) + +const routeOptionsFixture = `{"route":"model","directives":[{"key":"tps_limit_tps","label":"Decode rate threshold","group":"Thresholds","kind":"number","fallback":"73","detail":"Deployment default","value":81,"set":true}],"advanced":[],"peers":{"count":3},"adaptive_cc_derived":{"_":{"static_max":200,"min":80,"init":80}}}` + +func TestRouteOptionsWithoutGatewayDoesNotServeSPA(t *testing.T) { + cfg := &config.Config{} + cfg.Server.Auth.Disabled = true + srv := testServerWithConfig(t, cfg) + srv.SetWeb(fstest.MapFS{"index.html": {Data: []byte("SPA")}}) + rec := do(srv.Handler(), "GET", "/api/llm/route-options", "", "") + if rec.Code != http.StatusNotFound || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") { + t.Fatalf("missing gateway must select legacy fallback, got %d %s: %s", rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) + } +} + +func TestRouteOptionsLiveCatalog(t *testing.T) { + var paths []string + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" || r.Header.Get("X-Remote-User") != "" { + t.Error("credentials or identity forwarded to public route catalog") + } + if r.URL.Path == "/video/_route_opts" { + http.NotFound(w, r) + return + } + if r.URL.Path != "/model/_route_opts" { + t.Errorf("path = %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer up.Close() + base, _ := url.Parse(up.URL) + srv := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", Gateway: &config.Gateway{ConfigMap: "llm/routes", Service: "llm/openresty"}}}}) + srv.targetsOnce.Do(func() { + srv.targets = map[string]targetResolver{"llm": func(context.Context) (*backendTarget, error) { + return &backendTarget{url: *base, routes: []string{"video", "model"}, key: "secret", header: "Authorization"}, nil + }} + }) + h := srv.Handler() + admin := login(t, h, "admin", "admin-pw") + for _, path := range []string{"/api/llm/route-options", "/api/llm/route-options?route=model"} { + rec := do(h, "GET", path, admin, "") + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"fallback":"73"`) || !strings.Contains(rec.Body.String(), `"value":81`) { + t.Fatalf("live catalog = %d %s", rec.Code, rec.Body.String()) + } + } + if strings.Join(paths, ",") != "/video/_route_opts,/model/_route_opts,/model/_route_opts" { + t.Fatalf("paths = %v", paths) + } +} + +func TestRouteOptionsEmptyGatewayAndResolutionFailure(t *testing.T) { + for _, tc := range []struct { + ref string + want int + }{ + {"llm/openresty-conf", http.StatusNotFound}, + {"llm/missing-conf", http.StatusBadGateway}, + } { + t.Run(tc.ref, func(t *testing.T) { + objects := gatewayObjects(t) + objects[1].(*unstructured.Unstructured).Object["data"] = map[string]any{} + cfg := &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", Gateway: &config.Gateway{ConfigMap: tc.ref, Service: "llm/openresty"}}}} + h := testServerWithConfig(t, cfg, objects...).Handler() + admin := login(t, h, "admin", "admin-pw") + rec := do(h, "GET", "/api/llm/route-options", admin, "") + if rec.Code != tc.want { + t.Fatalf("catalog = %d, want %d: %s", rec.Code, tc.want, rec.Body.String()) + } + }) + } +} + +func TestRouteOptionsReadGrantAndStaticRoute(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/model/_route_opts" { + t.Errorf("path = %s", r.URL.Path) + } + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer up.Close() + h := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", URL: up.URL + "/model"}}}).Handler() + admin, bob := login(t, h, "admin", "admin-pw"), login(t, h, "bob", "bob-pw") + for _, tc := range []struct{ path, body string }{ + {"/api/iam/roles", `{"name":"catalog-reader","rules":[{"verbs":["get"],"apiGroups":["iam.theriseunion.io"],"resources":["backends"],"resourceNames":["llm"]}]}`}, + {"/api/iam/rolebindings", `{"name":"bob-catalog-reader","role":"catalog-reader","subjects":[{"kind":"User","name":"bob"}]}`}, + } { + if rec := do(h, "POST", tc.path, admin, tc.body); rec.Code != http.StatusCreated { + t.Fatalf("grant: %d %s", rec.Code, rec.Body.String()) + } + } + if rec := do(h, "GET", "/api/llm/route-options?route=model", bob, ""); rec.Code != http.StatusOK || rec.Body.String() != routeOptionsFixture { + t.Fatalf("read grant: %d %s", rec.Code, rec.Body.String()) + } + if rec := do(h, "GET", "/api/llm/route-options?route=another", bob, ""); rec.Code != http.StatusNotFound { + t.Fatalf("unknown route: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestRouteOptionsDoesNotFollowRedirects(t *testing.T) { + redirected := 0 + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + redirected++ + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer other.Close() + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, other.URL, http.StatusFound) })) + defer up.Close() + h := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", URL: up.URL + "/model"}}}).Handler() + admin := login(t, h, "admin", "admin-pw") + rec := do(h, "GET", "/api/llm/route-options", admin, "") + if rec.Code != http.StatusBadGateway || redirected != 0 { + t.Fatalf("redirect: %d, followed %d times", rec.Code, redirected) + } +} + +func TestReadRouteOptionsBoundsAndShape(t *testing.T) { + for _, body := range []string{"{}", `{"route":"model","directives":null,"advanced":[]}`, `{"route":"model","directives":[],"advanced":{}}`, strings.Repeat(" ", 1<<20) + routeOptionsFixture} { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })) + _, _, err := readRouteOptions(t.Context(), up.Client(), up.URL) + up.Close() + if err == nil { + t.Fatal("invalid or oversized catalog accepted") + } + } +} + +func TestRouteOptionsAccessAndFailures(t *testing.T) { + for _, status := range []int{200, 404, 401, 500} { + t.Run(http.StatusText(status), func(t *testing.T) { + hits := 0 + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.WriteHeader(status) + if status == 200 { + _, _ = w.Write([]byte("not json")) + } + })) + defer up.Close() + srv := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", URL: up.URL + "/model"}}}) + h := srv.Handler() + admin, bob := login(t, h, "admin", "admin-pw"), login(t, h, "bob", "bob-pw") + for _, tc := range []struct { + method, path, token string + want int + }{ + {"GET", "/api/llm/route-options", "", 401}, + {"GET", "/api/llm/route-options", bob, 403}, + {"POST", "/api/llm/route-options", admin, 405}, + {"GET", "/api/llm/route-options?route=..%2Felsewhere", admin, 400}, + } { + rec := do(h, tc.method, tc.path, tc.token, "") + if rec.Code != tc.want { + t.Fatalf("%s %s = %d, want %d", tc.method, tc.path, rec.Code, tc.want) + } + } + if hits != 0 { + t.Fatalf("rejected requests reached upstream: %d", hits) + } + rec := do(h, "GET", "/api/llm/route-options", admin, "") + want := http.StatusBadGateway + if status == 404 { + want = 404 + } + if rec.Code != want { + t.Fatalf("status = %d, want %d: %s", rec.Code, want, rec.Body.String()) + } + }) + } +} diff --git a/web/src/modules/inferences/components/DeployForm.test.tsx b/web/src/modules/inferences/components/DeployForm.test.tsx new file mode 100644 index 0000000..99fef84 --- /dev/null +++ b/web/src/modules/inferences/components/DeployForm.test.tsx @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { DeployForm } from "./DeployForm"; +import { EMPTY, planRequest, type Form } from "@swiss/components/DeploySettings"; +import type { RouteOptions } from "@/modules/inferences/route-options"; + +const catalog: RouteOptions = { + route: "model", + directives: [], + advanced: [{ key: "rt_limit_factor", label: "Runtime limit factor", group: "Capacity & failover", kind: "number", fallback: "7.25", detail: "Scales the pool limit.", needsAdaptive: false, value: 9, set: true }], +}; + +function render(routeOptions?: RouteOptions, upgrade = false, form: Form = { ...EMPTY, serviceId: "model", nginxExtras: [{ key: "rt_limit_factor", value: "" }] }, catalogRoute?: string) { + const props = { + form, upgrade, + onChange: () => {}, submitted: false, target: null, defaults: {}, gpuOptions: [], + expanded: { basic: false, resources: false, routing: true, advanced: false }, + onExpand: () => {}, idPrefix: "test-", routeOptions, catalogRoute, + }; + return renderToStaticMarkup(); +} + +describe("deploy route catalog", () => { + it("shows the running engine's default instead of the bundled default", () => { + expect(render(catalog).includes('placeholder="7.25"')).toBe(true); + }); + + it("does not show stale bundled defaults without a live catalog", () => { + expect(render().includes('placeholder="1"')).toBe(false); + expect(render().includes("0.4")).toBe(false); + }); + + it("does not show another route's runtime values on a new deployment or an unmatched upgrade", () => { + expect(render(catalog).includes("当前：9")).toBe(false); + expect(render({ ...catalog, route: "another" }, true).includes("当前：9")).toBe(false); + expect(render(catalog, true).includes("当前：9")).toBe(true); + expect(render(catalog, true).includes("路由配置")).toBe(true); + }); + + it("shows runtime state for a release fallback when the old plan has no explicit route identity", () => { + const legacyForm = { ...EMPTY, nginxExtras: [{ key: "rt_limit_factor", value: "" }] }; + expect(render(catalog, true, legacyForm, "model").includes("当前：9")).toBe(true); + expect(render(catalog, true, legacyForm, "another").includes("当前：9")).toBe(false); + expect(render(catalog, false, legacyForm, "model").includes("当前：9")).toBe(false); + }); + + it("renders new engine directives without a console release and leaves advanced defaults out of the default view", () => { + const live: RouteOptions = { ...catalog, directives: [{ key: "future_knob", label: "New engine knob", group: "New engine group", kind: "text", fallback: "dynamic default", detail: "New engine help", value: "current", set: false }] }; + const html = render(live); + expect(html.includes("New engine knob")).toBe(true); + expect(html.includes("New engine help")).toBe(true); + expect(html.includes('placeholder="dynamic default"')).toBe(true); + expect(html.includes("New engine group")).toBe(true); + const emptyForm = { ...EMPTY, serviceId: "model" }; + expect(render(live, false, emptyForm).includes('placeholder="7.25"')).toBe(false); + }); + + it("uses host defaults for named fields and does not mutate stored overrides when showing runtime state", () => { + const form = { ...EMPTY, serviceId: "model", route: "model", nginxExtras: [{ key: "rt_limit_factor", value: "2" }] }; + const before = planRequest(form, { model: "model" }); + const live: RouteOptions = { ...catalog, directives: [ + { key: "tps_limit_tps", label: "TPS", group: "Thresholds", kind: "number", fallback: "73", detail: "Engine threshold help", value: 81, set: true }, + { key: "adaptive_cc_min_frac", label: "Floor fraction", group: "Adaptive concurrency", kind: "number", fallback: "0.61", detail: "Engine fraction help", value: 0.61, set: false }, + ] }; + const html = render(live, true, form); + expect(html.includes('placeholder="73"')).toBe(true); + expect(html.includes('placeholder="0.61"')).toBe(true); + expect(html.includes('value="2"')).toBe(true); + expect(html.includes("当前：81")).toBe(true); + expect(html.includes("引擎默认值或派生值")).toBe(true); + expect(planRequest(form, { model: "model" })).toEqual(before); + }); +}); diff --git a/web/src/modules/inferences/components/DeployForm.tsx b/web/src/modules/inferences/components/DeployForm.tsx index 9bf3edd..495be85 100644 --- a/web/src/modules/inferences/components/DeployForm.tsx +++ b/web/src/modules/inferences/components/DeployForm.tsx @@ -1,4 +1,4 @@ -import { useState, type ReactNode } from "react"; +import { useId, useState, type ReactNode } from "react"; import { Badge, Button, @@ -23,9 +23,10 @@ import { import { Plus, X } from "lucide-react"; import type { ClusterInfo } from "@swiss/lib/api"; import { effective, type Form, type Toleration } from "@swiss/components/DeploySettings"; -import { DIRECTIVES, DIRECTIVES_BY_KEY, MAX_VALUES, directiveGroups, rejectKey } from "@swiss/lib/routeDirectives"; +import { MAX_VALUES, rejectKey } from "@swiss/lib/routeDirectives"; +import { FALLBACK_ROUTE_OPTIONS, routeDirectives, runtimeValue, type RouteDirective, type RouteOptions } from "@/modules/inferences/route-options"; import { useT } from "@/modules/inferences/i18n"; -import { NAMED_VALUES, extraValue, fracOutOfRange, initialValue, serviceIdError, type Section } from "@/modules/inferences/deploy-lib"; +import { NAMED_VALUES, extraValue, fracOutOfRange, serviceIdError, type Section } from "@/modules/inferences/deploy-lib"; const EFFECTS = ["", "NoSchedule", "PreferNoSchedule", "NoExecute"]; const digits = (v: string) => v.replace(/\D/g, ""); @@ -42,12 +43,22 @@ export interface DeployFormProps { expanded: Record; onExpand: (s: Section, open: boolean) => void; idPrefix: string; + routeOptions?: RouteOptions | null; + catalogRoute?: string; } -export function DeployForm({ form, onChange, upgrade, submitted, target, cluster, defaults, gpuOptions, expanded, onExpand, idPrefix }: DeployFormProps) { +export function DeployForm({ form, onChange, upgrade, submitted, target, cluster, defaults, gpuOptions, expanded, onExpand, idPrefix, routeOptions, catalogRoute }: DeployFormProps) { const t = useT(); const on = effective(form); const f = (k: string) => t(`deploy.fields.${k}`); + const byKey = new Map(routeDirectives(routeOptions).map((d) => [d.key, d])); + const showRuntime = !!upgrade && routeOptions?.route === (catalogRoute ?? (form.route || form.serviceId)); + const label = (key: string, fallback: string) => directiveText(t, byKey.get(key), "label") || f(fallback); + const hint = (key: string, fallback: string) => { + const d = byKey.get(key); + const detail = routeOptions ? directiveText(t, d, "detail") || f(fallback) : f(fallback); + return showRuntime && d?.set !== undefined ? `${detail} · ${runtimeText(t, d)}` : detail; + }; const text = (k: keyof Form) => ({ value: form[k] as string, onChange: (e: React.ChangeEvent) => onChange({ [k]: e.target.value }) }); const count = (k: "replicaCount" | "scalerMin" | "scalerMax") => ({ value: form[k], inputMode: "numeric" as const, onChange: (e: React.ChangeEvent) => onChange({ [k]: digits(e.target.value) }) }); const sw = (k: keyof Form, label: string) => onChange({ [k]: v })} />; @@ -151,7 +162,7 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster > onChange(v ? { modelRoute: true } : { modelRoute: false, route: "" })} /> - + {sw("exposeRoutedPeer", f("exposeRoutedPeer"))} @@ -170,22 +181,22 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster > {sw("cart", f("cart"))} - + {sw("slo", f("slo"))}
- - + + - - + +
@@ -193,13 +204,13 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster value={form.adaptiveCcMinFrac} onChange={(e) => onChange({ adaptiveCcMinFrac: e.target.value })} inputMode="decimal" - placeholder={`${f("adaptiveCcMinFrac")} · 0.4`} + placeholder={byKey.get("adaptive_cc_min_frac")?.fallback || undefined} aria-label={f("adaptiveCcMinFrac")} aria-invalid={fracOutOfRange(form.adaptiveCcMinFrac)} disabled={!!extraValue(form, "adaptive_cc_min")} />

- {fracOutOfRange(form.adaptiveCcMinFrac) ? f("fracRange") : f("adaptiveCcMinFracHint")} + {fracOutOfRange(form.adaptiveCcMinFrac) ? f("fracRange") : hint("adaptive_cc_min_frac", "adaptiveCcMinFracHint")}

} @@ -260,78 +271,98 @@ function Tolerations({ rows, onChange }: { rows: Toleration[]; onChange: (v: Tol ); } -function RouteExtras({ form, onChange }: { form: Form; onChange: (patch: Partial
) => void }) { +function directiveText(t: ReturnType, d: RouteDirective | undefined, field: "label" | "detail"): string { + return d ? t(`deploy.routeOptions.options.${d.key}.${field}`, { defaultValue: d[field] }) : ""; +} + +function runtimeText(t: ReturnType, d: RouteDirective): string { + return t("deploy.routeOptions.current", { value: runtimeValue(d.value), source: t(d.set ? "deploy.routeOptions.configured" : "deploy.routeOptions.inherited") }); +} + +export function RouteExtras({ form, onChange, options, showRuntime = false }: { form: Form; onChange: (patch: Partial) => void; options?: RouteOptions | null; showRuntime?: boolean }) { const t = useT(); const f = (k: string, v?: Record) => t(`deploy.fields.${k}`, v); const [custom, setCustom] = useState(""); - const taken = form.nginxExtras.map((r) => r.key); + const controlsId = useId(); + const catalog = options ?? FALLBACK_ROUTE_OPTIONS; + const byKey = new Map(routeDirectives(catalog).map((d) => [d.key, d])); + const taken = [...NAMED_VALUES, ...form.nginxExtras.map((r) => r.key)]; const used = NAMED_VALUES.size + form.nginxExtras.length; const full = used >= MAX_VALUES; const customError = custom.trim() ? rejectKey(custom, taken) : ""; const add = (key: string) => { - onChange({ nginxExtras: [...form.nginxExtras, { key, value: initialValue(key) }] }); + onChange({ nginxExtras: [...form.nginxExtras, { key, value: "" }] }); setCustom(""); }; - const set = (i: number, value: string) => onChange({ nginxExtras: form.nginxExtras.map((r, j) => (j === i ? { ...r, value } : r)) }); - + const set = (key: string, value: string) => { + const exists = form.nginxExtras.some((r) => r.key === key); + if (!exists && full) return; + onChange({ nginxExtras: exists ? form.nginxExtras.map((r) => r.key === key ? { ...r, value } : r) : [...form.nginxExtras, { key, value }] }); + }; + const remove = (key: string) => onChange({ nginxExtras: form.nginxExtras.filter((r) => r.key !== key) }); + const row = (key: string, d?: RouteDirective) => { + const value = form.nginxExtras.find((r) => r.key === key)?.value ?? ""; + const idle = !!d?.needsAdaptive && !form.adaptiveCc; + const label = directiveText(t, d, "label") || key; + const fraction = key.endsWith("_frac") && fracOutOfRange(value); + return ( +
+
+
{d && {directiveText(t, d, "detail")}}
+ {idle &&
{f("idle")}
} + {showRuntime && d?.set !== undefined &&
{runtimeText(t, d)}
} +
+
+ {d?.kind === "boolean" ? ( + set(key, v === "__inherit" ? "" : v)} disabled={full && !taken.includes(key)} + options={[{ value: "__inherit", label: d.fallback ? f("without", { fallback: d.fallback }) : t("deploy.routeOptions.inherit") }, { value: "true", label: t("deploy.routeOptions.on") }, { value: "false", label: t("deploy.routeOptions.off") }]} /> + ) : ( + set(key, e.target.value)} inputMode={d?.kind === "number" ? "decimal" : undefined} placeholder={d?.fallback || undefined} aria-label={label} aria-invalid={!!fraction} disabled={full && !taken.includes(key)} /> + )} + {fraction &&

{f("fracRange")}

} +
+ +
+ ); + }; + const primary = catalog.directives.filter((d) => !NAMED_VALUES.has(d.key)); + const primaryKeys = new Set(primary.map((d) => d.key)); return ( - - {form.nginxExtras.map((row, i) => { - const d = DIRECTIVES_BY_KEY.get(row.key); - const idle = !!d?.needsAdaptive && !form.adaptiveCc; - return ( -
-
-
{d?.label ?? row.key}
-
{idle ? f("idle") : f("without", { fallback: d?.fallback ?? "-" })}
-
- {d?.kind === "boolean" ? ( - set(i, v ? "true" : "false")} /> - ) : ( - set(i, e.target.value)} inputMode={d?.kind === "number" ? "decimal" : undefined} placeholder={d?.fallback} aria-label={d?.label ?? row.key} /> - )} - -
- ); - })} -
- - }> - {f("extrasAdd")} - - - {directiveGroups().map((group) => { - const rows = DIRECTIVES.filter((d) => d.group === group && !taken.includes(d.key)); - if (!rows.length) return null; - return ( - - {group} - {rows.map((d) => ( - add(d.key)} className="flex-col items-start gap-0.5"> - - {d.label} {d.key} - - {f("without", { fallback: d.fallback })} - - ))} - - - ); - })} - - - setCustom(e.target.value)} placeholder={f("extrasCustom")} aria-label={f("extrasCustom")} className="h-8 w-48" disabled={full} /> - - - {full ? f("extrasFull", { max: MAX_VALUES }) : f("extrasUsed", { used, max: MAX_VALUES })} - +
+ {[...new Set(primary.map((d) => d.group))].map((group) => ( +
+
{t(`deploy.routeOptions.groups.${group}`, { defaultValue: group })}
+ {primary.filter((d) => d.group === group).map((d) => row(d.key, d))}
- {customError &&

{customError}

} - + ))} + {showRuntime &&
+ {catalog.directives.filter((d) => NAMED_VALUES.has(d.key)).map((d) =>
{directiveText(t, d, "label")}: {runtimeText(t, d)}
)} +
} + {form.nginxExtras.filter((r) => !primaryKeys.has(r.key)).map((r) => row(r.key, byKey.get(r.key)))} +
+ + }> {t("deploy.routeOptions.advanced")} + + {[...new Set(catalog.advanced.map((d) => d.group))].map((group) => { + const rows = catalog.advanced.filter((d) => d.group === group && !taken.includes(d.key)); + if (!rows.length) return null; + return + {t(`deploy.routeOptions.groups.${group}`, { defaultValue: group })} + {rows.map((d) => add(d.key)} className="flex-col items-start gap-0.5"> + {directiveText(t, d, "label")} {d.key} + {d.fallback && {f("without", { fallback: d.fallback })}} + )} + + ; + })} + + + setCustom(e.target.value)} placeholder={f("extrasCustom")} aria-label={f("extrasCustom")} className="h-8 w-48" disabled={full} /> + + {full ? f("extrasFull", { max: MAX_VALUES }) : f("extrasUsed", { used, max: MAX_VALUES })} +
+ {customError &&

{customError}

} +
); } diff --git a/web/src/modules/inferences/components/DeploySheet.tsx b/web/src/modules/inferences/components/DeploySheet.tsx index bf248d2..3340200 100644 --- a/web/src/modules/inferences/components/DeploySheet.tsx +++ b/web/src/modules/inferences/components/DeploySheet.tsx @@ -37,6 +37,7 @@ import { useT } from "@/modules/inferences/i18n"; import { detailPath } from "@/modules/inferences/lib"; import { SECTIONS, chartChoice, defaultServiceId, gpuOptions, pipelineState, serviceIdError, whatMoves, type Section } from "@/modules/inferences/deploy-lib"; import { DeployForm } from "@/modules/inferences/components/DeployForm"; +import { readRouteOptions } from "@/modules/inferences/route-options"; export type DeployTarget = | { kind: "deploy"; model: string; catalog: string; version?: string; variant?: string } @@ -105,6 +106,14 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( const initial = useMemo(() => (upgrade ? (current.data ? formFromPlan(current.data) : EMPTY) : { ...EMPTY, serviceId: defaultServiceId(target.model) }), [upgrade, current.data, target]); const [form, setForm] = useState(initial); + const catalogRoute = upgrade && current.data ? initial.route || initial.serviceId || target.release : undefined; + const routeOptions = useQuery({ + queryKey: ["route-options", catalogRoute ?? ""], + queryFn: ({ signal }) => readRouteOptions(catalogRoute, signal), + enabled: !upgrade || !!current.data, + staleTime: 30_000, + retry: false, + }); useEffect(() => setForm(initial), [initial]); const dirty = useDirty({ form, version, variant, moveTo, chartVersion }, { form: initial, version: upgrade ? "" : (target.version ?? ""), variant: upgrade ? "" : (target.variant ?? ""), moveTo: "", chartVersion: "" }); @@ -270,6 +279,9 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( {t("deploy.review.readOnly")} )} + {routeOptions.error && ( + {t("deploy.routeOptions.error", { error: routeOptions.error.message })} + )} {(error || compose.error || apply.error) && ( @@ -293,6 +305,8 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( gpuOptions={gpuOptions(v?.requires.gpuProduct, nodes.data?.nodes.map((n) => n.GPUProduct))} expanded={expanded} onExpand={(s, o) => setExpanded((e) => ({ ...e, [s]: o }))} + routeOptions={routeOptions.data} + catalogRoute={catalogRoute} idPrefix={ID_PREFIX} />
diff --git a/web/src/modules/inferences/locales/en-US.json b/web/src/modules/inferences/locales/en-US.json index 7deafc4..0bc65d6 100644 --- a/web/src/modules/inferences/locales/en-US.json +++ b/web/src/modules/inferences/locales/en-US.json @@ -330,6 +330,72 @@ "routing": "Routing", "advanced": "Advanced" }, + "routeOptions": { + "advanced": "Advanced route options", + "inherit": "Use engine default", + "on": "On", + "off": "Off", + "reset": "Reset {label}", + "current": "Currently: {value} · {source}", + "configured": "route config", + "inherited": "engine default or derived", + "error": "Could not read route options: {error}", + "groups": { + "Adaptive concurrency": "Adaptive concurrency", + "Thresholds": "Thresholds", + "Capacity & failover": "Capacity & failover", + "Capacity": "Capacity & failover", + "Routing": "Routing", + "Observability": "Observability", + "Health checks": "Health checks", + "TTFT measurement": "TTFT measurement", + "TPS measurement": "TPS measurement", + "Rejection probes": "Rejection probes", + "Other": "Other" + }, + "options": { + "adaptive_cc": { "label": "Adaptive concurrency" }, + "adaptive_cc_min": { "label": "Adaptive floor" }, + "adaptive_cc_min_frac": { "label": "Floor fraction" }, + "adaptive_cc_init": { "label": "Start point" }, + "adaptive_cc_init_frac": { "label": "Start fraction" }, + "adaptive_cc_hold_when_idle": { "label": "Hold when idle" }, + "adaptive_cc_use_ttft": { "label": "Judge on TTFT too" }, + "tps_limit_tps": { "label": "Decode rate threshold" }, + "ttft_limit_ms": { "label": "TTFT threshold" }, + "default_max": { "label": "Default peer maximum" }, + "max_more_tries": { "label": "Retry budget" }, + "cross_tier_fallback": { "label": "Cross-tier fallback" }, + "health_probe_path": { "label": "Probe path" }, + "session_affinity_enabled": { "label": "Session affinity" }, + "expose_routed_peer": { "label": "Expose routed peer" }, + "bodylog_default_pct": { "label": "Body log sample percent" }, + "adaptive_cc_inc": { "label": "Climb multiplier" }, + "adaptive_cc_dec": { "label": "Shrink multiplier" }, + "adaptive_cc_abs": { "label": "Absolute headroom" }, + "adaptive_cc_pressure_frac": { "label": "Pressure fraction" }, + "adaptive_cc_slack_frac": { "label": "Slack fraction", "detail": "Slack band relative to the start point; inactive while hold-when-idle is enabled." }, + "adaptive_cc_ttl": { "label": "Start-point TTL", "detail": "Lifetime of the idle start-point state; inactive while hold-when-idle is enabled." }, + "adaptive_cc_interval": { "label": "Step interval" }, + "rt_limit_factor": { "label": "Runtime limit factor", "detail": "Scales the pool's runtime concurrency limit, not each peer's maximum." }, + "health_check_interval": { "label": "Probe interval" }, + "health_ban_ttl": { "label": "Ban TTL" }, + "cluster_avg_interval": { "label": "Cluster average interval" }, + "ttft_ewma_alpha": { "label": "TTFT EWMA alpha" }, + "ttft_window": { "label": "TTFT window" }, + "ttft_ttl": { "label": "TTFT TTL" }, + "tps_ewma_alpha": { "label": "TPS EWMA alpha" }, + "tps_window": { "label": "TPS window" }, + "tps_ttl": { "label": "TPS TTL" }, + "tps_min_tokens": { "label": "Minimum tokens" }, + "tps_min_decode_s": { "label": "Minimum decode seconds" }, + "ttft_probe_window": { "label": "TTFT probe window" }, + "ttft_probe_per_window": { "label": "TTFT probes per window", "detail": "Probe requests allowed through while TTFT limiting is active, per probe window." }, + "tps_probe_window": { "label": "TPS probe window" }, + "tps_probe_per_window": { "label": "TPS probes per window", "detail": "Probe requests allowed through while TPS limiting is active, per probe window." }, + "reject_rules_status": { "label": "Rejection status" } + } + }, "fields": { "model": "Model", "catalog": "Catalog", @@ -397,7 +463,7 @@ "adaptiveCc": "Adaptive concurrency", "adaptiveCcHint": "AIMD on the effective TPS signal; on by default", "adaptiveCcMinFrac": "Adaptive floor fraction", - "adaptiveCcMinFracHint": "Of the static peer max, 0 < value ≤ 1; empty leaves openresty's 0.4. Unused when the absolute floor is set", + "adaptiveCcMinFracHint": "Of the static maximum, 0 < value ≤ 1; empty uses the engine default. Unused when the absolute floor is set", "fracRange": "Must be over 0 and at most 1, or openresty ignores it", "monitor": "Monitor", "monitorHint": "Writes the backends into monitor's config", diff --git a/web/src/modules/inferences/locales/zh-CN.json b/web/src/modules/inferences/locales/zh-CN.json index 4739ccd..cab3e6e 100644 --- a/web/src/modules/inferences/locales/zh-CN.json +++ b/web/src/modules/inferences/locales/zh-CN.json @@ -319,6 +319,72 @@ "routing": "路由", "advanced": "高级" }, + "routeOptions": { + "advanced": "高级路由参数", + "inherit": "使用引擎默认值", + "on": "开启", + "off": "关闭", + "reset": "重置{label}", + "current": "当前：{value} · {source}", + "configured": "路由配置", + "inherited": "引擎默认值或派生值", + "error": "无法读取路由参数：{error}", + "groups": { + "Adaptive concurrency": "自适应并发", + "Thresholds": "阈值", + "Capacity & failover": "容量与故障切换", + "Capacity": "容量与故障切换", + "Routing": "路由", + "Observability": "可观测性", + "Health checks": "健康检查", + "TTFT measurement": "TTFT 测量", + "TPS measurement": "TPS 测量", + "Rejection probes": "限流探测", + "Other": "其他" + }, + "options": { + "adaptive_cc": { "label": "自适应并发" }, + "adaptive_cc_min": { "label": "自适应下限" }, + "adaptive_cc_min_frac": { "label": "下限比例" }, + "adaptive_cc_init": { "label": "起始并发" }, + "adaptive_cc_init_frac": { "label": "起始比例" }, + "adaptive_cc_hold_when_idle": { "label": "空闲时保持并发" }, + "adaptive_cc_use_ttft": { "label": "同时依据 TTFT 调整" }, + "tps_limit_tps": { "label": "解码速率阈值" }, + "ttft_limit_ms": { "label": "TTFT 阈值" }, + "default_max": { "label": "默认节点并发上限" }, + "max_more_tries": { "label": "重试次数" }, + "cross_tier_fallback": { "label": "跨层故障切换" }, + "health_probe_path": { "label": "探测路径" }, + "session_affinity_enabled": { "label": "会话亲和性" }, + "expose_routed_peer": { "label": "暴露路由后端" }, + "bodylog_default_pct": { "label": "请求体日志采样比例" }, + "adaptive_cc_inc": { "label": "增大倍率" }, + "adaptive_cc_dec": { "label": "缩小倍率" }, + "adaptive_cc_abs": { "label": "绝对余量" }, + "adaptive_cc_pressure_frac": { "label": "压力比例" }, + "adaptive_cc_slack_frac": { "label": "松弛比例", "detail": "相对起始并发的松弛区间；开启空闲保持时不生效。" }, + "adaptive_cc_ttl": { "label": "起始状态 TTL", "detail": "空闲起始状态的保留时间；开启空闲保持时不生效。" }, + "adaptive_cc_interval": { "label": "调整间隔" }, + "rt_limit_factor": { "label": "运行时并发倍率", "detail": "缩放整个池的运行时并发上限，不是每个节点的上限。" }, + "health_check_interval": { "label": "探测间隔" }, + "health_ban_ttl": { "label": "封禁 TTL" }, + "cluster_avg_interval": { "label": "集群均值计算间隔" }, + "ttft_ewma_alpha": { "label": "TTFT 平滑系数" }, + "ttft_window": { "label": "TTFT 时间窗口" }, + "ttft_ttl": { "label": "TTFT TTL" }, + "tps_ewma_alpha": { "label": "TPS 平滑系数" }, + "tps_window": { "label": "TPS 时间窗口" }, + "tps_ttl": { "label": "TPS TTL" }, + "tps_min_tokens": { "label": "最少 token 数" }, + "tps_min_decode_s": { "label": "最短解码时间" }, + "ttft_probe_window": { "label": "TTFT 探测窗口" }, + "ttft_probe_per_window": { "label": "每窗口 TTFT 探测请求数", "detail": "TTFT 限流期间每个探测窗口放行的请求数，不是采样率。" }, + "tps_probe_window": { "label": "TPS 探测窗口" }, + "tps_probe_per_window": { "label": "每窗口 TPS 探测请求数", "detail": "TPS 限流期间每个探测窗口放行的请求数，不是采样率。" }, + "reject_rules_status": { "label": "拒绝状态码" } + } + }, "fields": { "model": "模型", "catalog": "目录", @@ -386,7 +452,7 @@ "adaptiveCc": "自适应并发", "adaptiveCcHint": "按有效 TPS 做 AIMD 调整，默认开启", "adaptiveCcMinFrac": "自适应下限比例", - "adaptiveCcMinFracHint": "占静态并发上限的比例，0 < 值 ≤ 1，留空为 openresty 默认 0.4；设置了绝对下限时不使用", + "adaptiveCcMinFracHint": "占静态并发上限的比例，0 < 值 ≤ 1，留空使用引擎默认值；设置了绝对下限时不使用", "fracRange": "必须大于 0 且不超过 1，否则 openresty 会忽略", "monitor": "监控", "monitorHint": "把后端写入 monitor 的配置", diff --git a/web/src/modules/inferences/route-options.test.ts b/web/src/modules/inferences/route-options.test.ts new file mode 100644 index 0000000..fb4ee1d --- /dev/null +++ b/web/src/modules/inferences/route-options.test.ts @@ -0,0 +1,64 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { FALLBACK_ROUTE_OPTIONS, readRouteOptions, routeDirectives, runtimeValue } from "./route-options"; + +const fetchMock = vi.fn(); +const catalog = { + route: "model", + directives: [{ key: "new_engine_option", label: "New option", group: "Future group", kind: "number", fallback: "73", detail: "Engine help", value: 81, set: true }], + advanced: [], + peers: { count: 3 }, + adaptive_cc_derived: { _: { static_max: 200, min: 80, init: 80 } }, +}; +const response = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status }); + +beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal("fetch", fetchMock); + vi.stubGlobal("document", { cookie: "" }); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe("route options transport", () => { + it("reads an existing route, preserves host defaults and derived state, and accepts unknown options", async () => { + fetchMock.mockResolvedValue(response(catalog)); + expect(await readRouteOptions("model.with-dots")).toEqual(catalog); + expect(fetchMock.mock.calls[0][0]).toBe("/api/llm/route-options?route=model.with-dots"); + }); + it("uses a catalog-only probe for a new deployment", async () => { + fetchMock.mockResolvedValue(response(catalog)); + await readRouteOptions(); + expect(fetchMock.mock.calls[0][0]).toBe("/api/llm/route-options"); + }); + it("uses legacy metadata only for a 404", async () => { + fetchMock.mockResolvedValue(response({ error: "no catalog" }, 404)); + expect(await readRouteOptions()).toBeNull(); + }); + it.each([401, 403, 500, 502, 503])("surfaces HTTP %s rather than silently using legacy defaults", async (status) => { + fetchMock.mockResolvedValue(response({ error: "catalog failed" }, status)); + await expect(readRouteOptions()).rejects.toThrow("catalog failed"); + }); + it.each([ + {}, { ...catalog, directives: null }, { ...catalog, advanced: null }, + { ...catalog, directives: [{ ...catalog.directives[0], value: {} }] }, + { ...catalog, directives: [{ ...catalog.directives[0], set: "false" }] }, + { ...catalog, advanced: catalog.directives }, + ])("rejects invalid catalog data", async (body) => { + fetchMock.mockResolvedValue(response(body)); + await expect(readRouteOptions()).rejects.toThrow("Invalid route options response"); + }); +}); + +describe("legacy catalog", () => { + it("contains all 16 common and 24 advanced options without asserting engine defaults or route values", () => { + expect(FALLBACK_ROUTE_OPTIONS.directives).toHaveLength(16); + expect(FALLBACK_ROUTE_OPTIONS.advanced).toHaveLength(24); + for (const d of routeDirectives()) { + expect(d.fallback).toBe(""); + expect(d.value).toBeUndefined(); + expect(d.set).toBeUndefined(); + } + }); + it("keeps null, false and zero distinct", () => { + expect([null, false, true, 0, "the floor"].map(runtimeValue)).toEqual(["-", "off", "on", "0", "the floor"]); + }); +}); diff --git a/web/src/modules/inferences/route-options.ts b/web/src/modules/inferences/route-options.ts new file mode 100644 index 0000000..3730691 --- /dev/null +++ b/web/src/modules/inferences/route-options.ts @@ -0,0 +1,87 @@ +import { apiFetch, ApiError } from "@/shell"; +import { DIRECTIVES, type Directive } from "@swiss/lib/routeDirectives"; + +export type RouteValue = string | number | boolean | null; +export interface RouteDirective extends Directive { + value?: RouteValue; + set?: boolean; +} +export interface RouteOptions { + route: string; + directives: RouteDirective[]; + advanced: RouteDirective[]; + peers?: { count: number }; + adaptive_cc_derived?: Record; +} + +const common = new Set([ + "adaptive_cc", "adaptive_cc_min", "adaptive_cc_min_frac", "adaptive_cc_init", "adaptive_cc_init_frac", "adaptive_cc_hold_when_idle", "adaptive_cc_use_ttft", + "tps_limit_tps", "ttft_limit_ms", "default_max", "max_more_tries", "cross_tier_fallback", "health_probe_path", "session_affinity_enabled", "expose_routed_peer", "bodylog_default_pct", +]); +const additions: Directive[] = [ + { key: "adaptive_cc", label: "Adaptive concurrency", group: "Adaptive concurrency", kind: "boolean", fallback: "", detail: "Adjust the pool limit using measured performance." }, + { key: "adaptive_cc_min_frac", label: "Floor fraction", group: "Adaptive concurrency", kind: "number", fallback: "", detail: "Fraction of the static maximum used as the floor; greater than zero and at most one.", needsAdaptive: true }, + { key: "adaptive_cc_init", label: "Start point", group: "Adaptive concurrency", kind: "number", fallback: "", detail: "Absolute initial pool limit.", needsAdaptive: true }, + { key: "adaptive_cc_init_frac", label: "Start fraction", group: "Adaptive concurrency", kind: "number", fallback: "", detail: "Fraction of the static maximum used as the initial limit.", needsAdaptive: true }, + { key: "adaptive_cc_hold_when_idle", label: "Hold when idle", group: "Adaptive concurrency", kind: "boolean", fallback: "", detail: "Keep the learned limit when idle instead of returning to the start point.", needsAdaptive: true }, + { key: "tps_limit_tps", label: "Decode rate threshold", group: "Thresholds", kind: "number", fallback: "", detail: "Static decode-rate threshold, used when no TPS metrics are declared." }, + { key: "ttft_limit_ms", label: "TTFT threshold", group: "Thresholds", kind: "number", fallback: "", detail: "Static time-to-first-token threshold in milliseconds, used when no TTFT metrics are declared." }, + { key: "session_affinity_enabled", label: "Session affinity", group: "Routing", kind: "boolean", fallback: "", detail: "Keep requests in a session on the same peer." }, + { key: "expose_routed_peer", label: "Expose routed peer", group: "Observability", kind: "boolean", fallback: "", detail: "Return the selected peer in the X-Routed-Peer response header." }, +]; +const corrections: Record = { + adaptive_cc_ttl: "Lifetime of the idle start-point state; has no effect while hold-when-idle is enabled.", + adaptive_cc_slack_frac: "Slack band relative to the start point; has no effect while hold-when-idle is enabled.", + rt_limit_factor: "Scales the pool's runtime concurrency limit, not each peer's maximum.", + max_more_tries: "Upper bound on balancer retries for one request.", + ttft_probe_per_window: "Probe requests allowed through while TTFT limiting is active, per probe window.", + tps_probe_per_window: "Probe requests allowed through while TPS limiting is active, per probe window.", +}; +const bundled = [...DIRECTIVES.map((d) => ({ ...d, fallback: "", detail: corrections[d.key] ?? d.detail })), ...additions]; +export const FALLBACK_ROUTE_OPTIONS: RouteOptions = { + route: "", + directives: bundled.filter((d) => common.has(d.key)), + advanced: bundled.filter((d) => !common.has(d.key)), +}; + +export function routeDirectives(options?: RouteOptions | null): RouteDirective[] { + const catalog = options ?? FALLBACK_ROUTE_OPTIONS; + return [...catalog.directives, ...catalog.advanced]; +} + +function isRecord(v: unknown): v is Record { + return !!v && typeof v === "object" && !Array.isArray(v); +} +function isDirective(v: unknown): v is RouteDirective { + if (!isRecord(v)) return false; + return ["key", "label", "group", "fallback", "detail"].every((k) => typeof v[k] === "string") + && ["number", "boolean", "text"].includes(String(v.kind)) + && typeof v.set === "boolean" + && (v.needsAdaptive === undefined || typeof v.needsAdaptive === "boolean") + && (v.value === null || ["number", "boolean", "string"].includes(typeof v.value)); +} + +export async function readRouteOptions(route?: string, signal?: AbortSignal): Promise { + const query = route ? `?${new URLSearchParams({ route })}` : ""; + const res = await apiFetch(`/api/llm/route-options${query}`, { signal }); + if (res.status === 404) return null; + if (!res.ok) { + const body = await res.json().catch(() => null); + throw new ApiError(res.status, typeof body?.error === "string" ? body.error : res.statusText); + } + const body: unknown = await res.json(); + if (!isRecord(body) || typeof body.route !== "string" || !body.route + || !Array.isArray(body.directives) || !body.directives.every(isDirective) + || !Array.isArray(body.advanced) || !body.advanced.every(isDirective)) { + throw new Error("Invalid route options response"); + } + const keys = [...body.directives, ...body.advanced].map((d) => d.key); + if (new Set(keys).size !== keys.length) throw new Error("Invalid route options response"); + return body as unknown as RouteOptions; +} + +export function runtimeValue(value: RouteValue | undefined): string { + if (value === null || value === undefined) return "-"; + if (typeof value === "boolean") return value ? "on" : "off"; + return String(value); +} From d5213edca7286efde3b7c29644551aa94f5380cd Mon Sep 17 00:00:00 2001 From: imneov Date: Wed, 14 Oct 2026 22:17:43 -0700 Subject: [PATCH 2/2] fix(web): honor route catalog fallback and discovery semantics Keep legacy metadata exclusive to explicit unsupported/empty responses, discover catalog routes independently of inference pins, and put transport under the inference module API boundary. --- internal/gateway/gateway.go | 29 ++++--- internal/gateway/gateway_test.go | 71 +++++++++++++++++ internal/server/proxy.go | 12 ++- internal/server/route_options_test.go | 76 ++++++++++++++++--- .../{route-options.test.ts => api.test.ts} | 40 +++++----- web/src/modules/inferences/api.ts | 33 ++++++++ .../inferences/components/DeployForm.test.tsx | 23 +++++- .../inferences/components/DeployForm.tsx | 4 +- .../inferences/components/DeploySheet.tsx | 4 +- .../modules/inferences/routeOptions.test.ts | 27 +++++++ .../{route-options.ts => routeOptions.ts} | 36 +-------- 11 files changed, 271 insertions(+), 84 deletions(-) rename web/src/modules/inferences/{route-options.test.ts => api.test.ts} (61%) create mode 100644 web/src/modules/inferences/api.ts create mode 100644 web/src/modules/inferences/routeOptions.test.ts rename web/src/modules/inferences/{route-options.ts => routeOptions.ts} (69%) diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index a99669c..c1dffe2 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -91,7 +91,7 @@ func (r *Resolver) Resolve(ctx context.Context) (*Entry, error) { return nil, r.err } - entry, err := r.resolve(ctx) + entry, err := r.resolve(ctx, false) r.at = time.Now() switch { case err != nil && r.entry != nil: @@ -113,7 +113,12 @@ func (r *Resolver) Resolve(ctx context.Context) (*Entry, error) { return entry, nil } -func (r *Resolver) resolve(ctx context.Context) (*Entry, error) { +// ResolveCatalog discovers existing routes without inference pinning or credentials. +func (r *Resolver) ResolveCatalog(ctx context.Context) (*Entry, error) { + return r.resolve(ctx, true) +} + +func (r *Resolver) resolve(ctx context.Context, catalog bool) (*Entry, error) { gw := r.backend.Gateway if gw == nil { return nil, fmt.Errorf("backend %s is not a gateway", r.backend.Name) @@ -143,7 +148,7 @@ func (r *Resolver) resolve(ctx context.Context) (*Entry, error) { port = config.DefaultGatewayPort } - routes, source, err := r.routes(ctx, profile, profileRef) + routes, source, err := r.routes(ctx, profile, profileRef, catalog) if err != nil { return nil, err } @@ -157,23 +162,22 @@ func (r *Resolver) resolve(ctx context.Context) (*Entry, error) { Header: profile.Route.Auth.HeaderName(), Source: source, } - if err := r.credential(ctx, profile, svcNS, entry); err != nil { - return nil, err + if !catalog { + if err := r.credential(ctx, profile, svcNS, entry); err != nil { + return nil, err + } } return entry, nil } -// routes lists the routes to talk to: the explicit one, else every route in the -// route ConfigMap, aggregate ones (those serving several models) first. autoconfig -// writes one plain route per model, so a gateway with no aggregate route is the -// normal case, not an error. -func (r *Resolver) routes(ctx context.Context, p profile, profileRef string) (routes []string, source string, err error) { +// routes lists inference routes, or all configured routes for catalog discovery. +func (r *Resolver) routes(ctx context.Context, p profile, profileRef string, catalog bool) (routes []string, source string, err error) { gw := r.backend.Gateway ref := gw.ConfigMap if ref == "" { ref = p.Route.NginxConfigMap } - if gw.Route != "" { + if gw.Route != "" && (!catalog || ref == "") { source = ref if source == "" { source = profileRef @@ -181,6 +185,9 @@ func (r *Resolver) routes(ctx context.Context, p profile, profileRef string) (ro return []string{gw.Route}, source, nil } if ref == "" { + if catalog { + return nil, "", ErrNoRoutes + } return nil, "", fmt.Errorf("no gateway.route and no route configmap to choose one from: set gateway.route") } ns, name, err := splitRef(ref, "") diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 831f156..ecfb9a2 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "log/slog" "slices" "strings" @@ -99,6 +100,76 @@ func resolveOne(t *testing.T, cfg *config.Gateway, c *fakeCluster) (*Entry, erro return NewResolver(b, c, slog.New(slog.DiscardHandler)).Resolve(t.Context()) } +func TestResolveCatalogIndependentOfInferencePin(t *testing.T) { + for _, pin := range []string{"video", "stale"} { + t.Run(pin, func(t *testing.T) { + c := fixture() + b := config.Backend{Name: "llm", Gateway: &config.Gateway{Profile: "llm/site-profile", Route: pin}} + r := NewResolver(b, c, slog.New(slog.DiscardHandler)) + inference, err := r.Resolve(t.Context()) + if err != nil { + t.Fatal(err) + } + catalog, err := r.ResolveCatalog(t.Context()) + if err != nil { + t.Fatal(err) + } + if catalog.Base != "http://openresty.llm.svc:8080" || !slices.Equal(catalog.Routes, []string{"llm-gateway", "glm-5.1", "kimi-k2.6"}) || catalog.Key != "" { + t.Fatalf("catalog = %+v", catalog) + } + again, err := r.Resolve(t.Context()) + if err != nil || again != inference || !slices.Equal(again.Routes, []string{pin}) || again.Key != "Bearer k1" { + t.Fatalf("inference pin or credential changed: %+v, %v", again, err) + } + }) + } +} + +func TestResolveCatalogWithoutCredentialSecret(t *testing.T) { + c := fixture() + c.secrets = nil + t.Setenv("CONSOLE_TEST_CATALOG_KEY", "secret") + for _, keyEnv := range []string{"", "CONSOLE_TEST_CATALOG_KEY"} { + b := config.Backend{Name: "llm", APIKeyEnv: keyEnv, Gateway: &config.Gateway{Profile: "llm/site-profile"}} + entry, err := NewResolver(b, c, slog.New(slog.DiscardHandler)).ResolveCatalog(t.Context()) + if err != nil || entry.Key != "" || entry.KeySource != "" { + t.Fatalf("catalog required or exposed a credential: %+v, %v", entry, err) + } + } +} + +func TestResolveCatalogRouteSources(t *testing.T) { + for _, tc := range []struct { + name string + gw config.Gateway + routes []string + err error + }{ + {"explicit without configmap", config.Gateway{Service: "llm/openresty", Route: "model", Port: 9000}, []string{"model"}, nil}, + {"no route source", config.Gateway{Service: "llm/openresty"}, nil, ErrNoRoutes}, + {"empty configmap despite pin", config.Gateway{Service: "llm/openresty", ConfigMap: "llm/empty-conf", Route: "stale"}, nil, ErrNoRoutes}, + {"missing configmap despite pin", config.Gateway{Service: "llm/openresty", ConfigMap: "llm/missing", Route: "stale"}, nil, errNotFound}, + {"explicit configmap overrides profile", config.Gateway{Profile: "llm/site-profile", ConfigMap: "llm/many-routes-conf", Route: "stale"}, []string{"a", "b"}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + b := config.Backend{Name: "llm", Gateway: &tc.gw} + entry, err := NewResolver(b, fixture(), slog.New(slog.DiscardHandler)).ResolveCatalog(t.Context()) + if tc.err != nil { + if !errors.Is(err, tc.err) { + t.Fatalf("err = %v, want %v", err, tc.err) + } + return + } + if err != nil || !slices.Equal(entry.Routes, tc.routes) { + t.Fatalf("entry = %+v, err = %v", entry, err) + } + if tc.gw.Port == 9000 && entry.Base != "http://openresty.llm.svc:9000" { + t.Fatalf("explicit port ignored: %+v", entry) + } + }) + } +} + func TestResolveFromProfile(t *testing.T) { entry, err := resolveOne(t, &config.Gateway{Profile: "llm/site-profile"}, fixture()) if err != nil { diff --git a/internal/server/proxy.go b/internal/server/proxy.go index a5607bb..3ec90c0 100644 --- a/internal/server/proxy.go +++ b/internal/server/proxy.go @@ -65,8 +65,12 @@ func (s *Server) mountBackends(mux *http.ServeMux) { proxy := s.backendProxy(b) prefix := strings.TrimSuffix(b.Prefix, "/") catalogMounted = catalogMounted || prefix == "/api/llm" + catalogResolve := resolve + if b.Gateway != nil { + catalogResolve = gatewayResolver(gateway.NewResolver(b, s.kube, s.log).ResolveCatalog) + } mux.HandleFunc(prefix+"/route-options", func(w http.ResponseWriter, r *http.Request) { - s.handleRouteOptions(w, r, b, resolve) + s.handleRouteOptions(w, r, b, catalogResolve) }) mux.HandleFunc(strings.TrimSuffix(b.Prefix, "/")+"/", func(w http.ResponseWriter, r *http.Request) { if !s.authorize(w, r, backendVerb(r.Method), backendResource, b.Name) { @@ -126,7 +130,7 @@ func (s *Server) targetFunc(b config.Backend) (targetResolver, error) { if s.kube == nil { return nil, fmt.Errorf("gateway %+v needs cluster access, and this console has none", *b.Gateway) } - return gatewayResolver(gateway.NewResolver(b, s.kube, s.log)), nil + return gatewayResolver(gateway.NewResolver(b, s.kube, s.log).Resolve), nil } target, err := url.Parse(b.URL) @@ -144,9 +148,9 @@ func (s *Server) targetFunc(b config.Backend) (targetResolver, error) { return func(context.Context) (*backendTarget, error) { return fixed, nil }, nil } -func gatewayResolver(r *gateway.Resolver) targetResolver { +func gatewayResolver(resolve func(context.Context) (*gateway.Entry, error)) targetResolver { return func(ctx context.Context) (*backendTarget, error) { - entry, err := r.Resolve(ctx) + entry, err := resolve(ctx) if err != nil { return nil, err } diff --git a/internal/server/route_options_test.go b/internal/server/route_options_test.go index a1bdeb2..e1a85d5 100644 --- a/internal/server/route_options_test.go +++ b/internal/server/route_options_test.go @@ -16,6 +16,63 @@ import ( const routeOptionsFixture = `{"route":"model","directives":[{"key":"tps_limit_tps","label":"Decode rate threshold","group":"Thresholds","kind":"number","fallback":"73","detail":"Deployment default","value":81,"set":true}],"advanced":[],"peers":{"count":3},"adaptive_cc_derived":{"_":{"static_max":200,"min":80,"init":80}}}` +type routeOptionsTransport func(*http.Request) (*http.Response, error) + +func (f routeOptionsTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestRouteOptionsDiscoversCatalogDespiteInferencePin(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" || r.Header.Get("X-Remote-User") != "" { + t.Error("credentials or identity forwarded to public route catalog") + } + if r.URL.Path != "/model/_route_opts" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer up.Close() + base, _ := url.Parse(up.URL) + transport := http.DefaultTransport + http.DefaultTransport = routeOptionsTransport(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "openresty.llm.svc:8080" { + t.Errorf("gateway host = %q", r.URL.Host) + } + clone := r.Clone(r.Context()) + clone.URL.Scheme, clone.URL.Host = base.Scheme, base.Host + return transport.RoundTrip(clone) + }) + t.Cleanup(func() { http.DefaultTransport = transport }) + for _, tc := range []struct { + name string + gw config.Gateway + }{ + {"video", config.Gateway{Profile: "llm/site-profile", Route: "video"}}, + {"stale", config.Gateway{Profile: "llm/site-profile", Route: "stale"}}, + {"explicit without configmap", config.Gateway{Service: "llm/openresty", Route: "model", SecretRef: "llm/missing-keys"}}, + } { + t.Run(tc.name, func(t *testing.T) { + objects := gatewayObjects(t)[:2] + objects[1].(*unstructured.Unstructured).Object["data"] = map[string]any{ + "session_route_video.conf": "peers_by_model = {}", + "session_route_model.conf": "set $route model;", + } + cfg := &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", Gateway: &tc.gw}}} + h := testServerWithConfig(t, cfg, objects...).Handler() + admin := login(t, h, "admin", "admin-pw") + for _, path := range []string{"/api/llm/route-options", "/api/llm/route-options?route=model"} { + rec := do(h, "GET", path, admin, "") + if rec.Code != http.StatusOK || rec.Body.String() != routeOptionsFixture { + t.Fatalf("catalog with %q = %d %s", tc.name, rec.Code, rec.Body.String()) + } + } + if rec := do(h, "GET", "/api/llm/route-options?route=stale", admin, ""); rec.Code != http.StatusNotFound { + t.Fatalf("unknown route = %d %s", rec.Code, rec.Body.String()) + } + }) + } +} + func TestRouteOptionsWithoutGatewayDoesNotServeSPA(t *testing.T) { cfg := &config.Config{} cfg.Server.Auth.Disabled = true @@ -46,16 +103,16 @@ func TestRouteOptionsLiveCatalog(t *testing.T) { })) defer up.Close() base, _ := url.Parse(up.URL) - srv := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", Gateway: &config.Gateway{ConfigMap: "llm/routes", Service: "llm/openresty"}}}}) - srv.targetsOnce.Do(func() { - srv.targets = map[string]targetResolver{"llm": func(context.Context) (*backendTarget, error) { - return &backendTarget{url: *base, routes: []string{"video", "model"}, key: "secret", header: "Authorization"}, nil - }} - }) - h := srv.Handler() - admin := login(t, h, "admin", "admin-pw") + b := config.Backend{Name: "llm", Prefix: "/api/llm", Gateway: &config.Gateway{ConfigMap: "llm/routes", Service: "llm/openresty"}} + cfg := &config.Config{Backends: []config.Backend{b}} + cfg.Server.Auth.Disabled = true + srv := testServerWithConfig(t, cfg) + resolve := func(context.Context) (*backendTarget, error) { + return &backendTarget{url: *base, routes: []string{"video", "model"}, key: "secret", header: "Authorization"}, nil + } + h := srv.authenticate(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.handleRouteOptions(w, r, b, resolve) })) for _, path := range []string{"/api/llm/route-options", "/api/llm/route-options?route=model"} { - rec := do(h, "GET", path, admin, "") + rec := do(h, "GET", path, "", "") if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"fallback":"73"`) || !strings.Contains(rec.Body.String(), `"value":81`) { t.Fatalf("live catalog = %d %s", rec.Code, rec.Body.String()) } @@ -70,6 +127,7 @@ func TestRouteOptionsEmptyGatewayAndResolutionFailure(t *testing.T) { ref string want int }{ + {"", http.StatusNotFound}, {"llm/openresty-conf", http.StatusNotFound}, {"llm/missing-conf", http.StatusBadGateway}, } { diff --git a/web/src/modules/inferences/route-options.test.ts b/web/src/modules/inferences/api.test.ts similarity index 61% rename from web/src/modules/inferences/route-options.test.ts rename to web/src/modules/inferences/api.test.ts index fb4ee1d..d236f31 100644 --- a/web/src/modules/inferences/route-options.test.ts +++ b/web/src/modules/inferences/api.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { FALLBACK_ROUTE_OPTIONS, readRouteOptions, routeDirectives, runtimeValue } from "./route-options"; +import { readRouteOptions } from "./api"; const fetchMock = vi.fn(); const catalog = { @@ -35,10 +35,29 @@ describe("route options transport", () => { }); it.each([401, 403, 500, 502, 503])("surfaces HTTP %s rather than silently using legacy defaults", async (status) => { fetchMock.mockResolvedValue(response({ error: "catalog failed" }, status)); - await expect(readRouteOptions()).rejects.toThrow("catalog failed"); + await expect(readRouteOptions()).rejects.toMatchObject({ status, message: "catalog failed" }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it("surfaces non-JSON API failures", async () => { + fetchMock.mockResolvedValue(new Response("Bad Gateway", { status: 502, statusText: "Bad Gateway" })); + await expect(readRouteOptions()).rejects.toMatchObject({ status: 502, message: "Bad Gateway" }); + }); + it("surfaces network failures instead of reporting a legacy catalog", async () => { + fetchMock.mockRejectedValue(new TypeError("Network unavailable")); + await expect(readRouteOptions()).rejects.toThrow("Network unavailable"); + }); + it("surfaces malformed JSON instead of reporting a legacy catalog", async () => { + fetchMock.mockResolvedValue(new Response("not JSON")); + await expect(readRouteOptions()).rejects.toBeInstanceOf(SyntaxError); + }); + it("forwards query cancellation to the transport", async () => { + const controller = new AbortController(); + fetchMock.mockResolvedValue(response(catalog)); + await readRouteOptions("model", controller.signal); + expect(fetchMock.mock.calls[0][1].signal).toBe(controller.signal); }); it.each([ - {}, { ...catalog, directives: null }, { ...catalog, advanced: null }, + {}, { ...catalog, route: "" }, { ...catalog, directives: null }, { ...catalog, advanced: null }, { ...catalog, directives: [{ ...catalog.directives[0], value: {} }] }, { ...catalog, directives: [{ ...catalog.directives[0], set: "false" }] }, { ...catalog, advanced: catalog.directives }, @@ -47,18 +66,3 @@ describe("route options transport", () => { await expect(readRouteOptions()).rejects.toThrow("Invalid route options response"); }); }); - -describe("legacy catalog", () => { - it("contains all 16 common and 24 advanced options without asserting engine defaults or route values", () => { - expect(FALLBACK_ROUTE_OPTIONS.directives).toHaveLength(16); - expect(FALLBACK_ROUTE_OPTIONS.advanced).toHaveLength(24); - for (const d of routeDirectives()) { - expect(d.fallback).toBe(""); - expect(d.value).toBeUndefined(); - expect(d.set).toBeUndefined(); - } - }); - it("keeps null, false and zero distinct", () => { - expect([null, false, true, 0, "the floor"].map(runtimeValue)).toEqual(["-", "off", "on", "0", "the floor"]); - }); -}); diff --git a/web/src/modules/inferences/api.ts b/web/src/modules/inferences/api.ts new file mode 100644 index 0000000..73963a0 --- /dev/null +++ b/web/src/modules/inferences/api.ts @@ -0,0 +1,33 @@ +import { apiFetch, ApiError } from "@/shell"; +import type { RouteDirective, RouteOptions } from "./routeOptions"; + +function isRecord(v: unknown): v is Record { + return !!v && typeof v === "object" && !Array.isArray(v); +} +function isDirective(v: unknown): v is RouteDirective { + if (!isRecord(v)) return false; + return ["key", "label", "group", "fallback", "detail"].every((k) => typeof v[k] === "string") + && ["number", "boolean", "text"].includes(String(v.kind)) + && typeof v.set === "boolean" + && (v.needsAdaptive === undefined || typeof v.needsAdaptive === "boolean") + && (v.value === null || ["number", "boolean", "string"].includes(typeof v.value)); +} + +export async function readRouteOptions(route?: string, signal?: AbortSignal): Promise { + const query = route ? `?${new URLSearchParams({ route })}` : ""; + const res = await apiFetch(`/api/llm/route-options${query}`, { signal }); + if (res.status === 404) return null; + if (!res.ok) { + const body = await res.json().catch(() => null); + throw new ApiError(res.status, typeof body?.error === "string" ? body.error : res.statusText); + } + const body: unknown = await res.json(); + if (!isRecord(body) || typeof body.route !== "string" || !body.route + || !Array.isArray(body.directives) || !body.directives.every(isDirective) + || !Array.isArray(body.advanced) || !body.advanced.every(isDirective)) { + throw new Error("Invalid route options response"); + } + const keys = [...body.directives, ...body.advanced].map((d) => d.key); + if (new Set(keys).size !== keys.length) throw new Error("Invalid route options response"); + return body as unknown as RouteOptions; +} diff --git a/web/src/modules/inferences/components/DeployForm.test.tsx b/web/src/modules/inferences/components/DeployForm.test.tsx index 99fef84..edea55b 100644 --- a/web/src/modules/inferences/components/DeployForm.test.tsx +++ b/web/src/modules/inferences/components/DeployForm.test.tsx @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; import { DeployForm } from "./DeployForm"; import { EMPTY, planRequest, type Form } from "@swiss/components/DeploySettings"; -import type { RouteOptions } from "@/modules/inferences/route-options"; +import type { RouteOptions } from "@/modules/inferences/routeOptions"; const catalog: RouteOptions = { route: "model", @@ -10,7 +10,7 @@ const catalog: RouteOptions = { advanced: [{ key: "rt_limit_factor", label: "Runtime limit factor", group: "Capacity & failover", kind: "number", fallback: "7.25", detail: "Scales the pool limit.", needsAdaptive: false, value: 9, set: true }], }; -function render(routeOptions?: RouteOptions, upgrade = false, form: Form = { ...EMPTY, serviceId: "model", nginxExtras: [{ key: "rt_limit_factor", value: "" }] }, catalogRoute?: string) { +function render(routeOptions?: RouteOptions | null, upgrade = false, form: Form = { ...EMPTY, serviceId: "model", nginxExtras: [{ key: "rt_limit_factor", value: "" }] }, catalogRoute?: string) { const props = { form, upgrade, onChange: () => {}, submitted: false, target: null, defaults: {}, gpuOptions: [], @@ -25,9 +25,24 @@ describe("deploy route catalog", () => { expect(render(catalog).includes('placeholder="7.25"')).toBe(true); }); + it("shows bundled controls only for an explicit legacy catalog result", () => { + const form = { ...EMPTY, serviceId: "model" }; + expect(render(undefined, false, form).includes("adaptive_cc_init")).toBe(false); + expect(render(null, false, form).includes("adaptive_cc_init")).toBe(true); + }); + + it.each([undefined, null])("preserves arbitrary rows and named overrides when catalog data is %s", (options) => { + const form = { ...EMPTY, serviceId: "model", adaptiveCc: true, adaptiveCcMinFrac: "0.73", tpsLimitTps: "97", ttftLimitMs: "123", nginxExtras: [{ key: "future_knob", value: "keep-me" }, { key: "rt_limit_factor", value: "2" }] }; + const before = planRequest(form, { model: "model" }); + const html = render(options, true, form); + for (const value of ["keep-me", "2", "0.73", "97", "123"]) expect(html).toContain(`value="${value}"`); + expect(html).toContain("future_knob"); + expect(planRequest(form, { model: "model" })).toEqual(before); + }); + it("does not show stale bundled defaults without a live catalog", () => { - expect(render().includes('placeholder="1"')).toBe(false); - expect(render().includes("0.4")).toBe(false); + expect(render(null).includes('placeholder="1"')).toBe(false); + expect(render(null).includes("0.4")).toBe(false); }); it("does not show another route's runtime values on a new deployment or an unmatched upgrade", () => { diff --git a/web/src/modules/inferences/components/DeployForm.tsx b/web/src/modules/inferences/components/DeployForm.tsx index 495be85..22fddbf 100644 --- a/web/src/modules/inferences/components/DeployForm.tsx +++ b/web/src/modules/inferences/components/DeployForm.tsx @@ -24,7 +24,7 @@ import { Plus, X } from "lucide-react"; import type { ClusterInfo } from "@swiss/lib/api"; import { effective, type Form, type Toleration } from "@swiss/components/DeploySettings"; import { MAX_VALUES, rejectKey } from "@swiss/lib/routeDirectives"; -import { FALLBACK_ROUTE_OPTIONS, routeDirectives, runtimeValue, type RouteDirective, type RouteOptions } from "@/modules/inferences/route-options"; +import { FALLBACK_ROUTE_OPTIONS, routeDirectives, runtimeValue, type RouteDirective, type RouteOptions } from "@/modules/inferences/routeOptions"; import { useT } from "@/modules/inferences/i18n"; import { NAMED_VALUES, extraValue, fracOutOfRange, serviceIdError, type Section } from "@/modules/inferences/deploy-lib"; @@ -284,7 +284,7 @@ export function RouteExtras({ form, onChange, options, showRuntime = false }: { const f = (k: string, v?: Record) => t(`deploy.fields.${k}`, v); const [custom, setCustom] = useState(""); const controlsId = useId(); - const catalog = options ?? FALLBACK_ROUTE_OPTIONS; + const catalog = options === null ? FALLBACK_ROUTE_OPTIONS : options ?? { route: "", directives: [], advanced: [] }; const byKey = new Map(routeDirectives(catalog).map((d) => [d.key, d])); const taken = [...NAMED_VALUES, ...form.nginxExtras.map((r) => r.key)]; const used = NAMED_VALUES.size + form.nginxExtras.length; diff --git a/web/src/modules/inferences/components/DeploySheet.tsx b/web/src/modules/inferences/components/DeploySheet.tsx index 3340200..4d592e5 100644 --- a/web/src/modules/inferences/components/DeploySheet.tsx +++ b/web/src/modules/inferences/components/DeploySheet.tsx @@ -37,7 +37,7 @@ import { useT } from "@/modules/inferences/i18n"; import { detailPath } from "@/modules/inferences/lib"; import { SECTIONS, chartChoice, defaultServiceId, gpuOptions, pipelineState, serviceIdError, whatMoves, type Section } from "@/modules/inferences/deploy-lib"; import { DeployForm } from "@/modules/inferences/components/DeployForm"; -import { readRouteOptions } from "@/modules/inferences/route-options"; +import { readRouteOptions } from "@/modules/inferences/api"; export type DeployTarget = | { kind: "deploy"; model: string; catalog: string; version?: string; variant?: string } @@ -305,7 +305,7 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( gpuOptions={gpuOptions(v?.requires.gpuProduct, nodes.data?.nodes.map((n) => n.GPUProduct))} expanded={expanded} onExpand={(s, o) => setExpanded((e) => ({ ...e, [s]: o }))} - routeOptions={routeOptions.data} + routeOptions={routeOptions.isSuccess ? routeOptions.data : undefined} catalogRoute={catalogRoute} idPrefix={ID_PREFIX} /> diff --git a/web/src/modules/inferences/routeOptions.test.ts b/web/src/modules/inferences/routeOptions.test.ts new file mode 100644 index 0000000..3adcc98 --- /dev/null +++ b/web/src/modules/inferences/routeOptions.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { FALLBACK_ROUTE_OPTIONS, routeDirectives, runtimeValue } from "./routeOptions"; + +describe("legacy catalog", () => { + it("does not infer legacy support from pending or failed query data", () => { + expect(routeDirectives()).toEqual([]); + expect(routeDirectives(undefined)).toEqual([]); + expect(routeDirectives(null)).toEqual([...FALLBACK_ROUTE_OPTIONS.directives, ...FALLBACK_ROUTE_OPTIONS.advanced]); + }); + + it("keeps a successful empty catalog distinct from legacy fallback", () => { + expect(routeDirectives({ route: "model", directives: [], advanced: [] })).toEqual([]); + }); + + it("contains all 16 common and 24 advanced options without asserting engine defaults or route values", () => { + expect(FALLBACK_ROUTE_OPTIONS.directives).toHaveLength(16); + expect(FALLBACK_ROUTE_OPTIONS.advanced).toHaveLength(24); + for (const d of routeDirectives(null)) { + expect(d.fallback).toBe(""); + expect(d.value).toBeUndefined(); + expect(d.set).toBeUndefined(); + } + }); + it("keeps null, false and zero distinct", () => { + expect([null, false, true, 0, "the floor"].map(runtimeValue)).toEqual(["-", "off", "on", "0", "the floor"]); + }); +}); diff --git a/web/src/modules/inferences/route-options.ts b/web/src/modules/inferences/routeOptions.ts similarity index 69% rename from web/src/modules/inferences/route-options.ts rename to web/src/modules/inferences/routeOptions.ts index 3730691..20186e1 100644 --- a/web/src/modules/inferences/route-options.ts +++ b/web/src/modules/inferences/routeOptions.ts @@ -1,4 +1,3 @@ -import { apiFetch, ApiError } from "@/shell"; import { DIRECTIVES, type Directive } from "@swiss/lib/routeDirectives"; export type RouteValue = string | number | boolean | null; @@ -45,39 +44,8 @@ export const FALLBACK_ROUTE_OPTIONS: RouteOptions = { }; export function routeDirectives(options?: RouteOptions | null): RouteDirective[] { - const catalog = options ?? FALLBACK_ROUTE_OPTIONS; - return [...catalog.directives, ...catalog.advanced]; -} - -function isRecord(v: unknown): v is Record { - return !!v && typeof v === "object" && !Array.isArray(v); -} -function isDirective(v: unknown): v is RouteDirective { - if (!isRecord(v)) return false; - return ["key", "label", "group", "fallback", "detail"].every((k) => typeof v[k] === "string") - && ["number", "boolean", "text"].includes(String(v.kind)) - && typeof v.set === "boolean" - && (v.needsAdaptive === undefined || typeof v.needsAdaptive === "boolean") - && (v.value === null || ["number", "boolean", "string"].includes(typeof v.value)); -} - -export async function readRouteOptions(route?: string, signal?: AbortSignal): Promise { - const query = route ? `?${new URLSearchParams({ route })}` : ""; - const res = await apiFetch(`/api/llm/route-options${query}`, { signal }); - if (res.status === 404) return null; - if (!res.ok) { - const body = await res.json().catch(() => null); - throw new ApiError(res.status, typeof body?.error === "string" ? body.error : res.statusText); - } - const body: unknown = await res.json(); - if (!isRecord(body) || typeof body.route !== "string" || !body.route - || !Array.isArray(body.directives) || !body.directives.every(isDirective) - || !Array.isArray(body.advanced) || !body.advanced.every(isDirective)) { - throw new Error("Invalid route options response"); - } - const keys = [...body.directives, ...body.advanced].map((d) => d.key); - if (new Set(keys).size !== keys.length) throw new Error("Invalid route options response"); - return body as unknown as RouteOptions; + const catalog = options === null ? FALLBACK_ROUTE_OPTIONS : options; + return catalog ? [...catalog.directives, ...catalog.advanced] : []; } export function runtimeValue(value: RouteValue | undefined): string {