diff --git a/docs/console-design.md b/docs/console-design.md index f442650..d79c5fd 100644 --- a/docs/console-design.md +++ b/docs/console-design.md @@ -307,14 +307,34 @@ Not there yet: - **Search on the services list.** swissd pages its list (at most 100 a page) and takes no query; filtering one page on the client would miss the others. It needs `?q=` on swissd's `/api/deployments`. -- **swiss's panels and the route directive texts are English.** They are swiss's, - and swiss's pages have no Chinese yet. +- **swiss's panels are English.** They are swiss's, and swiss's pages have no Chinese yet. Route option labels are translated by inferences; untranslated help and new options use the engine's text. - **Cluster resources needs a recent swissd.** An older one (0.5.6, for one) has no `/objects`: the tab says so, and the install track reports the route and scaler as unreadable rather than absent. - **Upgrade picks chart versions through `/api/catalog/:name/chart-versions`**, which swissd has from 0.6.2 (swiss #13). +### Deploy route options + +The deploy and upgrade form reads the running openresty's catalog instead of treating bundled defaults as deployment defaults (console #52; [agreed design](development/route-options-catalog.md)). + +```text +DeploySheet -> GET /api/llm/route-options[?route=] + -> configured gateway -> //_route_opts + -> common grouped rows + advanced picker + runtime state +``` + +| Concern | Decision | +|---|---| +| Authorization | Console login and `get backends` for `llm`, the same grant as the inference proxy | +| New deployment | Probe existing ConfigMap routes for a catalog; never show their per-route values as the new deployment's | +| Upgrade | Query the recorded route, display `value` and `set` separately from saved overrides | +| Defaults | Live `fallback` supplies placeholders; no copied values are submitted automatically | +| Legacy or empty gateway | 404 uses bundled metadata with all fallback values stripped | +| Other failures | Surface a catalog warning; do not claim a deployment default | +| Text | Per-key locale translations where present; new options retain engine text | +| Ownership | Inferences adapts the Swiss metadata without editing the copied upstream module | + ### Workload reads (`/api/k8s/...`) Events and logs are Kubernetes data, not swiss's, so console serves them itself diff --git a/docs/development/route-options-catalog.md b/docs/development/route-options-catalog.md new file mode 100644 index 0000000..71e6d24 --- /dev/null +++ b/docs/development/route-options-catalog.md @@ -0,0 +1,61 @@ +# Route Options Catalog + +Design agreed from console issue #52 on 2026-10-12. + +```text +DeploySheet -> GET /api/llm/route-options[?route=] + -> configured gateway -> //_route_opts + -> common grouped rows + advanced picker + runtime state +``` + +| Concern | Decision | +|---|---| +| API | Each configured backend exposes GET `/route-options`; gateway backends select resolved ConfigMap routes, while static URLs already name a route | +| Authorization | Console login and `get backends` on the configured backend name | +| Request | Optional existing route; reject invalid names and unknown routes; without one try available routes until a catalog answers | +| Credentials | Do not forward the browser session, cookies or gateway API key to the public engine endpoint | +| Compatibility | 404 for an unsupported engine, missing backend, empty gateway or unavailable route; bundled metadata has no fallback values | +| Failures | Permission, transport and malformed response failures are surfaced, not silently treated as legacy engines | +| Form state | Fetching metadata never changes stored overrides; `value` and `set` describe the current runtime separately | +| Runtime identity | Only show runtime values for the canonical requested upgrade route, including the release-name fallback for older plans | +| Named controls | Preserve chart and SLO semantics, use live placeholders, and avoid duplicate rows | +| Advanced | Engine internals remain behind the add-setting picker, which respects available viewport height | +| i18n | Translate known keys; unknown keys use engine text | +| Swiss | Adapt bundled metadata in inferences without editing the copied Swiss module | + +No CRD, chart value or configuration schema changes. Validation remains in the engine. + +## Regression Coverage + +| Scenario | Signal | +|---|---| +| SPA with no gateway backend | Catalog API returns 404 JSON instead of 200 HTML | +| Host defaults differ from bundled values | Render the live defaults, never copied placeholders | +| Existing route differs from a new deployment | Do not leak the probed route's runtime state into a new form | +| Upgrade without explicit route/serviceId overrides | Use the same canonical route for fetching and display | +| Unknown engine option | Render engine metadata without a console release | +| Optional boolean override | Preserve unset, true and false distinctly | +| Empty advanced setting | Omit the setting rather than sending its displayed default | +| Catalog response | Bound response size, reject malformed data and do not follow redirects | + +## Real Engine Verification + +Verification used a released `4pdosc/llm-openresty:0.1.26` image with an isolated catalog-only route and the existing preview application services. The route is not a working inference pool or a business deployment. + +| Check | Result | +|---|---| +| Engine catalog | 16 common directives and 24 advanced directives | +| Different host defaults | TPS fallback 73 and floor fraction 0.61 appear as placeholders | +| Separate route state | TPS value 81 with set=true is distinct from fallback 73 | +| Proxy | Preserves the engine response; known routes return 200 and unknown routes 404 | +| Browser compose | Returns 200; live defaults and empty advanced settings remain omitted | +| Native mouse interaction | Advanced option is pointer-reachable; selection keeps the form open and the value unset | +| Local checks | Go suite, frontend suite, typecheck and production build pass | + +The standalone browser check requires an auth-disabled preview, a real configured engine and the GLM sample model: + +```sh +CONSOLE_URL=http://127.0.0.1:8080 node hack/check-route-catalog-live.mjs +``` + +It checks live placeholders, viewport containment, pointer hit testing and native selection. Acceptance is desktop-only. No install/apply operation was performed, and existing stored release history was not rewritten. diff --git a/hack/check-route-catalog-live.mjs b/hack/check-route-catalog-live.mjs new file mode 100644 index 0000000..e7094a6 --- /dev/null +++ b/hack/check-route-catalog-live.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; + +const base = process.env.CONSOLE_URL; +assert.ok(base, "Set CONSOLE_URL to an auth-disabled preview with a real engine catalog"); +const session = "route-catalog-live-check"; +function browser(...args) { + const result = JSON.parse(execFileSync("agent-browser", ["--session", session, "--json", ...args], { encoding: "utf8", timeout: 40_000 })); + assert.equal(result.success, true, result.error); + return result.data; +} +const evaluate = (script) => browser("eval", script).result; + +const response = await fetch(`${base}/api/llm/route-options`); +assert.equal(response.status, 200, "A real engine catalog must be configured"); +const catalog = await response.json(); +const options = new Map([...catalog.directives, ...catalog.advanced].map((d) => [d.key, d])); + +try { + browser("open", base); + browser("set", "viewport", process.env.VIEWPORT_WIDTH ?? "1280", process.env.VIEWPORT_HEIGHT ?? "720"); + evaluate('localStorage.setItem("rise-locale", "en-US")'); + browser("open", `${base}/inferences/catalog/glm5.1?catalog=default`); + browser("wait", "--text", "Deploy"); + const defaults = evaluate(`(async () => { + const frame = () => new Promise(requestAnimationFrame); + [...document.querySelectorAll("button")].filter((x) => x.textContent.trim() === "Deploy").at(-1).click(); + for (let i = 0; i < 300 && !document.querySelector("#deploy-sec-routing-section"); i++) await frame(); + const section = document.querySelector("#deploy-sec-routing-section"); + if (!section) throw new Error("Deploy routing section did not render"); + const sheet = section.closest('[data-slot="sheet-content"]'); + const sheetRect = sheet.getBoundingClientRect(); + if (sheetRect.left < 0 || sheetRect.right > innerWidth) throw new Error("Deploy sheet exceeds viewport width"); + section.querySelector("button").click(); + await frame(); + const tpsLabel = [...document.querySelectorAll("label")].find((x) => x.textContent.includes("TPS fallback")); + const tps = document.getElementById(tpsLabel.htmlFor); + const floor = document.querySelector('input[aria-label="Adaptive floor fraction"]'); + const trigger = section.querySelector('button[aria-haspopup="menu"]'); + trigger.scrollIntoView({ block: "center" }); + for (let i = 0; i < 12; i++) await frame(); + trigger.click(); + for (let i = 0; i < 12; i++) await frame(); + const item = [...document.querySelectorAll('[role="menuitem"]')].find((x) => x.textContent.includes("rt_limit_factor")); + const menu = item.closest('[role="menu"]'); + const menuRect = menu.getBoundingClientRect(); + if (menuRect.top < 0 || menuRect.bottom > innerHeight) throw new Error("Advanced menu exceeds available viewport height"); + const initial = item.getBoundingClientRect(); + menu.scrollTop += initial.top - menuRect.top - (menu.clientHeight - initial.height) / 2; + for (let i = 0; i < 12; i++) await frame(); + const rect = item.getBoundingClientRect(); + const x = rect.left + rect.width / 2; + const y = rect.top + rect.height / 2; + if (document.elementFromPoint(x, y)?.closest('[role="menuitem"]') !== item) throw new Error("Advanced item is not pointer reachable"); + return { tps: tps.placeholder, floor: floor.placeholder, tpsValue: tps.value, floorValue: floor.value, x, y }; + })()`); + assert.equal(defaults.tps, options.get("tps_limit_tps").fallback); + assert.equal(defaults.floor, options.get("adaptive_cc_min_frac").fallback); + assert.equal(defaults.tpsValue, ""); + assert.equal(defaults.floorValue, ""); + browser("mouse", "move", String(Math.round(defaults.x)), String(Math.round(defaults.y))); + browser("mouse", "down"); + browser("mouse", "up"); + const selected = evaluate(`(() => { + const input = document.querySelector('input[id$="-rt_limit_factor"]'); + if (!input) throw new Error("Native pointer selection did not add the option"); + return { value: input.value, placeholder: input.placeholder }; + })()`); + assert.equal(selected.value, ""); + assert.equal(selected.placeholder, options.get("rt_limit_factor").fallback); + console.log(JSON.stringify({ realCatalog: true, route: catalog.route, hostDefaults: { tps: defaults.tps, floor: defaults.floor }, viewportAndNativePointer: true, unsetAdvancedPreserved: true })); +} finally { + browser("close"); +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 487ebea..c1dffe2 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -52,6 +52,8 @@ type Entry struct { KeySource string } +var ErrNoRoutes = fmt.Errorf("no session_route_.conf entry") + // refresh is how long a resolved entrypoint is reused. Long enough that a busy // Playground does not read the same two ConfigMaps per request, short enough that // a deploy shows up without touching the console. @@ -89,7 +91,7 @@ func (r *Resolver) Resolve(ctx context.Context) (*Entry, error) { return nil, r.err } - entry, err := r.resolve(ctx) + entry, err := r.resolve(ctx, false) r.at = time.Now() switch { case err != nil && r.entry != nil: @@ -111,7 +113,12 @@ func (r *Resolver) Resolve(ctx context.Context) (*Entry, error) { return entry, nil } -func (r *Resolver) resolve(ctx context.Context) (*Entry, error) { +// ResolveCatalog discovers existing routes without inference pinning or credentials. +func (r *Resolver) ResolveCatalog(ctx context.Context) (*Entry, error) { + return r.resolve(ctx, true) +} + +func (r *Resolver) resolve(ctx context.Context, catalog bool) (*Entry, error) { gw := r.backend.Gateway if gw == nil { return nil, fmt.Errorf("backend %s is not a gateway", r.backend.Name) @@ -141,7 +148,7 @@ func (r *Resolver) resolve(ctx context.Context) (*Entry, error) { port = config.DefaultGatewayPort } - routes, source, err := r.routes(ctx, profile, profileRef) + routes, source, err := r.routes(ctx, profile, profileRef, catalog) if err != nil { return nil, err } @@ -155,23 +162,22 @@ func (r *Resolver) resolve(ctx context.Context) (*Entry, error) { Header: profile.Route.Auth.HeaderName(), Source: source, } - if err := r.credential(ctx, profile, svcNS, entry); err != nil { - return nil, err + if !catalog { + if err := r.credential(ctx, profile, svcNS, entry); err != nil { + return nil, err + } } return entry, nil } -// routes lists the routes to talk to: the explicit one, else every route in the -// route ConfigMap, aggregate ones (those serving several models) first. autoconfig -// writes one plain route per model, so a gateway with no aggregate route is the -// normal case, not an error. -func (r *Resolver) routes(ctx context.Context, p profile, profileRef string) (routes []string, source string, err error) { +// routes lists inference routes, or all configured routes for catalog discovery. +func (r *Resolver) routes(ctx context.Context, p profile, profileRef string, catalog bool) (routes []string, source string, err error) { gw := r.backend.Gateway ref := gw.ConfigMap if ref == "" { ref = p.Route.NginxConfigMap } - if gw.Route != "" { + if gw.Route != "" && (!catalog || ref == "") { source = ref if source == "" { source = profileRef @@ -179,6 +185,9 @@ func (r *Resolver) routes(ctx context.Context, p profile, profileRef string) (ro return []string{gw.Route}, source, nil } if ref == "" { + if catalog { + return nil, "", ErrNoRoutes + } return nil, "", fmt.Errorf("no gateway.route and no route configmap to choose one from: set gateway.route") } ns, name, err := splitRef(ref, "") @@ -203,7 +212,7 @@ func (r *Resolver) routes(ctx context.Context, p profile, profileRef string) (ro } } if len(aggregates)+len(plain) == 0 { - return nil, "", fmt.Errorf("route configmap %s has no session_route_.conf entry; is this the entrypoint's shared configmap?", ref) + return nil, "", fmt.Errorf("route configmap %s has %w; is this the entrypoint's shared configmap?", ref, ErrNoRoutes) } sort.Strings(aggregates) sort.Strings(plain) diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 831f156..ecfb9a2 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "errors" "log/slog" "slices" "strings" @@ -99,6 +100,76 @@ func resolveOne(t *testing.T, cfg *config.Gateway, c *fakeCluster) (*Entry, erro return NewResolver(b, c, slog.New(slog.DiscardHandler)).Resolve(t.Context()) } +func TestResolveCatalogIndependentOfInferencePin(t *testing.T) { + for _, pin := range []string{"video", "stale"} { + t.Run(pin, func(t *testing.T) { + c := fixture() + b := config.Backend{Name: "llm", Gateway: &config.Gateway{Profile: "llm/site-profile", Route: pin}} + r := NewResolver(b, c, slog.New(slog.DiscardHandler)) + inference, err := r.Resolve(t.Context()) + if err != nil { + t.Fatal(err) + } + catalog, err := r.ResolveCatalog(t.Context()) + if err != nil { + t.Fatal(err) + } + if catalog.Base != "http://openresty.llm.svc:8080" || !slices.Equal(catalog.Routes, []string{"llm-gateway", "glm-5.1", "kimi-k2.6"}) || catalog.Key != "" { + t.Fatalf("catalog = %+v", catalog) + } + again, err := r.Resolve(t.Context()) + if err != nil || again != inference || !slices.Equal(again.Routes, []string{pin}) || again.Key != "Bearer k1" { + t.Fatalf("inference pin or credential changed: %+v, %v", again, err) + } + }) + } +} + +func TestResolveCatalogWithoutCredentialSecret(t *testing.T) { + c := fixture() + c.secrets = nil + t.Setenv("CONSOLE_TEST_CATALOG_KEY", "secret") + for _, keyEnv := range []string{"", "CONSOLE_TEST_CATALOG_KEY"} { + b := config.Backend{Name: "llm", APIKeyEnv: keyEnv, Gateway: &config.Gateway{Profile: "llm/site-profile"}} + entry, err := NewResolver(b, c, slog.New(slog.DiscardHandler)).ResolveCatalog(t.Context()) + if err != nil || entry.Key != "" || entry.KeySource != "" { + t.Fatalf("catalog required or exposed a credential: %+v, %v", entry, err) + } + } +} + +func TestResolveCatalogRouteSources(t *testing.T) { + for _, tc := range []struct { + name string + gw config.Gateway + routes []string + err error + }{ + {"explicit without configmap", config.Gateway{Service: "llm/openresty", Route: "model", Port: 9000}, []string{"model"}, nil}, + {"no route source", config.Gateway{Service: "llm/openresty"}, nil, ErrNoRoutes}, + {"empty configmap despite pin", config.Gateway{Service: "llm/openresty", ConfigMap: "llm/empty-conf", Route: "stale"}, nil, ErrNoRoutes}, + {"missing configmap despite pin", config.Gateway{Service: "llm/openresty", ConfigMap: "llm/missing", Route: "stale"}, nil, errNotFound}, + {"explicit configmap overrides profile", config.Gateway{Profile: "llm/site-profile", ConfigMap: "llm/many-routes-conf", Route: "stale"}, []string{"a", "b"}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + b := config.Backend{Name: "llm", Gateway: &tc.gw} + entry, err := NewResolver(b, fixture(), slog.New(slog.DiscardHandler)).ResolveCatalog(t.Context()) + if tc.err != nil { + if !errors.Is(err, tc.err) { + t.Fatalf("err = %v, want %v", err, tc.err) + } + return + } + if err != nil || !slices.Equal(entry.Routes, tc.routes) { + t.Fatalf("entry = %+v, err = %v", entry, err) + } + if tc.gw.Port == 9000 && entry.Base != "http://openresty.llm.svc:9000" { + t.Fatalf("explicit port ignored: %+v", entry) + } + }) + } +} + func TestResolveFromProfile(t *testing.T) { entry, err := resolveOne(t, &config.Gateway{Profile: "llm/site-profile"}, fixture()) if err != nil { diff --git a/internal/server/proxy.go b/internal/server/proxy.go index e986885..3ec90c0 100644 --- a/internal/server/proxy.go +++ b/internal/server/proxy.go @@ -56,12 +56,22 @@ var unresolved = &backendTarget{url: url.URL{Scheme: "http", Host: "backend-with // is skipped and logged rather than taking the server down. func (s *Server) mountBackends(mux *http.ServeMux) { targets := s.backendTargets() + catalogMounted := false for _, b := range s.cfg.Backends { resolve, ok := targets[b.Name] if !ok { continue } proxy := s.backendProxy(b) + prefix := strings.TrimSuffix(b.Prefix, "/") + catalogMounted = catalogMounted || prefix == "/api/llm" + catalogResolve := resolve + if b.Gateway != nil { + catalogResolve = gatewayResolver(gateway.NewResolver(b, s.kube, s.log).ResolveCatalog) + } + mux.HandleFunc(prefix+"/route-options", func(w http.ResponseWriter, r *http.Request) { + s.handleRouteOptions(w, r, b, catalogResolve) + }) mux.HandleFunc(strings.TrimSuffix(b.Prefix, "/")+"/", func(w http.ResponseWriter, r *http.Request) { if !s.authorize(w, r, backendVerb(r.Method), backendResource, b.Name) { return @@ -82,6 +92,16 @@ func (s *Server) mountBackends(mux *http.ServeMux) { proxy.ServeHTTP(w, r.WithContext(withTarget(r.Context(), target))) }) } + if !catalogMounted { + mux.HandleFunc("/api/llm/route-options", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", "GET") + writeError(w, http.StatusMethodNotAllowed, "GET required") + return + } + writeError(w, http.StatusNotFound, "no gateway backend configured") + }) + } } // backendTargets resolves each backend once for the process: a gateway resolver @@ -110,7 +130,7 @@ func (s *Server) targetFunc(b config.Backend) (targetResolver, error) { if s.kube == nil { return nil, fmt.Errorf("gateway %+v needs cluster access, and this console has none", *b.Gateway) } - return gatewayResolver(gateway.NewResolver(b, s.kube, s.log)), nil + return gatewayResolver(gateway.NewResolver(b, s.kube, s.log).Resolve), nil } target, err := url.Parse(b.URL) @@ -128,9 +148,9 @@ func (s *Server) targetFunc(b config.Backend) (targetResolver, error) { return func(context.Context) (*backendTarget, error) { return fixed, nil }, nil } -func gatewayResolver(r *gateway.Resolver) targetResolver { +func gatewayResolver(resolve func(context.Context) (*gateway.Entry, error)) targetResolver { return func(ctx context.Context) (*backendTarget, error) { - entry, err := r.Resolve(ctx) + entry, err := resolve(ctx) if err != nil { return nil, err } diff --git a/internal/server/route_options.go b/internal/server/route_options.go new file mode 100644 index 0000000..6b074a7 --- /dev/null +++ b/internal/server/route_options.go @@ -0,0 +1,110 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "slices" + "strings" + + "github.com/modelsphere/console/internal/config" + "github.com/modelsphere/console/internal/gateway" +) + +var routeOptionName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) + +func (s *Server) handleRouteOptions(w http.ResponseWriter, r *http.Request, b config.Backend, resolve targetResolver) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", "GET") + writeError(w, http.StatusMethodNotAllowed, "GET required") + return + } + if !s.authorize(w, r, "get", backendResource, b.Name) { + return + } + route := r.URL.Query().Get("route") + if route != "" && !routeOptionName.MatchString(route) { + writeError(w, http.StatusBadRequest, "invalid route name") + return + } + ctx, cancel := context.WithTimeout(r.Context(), probeTimeout) + defer cancel() + target, err := resolve(ctx) + if err != nil { + if errors.Is(err, gateway.ErrNoRoutes) { + writeError(w, http.StatusNotFound, "no route catalog available") + return + } + writeError(w, http.StatusBadGateway, fmt.Sprintf("backend %s: %v", b.Name, err)) + return + } + routes := target.routes + if b.Gateway == nil { + if route != "" && route != strings.TrimPrefix(target.url.Path, "/") { + writeError(w, http.StatusNotFound, "route not served by backend") + return + } + routes = []string{""} + } else if route != "" { + if !slices.Contains(routes, route) { + writeError(w, http.StatusNotFound, "route not served by backend") + return + } + routes = []string{route} + } + client := &http.Client{Timeout: probeTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + failed := false + for _, name := range routes { + body, status, err := readRouteOptions(ctx, client, routeURL(target, name)+"/_route_opts") + if status == http.StatusNotFound { + continue + } + if err != nil || status != http.StatusOK { + failed = true + continue + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _, _ = w.Write(body) + return + } + if failed { + writeError(w, http.StatusBadGateway, "gateway route catalog unavailable or invalid") + return + } + writeError(w, http.StatusNotFound, "no route catalog available") +} + +func readRouteOptions(ctx context.Context, client *http.Client, endpoint string) ([]byte, int, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, 0, err + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return nil, 0, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, resp.StatusCode, nil + } + const max = 1 << 20 + body, err := io.ReadAll(io.LimitReader(resp.Body, max+1)) + if err != nil { + return nil, resp.StatusCode, err + } + var catalog struct { + Route string `json:"route"` + Directives []json.RawMessage `json:"directives"` + Advanced []json.RawMessage `json:"advanced"` + } + if len(body) > max || json.Unmarshal(body, &catalog) != nil || catalog.Route == "" || catalog.Directives == nil || catalog.Advanced == nil { + return nil, resp.StatusCode, fmt.Errorf("invalid route catalog") + } + return body, resp.StatusCode, nil +} diff --git a/internal/server/route_options_test.go b/internal/server/route_options_test.go new file mode 100644 index 0000000..e1a85d5 --- /dev/null +++ b/internal/server/route_options_test.go @@ -0,0 +1,244 @@ +package server + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "testing/fstest" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/modelsphere/console/internal/config" +) + +const routeOptionsFixture = `{"route":"model","directives":[{"key":"tps_limit_tps","label":"Decode rate threshold","group":"Thresholds","kind":"number","fallback":"73","detail":"Deployment default","value":81,"set":true}],"advanced":[],"peers":{"count":3},"adaptive_cc_derived":{"_":{"static_max":200,"min":80,"init":80}}}` + +type routeOptionsTransport func(*http.Request) (*http.Response, error) + +func (f routeOptionsTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestRouteOptionsDiscoversCatalogDespiteInferencePin(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" || r.Header.Get("X-Remote-User") != "" { + t.Error("credentials or identity forwarded to public route catalog") + } + if r.URL.Path != "/model/_route_opts" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer up.Close() + base, _ := url.Parse(up.URL) + transport := http.DefaultTransport + http.DefaultTransport = routeOptionsTransport(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "openresty.llm.svc:8080" { + t.Errorf("gateway host = %q", r.URL.Host) + } + clone := r.Clone(r.Context()) + clone.URL.Scheme, clone.URL.Host = base.Scheme, base.Host + return transport.RoundTrip(clone) + }) + t.Cleanup(func() { http.DefaultTransport = transport }) + for _, tc := range []struct { + name string + gw config.Gateway + }{ + {"video", config.Gateway{Profile: "llm/site-profile", Route: "video"}}, + {"stale", config.Gateway{Profile: "llm/site-profile", Route: "stale"}}, + {"explicit without configmap", config.Gateway{Service: "llm/openresty", Route: "model", SecretRef: "llm/missing-keys"}}, + } { + t.Run(tc.name, func(t *testing.T) { + objects := gatewayObjects(t)[:2] + objects[1].(*unstructured.Unstructured).Object["data"] = map[string]any{ + "session_route_video.conf": "peers_by_model = {}", + "session_route_model.conf": "set $route model;", + } + cfg := &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", Gateway: &tc.gw}}} + h := testServerWithConfig(t, cfg, objects...).Handler() + admin := login(t, h, "admin", "admin-pw") + for _, path := range []string{"/api/llm/route-options", "/api/llm/route-options?route=model"} { + rec := do(h, "GET", path, admin, "") + if rec.Code != http.StatusOK || rec.Body.String() != routeOptionsFixture { + t.Fatalf("catalog with %q = %d %s", tc.name, rec.Code, rec.Body.String()) + } + } + if rec := do(h, "GET", "/api/llm/route-options?route=stale", admin, ""); rec.Code != http.StatusNotFound { + t.Fatalf("unknown route = %d %s", rec.Code, rec.Body.String()) + } + }) + } +} + +func TestRouteOptionsWithoutGatewayDoesNotServeSPA(t *testing.T) { + cfg := &config.Config{} + cfg.Server.Auth.Disabled = true + srv := testServerWithConfig(t, cfg) + srv.SetWeb(fstest.MapFS{"index.html": {Data: []byte("SPA")}}) + rec := do(srv.Handler(), "GET", "/api/llm/route-options", "", "") + if rec.Code != http.StatusNotFound || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") { + t.Fatalf("missing gateway must select legacy fallback, got %d %s: %s", rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) + } +} + +func TestRouteOptionsLiveCatalog(t *testing.T) { + var paths []string + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" || r.Header.Get("X-Remote-User") != "" { + t.Error("credentials or identity forwarded to public route catalog") + } + if r.URL.Path == "/video/_route_opts" { + http.NotFound(w, r) + return + } + if r.URL.Path != "/model/_route_opts" { + t.Errorf("path = %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer up.Close() + base, _ := url.Parse(up.URL) + b := config.Backend{Name: "llm", Prefix: "/api/llm", Gateway: &config.Gateway{ConfigMap: "llm/routes", Service: "llm/openresty"}} + cfg := &config.Config{Backends: []config.Backend{b}} + cfg.Server.Auth.Disabled = true + srv := testServerWithConfig(t, cfg) + resolve := func(context.Context) (*backendTarget, error) { + return &backendTarget{url: *base, routes: []string{"video", "model"}, key: "secret", header: "Authorization"}, nil + } + h := srv.authenticate(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv.handleRouteOptions(w, r, b, resolve) })) + for _, path := range []string{"/api/llm/route-options", "/api/llm/route-options?route=model"} { + rec := do(h, "GET", path, "", "") + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"fallback":"73"`) || !strings.Contains(rec.Body.String(), `"value":81`) { + t.Fatalf("live catalog = %d %s", rec.Code, rec.Body.String()) + } + } + if strings.Join(paths, ",") != "/video/_route_opts,/model/_route_opts,/model/_route_opts" { + t.Fatalf("paths = %v", paths) + } +} + +func TestRouteOptionsEmptyGatewayAndResolutionFailure(t *testing.T) { + for _, tc := range []struct { + ref string + want int + }{ + {"", http.StatusNotFound}, + {"llm/openresty-conf", http.StatusNotFound}, + {"llm/missing-conf", http.StatusBadGateway}, + } { + t.Run(tc.ref, func(t *testing.T) { + objects := gatewayObjects(t) + objects[1].(*unstructured.Unstructured).Object["data"] = map[string]any{} + cfg := &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", Gateway: &config.Gateway{ConfigMap: tc.ref, Service: "llm/openresty"}}}} + h := testServerWithConfig(t, cfg, objects...).Handler() + admin := login(t, h, "admin", "admin-pw") + rec := do(h, "GET", "/api/llm/route-options", admin, "") + if rec.Code != tc.want { + t.Fatalf("catalog = %d, want %d: %s", rec.Code, tc.want, rec.Body.String()) + } + }) + } +} + +func TestRouteOptionsReadGrantAndStaticRoute(t *testing.T) { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/model/_route_opts" { + t.Errorf("path = %s", r.URL.Path) + } + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer up.Close() + h := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", URL: up.URL + "/model"}}}).Handler() + admin, bob := login(t, h, "admin", "admin-pw"), login(t, h, "bob", "bob-pw") + for _, tc := range []struct{ path, body string }{ + {"/api/iam/roles", `{"name":"catalog-reader","rules":[{"verbs":["get"],"apiGroups":["iam.theriseunion.io"],"resources":["backends"],"resourceNames":["llm"]}]}`}, + {"/api/iam/rolebindings", `{"name":"bob-catalog-reader","role":"catalog-reader","subjects":[{"kind":"User","name":"bob"}]}`}, + } { + if rec := do(h, "POST", tc.path, admin, tc.body); rec.Code != http.StatusCreated { + t.Fatalf("grant: %d %s", rec.Code, rec.Body.String()) + } + } + if rec := do(h, "GET", "/api/llm/route-options?route=model", bob, ""); rec.Code != http.StatusOK || rec.Body.String() != routeOptionsFixture { + t.Fatalf("read grant: %d %s", rec.Code, rec.Body.String()) + } + if rec := do(h, "GET", "/api/llm/route-options?route=another", bob, ""); rec.Code != http.StatusNotFound { + t.Fatalf("unknown route: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestRouteOptionsDoesNotFollowRedirects(t *testing.T) { + redirected := 0 + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + redirected++ + _, _ = w.Write([]byte(routeOptionsFixture)) + })) + defer other.Close() + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, other.URL, http.StatusFound) })) + defer up.Close() + h := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", URL: up.URL + "/model"}}}).Handler() + admin := login(t, h, "admin", "admin-pw") + rec := do(h, "GET", "/api/llm/route-options", admin, "") + if rec.Code != http.StatusBadGateway || redirected != 0 { + t.Fatalf("redirect: %d, followed %d times", rec.Code, redirected) + } +} + +func TestReadRouteOptionsBoundsAndShape(t *testing.T) { + for _, body := range []string{"{}", `{"route":"model","directives":null,"advanced":[]}`, `{"route":"model","directives":[],"advanced":{}}`, strings.Repeat(" ", 1<<20) + routeOptionsFixture} { + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(body)) })) + _, _, err := readRouteOptions(t.Context(), up.Client(), up.URL) + up.Close() + if err == nil { + t.Fatal("invalid or oversized catalog accepted") + } + } +} + +func TestRouteOptionsAccessAndFailures(t *testing.T) { + for _, status := range []int{200, 404, 401, 500} { + t.Run(http.StatusText(status), func(t *testing.T) { + hits := 0 + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.WriteHeader(status) + if status == 200 { + _, _ = w.Write([]byte("not json")) + } + })) + defer up.Close() + srv := testServerWithConfig(t, &config.Config{Backends: []config.Backend{{Name: "llm", Prefix: "/api/llm", URL: up.URL + "/model"}}}) + h := srv.Handler() + admin, bob := login(t, h, "admin", "admin-pw"), login(t, h, "bob", "bob-pw") + for _, tc := range []struct { + method, path, token string + want int + }{ + {"GET", "/api/llm/route-options", "", 401}, + {"GET", "/api/llm/route-options", bob, 403}, + {"POST", "/api/llm/route-options", admin, 405}, + {"GET", "/api/llm/route-options?route=..%2Felsewhere", admin, 400}, + } { + rec := do(h, tc.method, tc.path, tc.token, "") + if rec.Code != tc.want { + t.Fatalf("%s %s = %d, want %d", tc.method, tc.path, rec.Code, tc.want) + } + } + if hits != 0 { + t.Fatalf("rejected requests reached upstream: %d", hits) + } + rec := do(h, "GET", "/api/llm/route-options", admin, "") + want := http.StatusBadGateway + if status == 404 { + want = 404 + } + if rec.Code != want { + t.Fatalf("status = %d, want %d: %s", rec.Code, want, rec.Body.String()) + } + }) + } +} diff --git a/web/src/modules/inferences/api.test.ts b/web/src/modules/inferences/api.test.ts new file mode 100644 index 0000000..d236f31 --- /dev/null +++ b/web/src/modules/inferences/api.test.ts @@ -0,0 +1,68 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { readRouteOptions } from "./api"; + +const fetchMock = vi.fn(); +const catalog = { + route: "model", + directives: [{ key: "new_engine_option", label: "New option", group: "Future group", kind: "number", fallback: "73", detail: "Engine help", value: 81, set: true }], + advanced: [], + peers: { count: 3 }, + adaptive_cc_derived: { _: { static_max: 200, min: 80, init: 80 } }, +}; +const response = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status }); + +beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal("fetch", fetchMock); + vi.stubGlobal("document", { cookie: "" }); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe("route options transport", () => { + it("reads an existing route, preserves host defaults and derived state, and accepts unknown options", async () => { + fetchMock.mockResolvedValue(response(catalog)); + expect(await readRouteOptions("model.with-dots")).toEqual(catalog); + expect(fetchMock.mock.calls[0][0]).toBe("/api/llm/route-options?route=model.with-dots"); + }); + it("uses a catalog-only probe for a new deployment", async () => { + fetchMock.mockResolvedValue(response(catalog)); + await readRouteOptions(); + expect(fetchMock.mock.calls[0][0]).toBe("/api/llm/route-options"); + }); + it("uses legacy metadata only for a 404", async () => { + fetchMock.mockResolvedValue(response({ error: "no catalog" }, 404)); + expect(await readRouteOptions()).toBeNull(); + }); + it.each([401, 403, 500, 502, 503])("surfaces HTTP %s rather than silently using legacy defaults", async (status) => { + fetchMock.mockResolvedValue(response({ error: "catalog failed" }, status)); + await expect(readRouteOptions()).rejects.toMatchObject({ status, message: "catalog failed" }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it("surfaces non-JSON API failures", async () => { + fetchMock.mockResolvedValue(new Response("Bad Gateway", { status: 502, statusText: "Bad Gateway" })); + await expect(readRouteOptions()).rejects.toMatchObject({ status: 502, message: "Bad Gateway" }); + }); + it("surfaces network failures instead of reporting a legacy catalog", async () => { + fetchMock.mockRejectedValue(new TypeError("Network unavailable")); + await expect(readRouteOptions()).rejects.toThrow("Network unavailable"); + }); + it("surfaces malformed JSON instead of reporting a legacy catalog", async () => { + fetchMock.mockResolvedValue(new Response("not JSON")); + await expect(readRouteOptions()).rejects.toBeInstanceOf(SyntaxError); + }); + it("forwards query cancellation to the transport", async () => { + const controller = new AbortController(); + fetchMock.mockResolvedValue(response(catalog)); + await readRouteOptions("model", controller.signal); + expect(fetchMock.mock.calls[0][1].signal).toBe(controller.signal); + }); + it.each([ + {}, { ...catalog, route: "" }, { ...catalog, directives: null }, { ...catalog, advanced: null }, + { ...catalog, directives: [{ ...catalog.directives[0], value: {} }] }, + { ...catalog, directives: [{ ...catalog.directives[0], set: "false" }] }, + { ...catalog, advanced: catalog.directives }, + ])("rejects invalid catalog data", async (body) => { + fetchMock.mockResolvedValue(response(body)); + await expect(readRouteOptions()).rejects.toThrow("Invalid route options response"); + }); +}); diff --git a/web/src/modules/inferences/api.ts b/web/src/modules/inferences/api.ts new file mode 100644 index 0000000..73963a0 --- /dev/null +++ b/web/src/modules/inferences/api.ts @@ -0,0 +1,33 @@ +import { apiFetch, ApiError } from "@/shell"; +import type { RouteDirective, RouteOptions } from "./routeOptions"; + +function isRecord(v: unknown): v is Record { + return !!v && typeof v === "object" && !Array.isArray(v); +} +function isDirective(v: unknown): v is RouteDirective { + if (!isRecord(v)) return false; + return ["key", "label", "group", "fallback", "detail"].every((k) => typeof v[k] === "string") + && ["number", "boolean", "text"].includes(String(v.kind)) + && typeof v.set === "boolean" + && (v.needsAdaptive === undefined || typeof v.needsAdaptive === "boolean") + && (v.value === null || ["number", "boolean", "string"].includes(typeof v.value)); +} + +export async function readRouteOptions(route?: string, signal?: AbortSignal): Promise { + const query = route ? `?${new URLSearchParams({ route })}` : ""; + const res = await apiFetch(`/api/llm/route-options${query}`, { signal }); + if (res.status === 404) return null; + if (!res.ok) { + const body = await res.json().catch(() => null); + throw new ApiError(res.status, typeof body?.error === "string" ? body.error : res.statusText); + } + const body: unknown = await res.json(); + if (!isRecord(body) || typeof body.route !== "string" || !body.route + || !Array.isArray(body.directives) || !body.directives.every(isDirective) + || !Array.isArray(body.advanced) || !body.advanced.every(isDirective)) { + throw new Error("Invalid route options response"); + } + const keys = [...body.directives, ...body.advanced].map((d) => d.key); + if (new Set(keys).size !== keys.length) throw new Error("Invalid route options response"); + return body as unknown as RouteOptions; +} diff --git a/web/src/modules/inferences/components/DeployForm.test.tsx b/web/src/modules/inferences/components/DeployForm.test.tsx new file mode 100644 index 0000000..edea55b --- /dev/null +++ b/web/src/modules/inferences/components/DeployForm.test.tsx @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { DeployForm } from "./DeployForm"; +import { EMPTY, planRequest, type Form } from "@swiss/components/DeploySettings"; +import type { RouteOptions } from "@/modules/inferences/routeOptions"; + +const catalog: RouteOptions = { + route: "model", + directives: [], + advanced: [{ key: "rt_limit_factor", label: "Runtime limit factor", group: "Capacity & failover", kind: "number", fallback: "7.25", detail: "Scales the pool limit.", needsAdaptive: false, value: 9, set: true }], +}; + +function render(routeOptions?: RouteOptions | null, upgrade = false, form: Form = { ...EMPTY, serviceId: "model", nginxExtras: [{ key: "rt_limit_factor", value: "" }] }, catalogRoute?: string) { + const props = { + form, upgrade, + onChange: () => {}, submitted: false, target: null, defaults: {}, gpuOptions: [], + expanded: { basic: false, resources: false, routing: true, advanced: false }, + onExpand: () => {}, idPrefix: "test-", routeOptions, catalogRoute, + }; + return renderToStaticMarkup(); +} + +describe("deploy route catalog", () => { + it("shows the running engine's default instead of the bundled default", () => { + expect(render(catalog).includes('placeholder="7.25"')).toBe(true); + }); + + it("shows bundled controls only for an explicit legacy catalog result", () => { + const form = { ...EMPTY, serviceId: "model" }; + expect(render(undefined, false, form).includes("adaptive_cc_init")).toBe(false); + expect(render(null, false, form).includes("adaptive_cc_init")).toBe(true); + }); + + it.each([undefined, null])("preserves arbitrary rows and named overrides when catalog data is %s", (options) => { + const form = { ...EMPTY, serviceId: "model", adaptiveCc: true, adaptiveCcMinFrac: "0.73", tpsLimitTps: "97", ttftLimitMs: "123", nginxExtras: [{ key: "future_knob", value: "keep-me" }, { key: "rt_limit_factor", value: "2" }] }; + const before = planRequest(form, { model: "model" }); + const html = render(options, true, form); + for (const value of ["keep-me", "2", "0.73", "97", "123"]) expect(html).toContain(`value="${value}"`); + expect(html).toContain("future_knob"); + expect(planRequest(form, { model: "model" })).toEqual(before); + }); + + it("does not show stale bundled defaults without a live catalog", () => { + expect(render(null).includes('placeholder="1"')).toBe(false); + expect(render(null).includes("0.4")).toBe(false); + }); + + it("does not show another route's runtime values on a new deployment or an unmatched upgrade", () => { + expect(render(catalog).includes("当前：9")).toBe(false); + expect(render({ ...catalog, route: "another" }, true).includes("当前：9")).toBe(false); + expect(render(catalog, true).includes("当前：9")).toBe(true); + expect(render(catalog, true).includes("路由配置")).toBe(true); + }); + + it("shows runtime state for a release fallback when the old plan has no explicit route identity", () => { + const legacyForm = { ...EMPTY, nginxExtras: [{ key: "rt_limit_factor", value: "" }] }; + expect(render(catalog, true, legacyForm, "model").includes("当前：9")).toBe(true); + expect(render(catalog, true, legacyForm, "another").includes("当前：9")).toBe(false); + expect(render(catalog, false, legacyForm, "model").includes("当前：9")).toBe(false); + }); + + it("renders new engine directives without a console release and leaves advanced defaults out of the default view", () => { + const live: RouteOptions = { ...catalog, directives: [{ key: "future_knob", label: "New engine knob", group: "New engine group", kind: "text", fallback: "dynamic default", detail: "New engine help", value: "current", set: false }] }; + const html = render(live); + expect(html.includes("New engine knob")).toBe(true); + expect(html.includes("New engine help")).toBe(true); + expect(html.includes('placeholder="dynamic default"')).toBe(true); + expect(html.includes("New engine group")).toBe(true); + const emptyForm = { ...EMPTY, serviceId: "model" }; + expect(render(live, false, emptyForm).includes('placeholder="7.25"')).toBe(false); + }); + + it("uses host defaults for named fields and does not mutate stored overrides when showing runtime state", () => { + const form = { ...EMPTY, serviceId: "model", route: "model", nginxExtras: [{ key: "rt_limit_factor", value: "2" }] }; + const before = planRequest(form, { model: "model" }); + const live: RouteOptions = { ...catalog, directives: [ + { key: "tps_limit_tps", label: "TPS", group: "Thresholds", kind: "number", fallback: "73", detail: "Engine threshold help", value: 81, set: true }, + { key: "adaptive_cc_min_frac", label: "Floor fraction", group: "Adaptive concurrency", kind: "number", fallback: "0.61", detail: "Engine fraction help", value: 0.61, set: false }, + ] }; + const html = render(live, true, form); + expect(html.includes('placeholder="73"')).toBe(true); + expect(html.includes('placeholder="0.61"')).toBe(true); + expect(html.includes('value="2"')).toBe(true); + expect(html.includes("当前：81")).toBe(true); + expect(html.includes("引擎默认值或派生值")).toBe(true); + expect(planRequest(form, { model: "model" })).toEqual(before); + }); +}); diff --git a/web/src/modules/inferences/components/DeployForm.tsx b/web/src/modules/inferences/components/DeployForm.tsx index 9bf3edd..22fddbf 100644 --- a/web/src/modules/inferences/components/DeployForm.tsx +++ b/web/src/modules/inferences/components/DeployForm.tsx @@ -1,4 +1,4 @@ -import { useState, type ReactNode } from "react"; +import { useId, useState, type ReactNode } from "react"; import { Badge, Button, @@ -23,9 +23,10 @@ import { import { Plus, X } from "lucide-react"; import type { ClusterInfo } from "@swiss/lib/api"; import { effective, type Form, type Toleration } from "@swiss/components/DeploySettings"; -import { DIRECTIVES, DIRECTIVES_BY_KEY, MAX_VALUES, directiveGroups, rejectKey } from "@swiss/lib/routeDirectives"; +import { MAX_VALUES, rejectKey } from "@swiss/lib/routeDirectives"; +import { FALLBACK_ROUTE_OPTIONS, routeDirectives, runtimeValue, type RouteDirective, type RouteOptions } from "@/modules/inferences/routeOptions"; import { useT } from "@/modules/inferences/i18n"; -import { NAMED_VALUES, extraValue, fracOutOfRange, initialValue, serviceIdError, type Section } from "@/modules/inferences/deploy-lib"; +import { NAMED_VALUES, extraValue, fracOutOfRange, serviceIdError, type Section } from "@/modules/inferences/deploy-lib"; const EFFECTS = ["", "NoSchedule", "PreferNoSchedule", "NoExecute"]; const digits = (v: string) => v.replace(/\D/g, ""); @@ -42,12 +43,22 @@ export interface DeployFormProps { expanded: Record; onExpand: (s: Section, open: boolean) => void; idPrefix: string; + routeOptions?: RouteOptions | null; + catalogRoute?: string; } -export function DeployForm({ form, onChange, upgrade, submitted, target, cluster, defaults, gpuOptions, expanded, onExpand, idPrefix }: DeployFormProps) { +export function DeployForm({ form, onChange, upgrade, submitted, target, cluster, defaults, gpuOptions, expanded, onExpand, idPrefix, routeOptions, catalogRoute }: DeployFormProps) { const t = useT(); const on = effective(form); const f = (k: string) => t(`deploy.fields.${k}`); + const byKey = new Map(routeDirectives(routeOptions).map((d) => [d.key, d])); + const showRuntime = !!upgrade && routeOptions?.route === (catalogRoute ?? (form.route || form.serviceId)); + const label = (key: string, fallback: string) => directiveText(t, byKey.get(key), "label") || f(fallback); + const hint = (key: string, fallback: string) => { + const d = byKey.get(key); + const detail = routeOptions ? directiveText(t, d, "detail") || f(fallback) : f(fallback); + return showRuntime && d?.set !== undefined ? `${detail} · ${runtimeText(t, d)}` : detail; + }; const text = (k: keyof Form) => ({ value: form[k] as string, onChange: (e: React.ChangeEvent) => onChange({ [k]: e.target.value }) }); const count = (k: "replicaCount" | "scalerMin" | "scalerMax") => ({ value: form[k], inputMode: "numeric" as const, onChange: (e: React.ChangeEvent) => onChange({ [k]: digits(e.target.value) }) }); const sw = (k: keyof Form, label: string) => onChange({ [k]: v })} />; @@ -151,7 +162,7 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster > onChange(v ? { modelRoute: true } : { modelRoute: false, route: "" })} /> - + {sw("exposeRoutedPeer", f("exposeRoutedPeer"))} @@ -170,22 +181,22 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster > {sw("cart", f("cart"))} - + {sw("slo", f("slo"))}
- - + + - - + +
@@ -193,13 +204,13 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster value={form.adaptiveCcMinFrac} onChange={(e) => onChange({ adaptiveCcMinFrac: e.target.value })} inputMode="decimal" - placeholder={`${f("adaptiveCcMinFrac")} · 0.4`} + placeholder={byKey.get("adaptive_cc_min_frac")?.fallback || undefined} aria-label={f("adaptiveCcMinFrac")} aria-invalid={fracOutOfRange(form.adaptiveCcMinFrac)} disabled={!!extraValue(form, "adaptive_cc_min")} />

- {fracOutOfRange(form.adaptiveCcMinFrac) ? f("fracRange") : f("adaptiveCcMinFracHint")} + {fracOutOfRange(form.adaptiveCcMinFrac) ? f("fracRange") : hint("adaptive_cc_min_frac", "adaptiveCcMinFracHint")}

} @@ -260,78 +271,98 @@ function Tolerations({ rows, onChange }: { rows: Toleration[]; onChange: (v: Tol ); } -function RouteExtras({ form, onChange }: { form: Form; onChange: (patch: Partial
) => void }) { +function directiveText(t: ReturnType, d: RouteDirective | undefined, field: "label" | "detail"): string { + return d ? t(`deploy.routeOptions.options.${d.key}.${field}`, { defaultValue: d[field] }) : ""; +} + +function runtimeText(t: ReturnType, d: RouteDirective): string { + return t("deploy.routeOptions.current", { value: runtimeValue(d.value), source: t(d.set ? "deploy.routeOptions.configured" : "deploy.routeOptions.inherited") }); +} + +export function RouteExtras({ form, onChange, options, showRuntime = false }: { form: Form; onChange: (patch: Partial) => void; options?: RouteOptions | null; showRuntime?: boolean }) { const t = useT(); const f = (k: string, v?: Record) => t(`deploy.fields.${k}`, v); const [custom, setCustom] = useState(""); - const taken = form.nginxExtras.map((r) => r.key); + const controlsId = useId(); + const catalog = options === null ? FALLBACK_ROUTE_OPTIONS : options ?? { route: "", directives: [], advanced: [] }; + const byKey = new Map(routeDirectives(catalog).map((d) => [d.key, d])); + const taken = [...NAMED_VALUES, ...form.nginxExtras.map((r) => r.key)]; const used = NAMED_VALUES.size + form.nginxExtras.length; const full = used >= MAX_VALUES; const customError = custom.trim() ? rejectKey(custom, taken) : ""; const add = (key: string) => { - onChange({ nginxExtras: [...form.nginxExtras, { key, value: initialValue(key) }] }); + onChange({ nginxExtras: [...form.nginxExtras, { key, value: "" }] }); setCustom(""); }; - const set = (i: number, value: string) => onChange({ nginxExtras: form.nginxExtras.map((r, j) => (j === i ? { ...r, value } : r)) }); - + const set = (key: string, value: string) => { + const exists = form.nginxExtras.some((r) => r.key === key); + if (!exists && full) return; + onChange({ nginxExtras: exists ? form.nginxExtras.map((r) => r.key === key ? { ...r, value } : r) : [...form.nginxExtras, { key, value }] }); + }; + const remove = (key: string) => onChange({ nginxExtras: form.nginxExtras.filter((r) => r.key !== key) }); + const row = (key: string, d?: RouteDirective) => { + const value = form.nginxExtras.find((r) => r.key === key)?.value ?? ""; + const idle = !!d?.needsAdaptive && !form.adaptiveCc; + const label = directiveText(t, d, "label") || key; + const fraction = key.endsWith("_frac") && fracOutOfRange(value); + return ( +
+
+
{d && {directiveText(t, d, "detail")}}
+ {idle &&
{f("idle")}
} + {showRuntime && d?.set !== undefined &&
{runtimeText(t, d)}
} +
+
+ {d?.kind === "boolean" ? ( + set(key, v === "__inherit" ? "" : v)} disabled={full && !taken.includes(key)} + options={[{ value: "__inherit", label: d.fallback ? f("without", { fallback: d.fallback }) : t("deploy.routeOptions.inherit") }, { value: "true", label: t("deploy.routeOptions.on") }, { value: "false", label: t("deploy.routeOptions.off") }]} /> + ) : ( + set(key, e.target.value)} inputMode={d?.kind === "number" ? "decimal" : undefined} placeholder={d?.fallback || undefined} aria-label={label} aria-invalid={!!fraction} disabled={full && !taken.includes(key)} /> + )} + {fraction &&

{f("fracRange")}

} +
+ +
+ ); + }; + const primary = catalog.directives.filter((d) => !NAMED_VALUES.has(d.key)); + const primaryKeys = new Set(primary.map((d) => d.key)); return ( - - {form.nginxExtras.map((row, i) => { - const d = DIRECTIVES_BY_KEY.get(row.key); - const idle = !!d?.needsAdaptive && !form.adaptiveCc; - return ( -
-
-
{d?.label ?? row.key}
-
{idle ? f("idle") : f("without", { fallback: d?.fallback ?? "-" })}
-
- {d?.kind === "boolean" ? ( - set(i, v ? "true" : "false")} /> - ) : ( - set(i, e.target.value)} inputMode={d?.kind === "number" ? "decimal" : undefined} placeholder={d?.fallback} aria-label={d?.label ?? row.key} /> - )} - -
- ); - })} -
- - }> - {f("extrasAdd")} - - - {directiveGroups().map((group) => { - const rows = DIRECTIVES.filter((d) => d.group === group && !taken.includes(d.key)); - if (!rows.length) return null; - return ( - - {group} - {rows.map((d) => ( - add(d.key)} className="flex-col items-start gap-0.5"> - - {d.label} {d.key} - - {f("without", { fallback: d.fallback })} - - ))} - - - ); - })} - - - setCustom(e.target.value)} placeholder={f("extrasCustom")} aria-label={f("extrasCustom")} className="h-8 w-48" disabled={full} /> - - - {full ? f("extrasFull", { max: MAX_VALUES }) : f("extrasUsed", { used, max: MAX_VALUES })} - +
+ {[...new Set(primary.map((d) => d.group))].map((group) => ( +
+
{t(`deploy.routeOptions.groups.${group}`, { defaultValue: group })}
+ {primary.filter((d) => d.group === group).map((d) => row(d.key, d))}
- {customError &&

{customError}

} - + ))} + {showRuntime &&
+ {catalog.directives.filter((d) => NAMED_VALUES.has(d.key)).map((d) =>
{directiveText(t, d, "label")}: {runtimeText(t, d)}
)} +
} + {form.nginxExtras.filter((r) => !primaryKeys.has(r.key)).map((r) => row(r.key, byKey.get(r.key)))} +
+ + }> {t("deploy.routeOptions.advanced")} + + {[...new Set(catalog.advanced.map((d) => d.group))].map((group) => { + const rows = catalog.advanced.filter((d) => d.group === group && !taken.includes(d.key)); + if (!rows.length) return null; + return + {t(`deploy.routeOptions.groups.${group}`, { defaultValue: group })} + {rows.map((d) => add(d.key)} className="flex-col items-start gap-0.5"> + {directiveText(t, d, "label")} {d.key} + {d.fallback && {f("without", { fallback: d.fallback })}} + )} + + ; + })} + + + setCustom(e.target.value)} placeholder={f("extrasCustom")} aria-label={f("extrasCustom")} className="h-8 w-48" disabled={full} /> + + {full ? f("extrasFull", { max: MAX_VALUES }) : f("extrasUsed", { used, max: MAX_VALUES })} +
+ {customError &&

{customError}

} +
); } diff --git a/web/src/modules/inferences/components/DeploySheet.tsx b/web/src/modules/inferences/components/DeploySheet.tsx index bf248d2..4d592e5 100644 --- a/web/src/modules/inferences/components/DeploySheet.tsx +++ b/web/src/modules/inferences/components/DeploySheet.tsx @@ -37,6 +37,7 @@ import { useT } from "@/modules/inferences/i18n"; import { detailPath } from "@/modules/inferences/lib"; import { SECTIONS, chartChoice, defaultServiceId, gpuOptions, pipelineState, serviceIdError, whatMoves, type Section } from "@/modules/inferences/deploy-lib"; import { DeployForm } from "@/modules/inferences/components/DeployForm"; +import { readRouteOptions } from "@/modules/inferences/api"; export type DeployTarget = | { kind: "deploy"; model: string; catalog: string; version?: string; variant?: string } @@ -105,6 +106,14 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( const initial = useMemo(() => (upgrade ? (current.data ? formFromPlan(current.data) : EMPTY) : { ...EMPTY, serviceId: defaultServiceId(target.model) }), [upgrade, current.data, target]); const [form, setForm] = useState(initial); + const catalogRoute = upgrade && current.data ? initial.route || initial.serviceId || target.release : undefined; + const routeOptions = useQuery({ + queryKey: ["route-options", catalogRoute ?? ""], + queryFn: ({ signal }) => readRouteOptions(catalogRoute, signal), + enabled: !upgrade || !!current.data, + staleTime: 30_000, + retry: false, + }); useEffect(() => setForm(initial), [initial]); const dirty = useDirty({ form, version, variant, moveTo, chartVersion }, { form: initial, version: upgrade ? "" : (target.version ?? ""), variant: upgrade ? "" : (target.variant ?? ""), moveTo: "", chartVersion: "" }); @@ -270,6 +279,9 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( {t("deploy.review.readOnly")} )} + {routeOptions.error && ( + {t("deploy.routeOptions.error", { error: routeOptions.error.message })} + )} {(error || compose.error || apply.error) && ( @@ -293,6 +305,8 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( gpuOptions={gpuOptions(v?.requires.gpuProduct, nodes.data?.nodes.map((n) => n.GPUProduct))} expanded={expanded} onExpand={(s, o) => setExpanded((e) => ({ ...e, [s]: o }))} + routeOptions={routeOptions.isSuccess ? routeOptions.data : undefined} + catalogRoute={catalogRoute} idPrefix={ID_PREFIX} />
diff --git a/web/src/modules/inferences/locales/en-US.json b/web/src/modules/inferences/locales/en-US.json index 7deafc4..0bc65d6 100644 --- a/web/src/modules/inferences/locales/en-US.json +++ b/web/src/modules/inferences/locales/en-US.json @@ -330,6 +330,72 @@ "routing": "Routing", "advanced": "Advanced" }, + "routeOptions": { + "advanced": "Advanced route options", + "inherit": "Use engine default", + "on": "On", + "off": "Off", + "reset": "Reset {label}", + "current": "Currently: {value} · {source}", + "configured": "route config", + "inherited": "engine default or derived", + "error": "Could not read route options: {error}", + "groups": { + "Adaptive concurrency": "Adaptive concurrency", + "Thresholds": "Thresholds", + "Capacity & failover": "Capacity & failover", + "Capacity": "Capacity & failover", + "Routing": "Routing", + "Observability": "Observability", + "Health checks": "Health checks", + "TTFT measurement": "TTFT measurement", + "TPS measurement": "TPS measurement", + "Rejection probes": "Rejection probes", + "Other": "Other" + }, + "options": { + "adaptive_cc": { "label": "Adaptive concurrency" }, + "adaptive_cc_min": { "label": "Adaptive floor" }, + "adaptive_cc_min_frac": { "label": "Floor fraction" }, + "adaptive_cc_init": { "label": "Start point" }, + "adaptive_cc_init_frac": { "label": "Start fraction" }, + "adaptive_cc_hold_when_idle": { "label": "Hold when idle" }, + "adaptive_cc_use_ttft": { "label": "Judge on TTFT too" }, + "tps_limit_tps": { "label": "Decode rate threshold" }, + "ttft_limit_ms": { "label": "TTFT threshold" }, + "default_max": { "label": "Default peer maximum" }, + "max_more_tries": { "label": "Retry budget" }, + "cross_tier_fallback": { "label": "Cross-tier fallback" }, + "health_probe_path": { "label": "Probe path" }, + "session_affinity_enabled": { "label": "Session affinity" }, + "expose_routed_peer": { "label": "Expose routed peer" }, + "bodylog_default_pct": { "label": "Body log sample percent" }, + "adaptive_cc_inc": { "label": "Climb multiplier" }, + "adaptive_cc_dec": { "label": "Shrink multiplier" }, + "adaptive_cc_abs": { "label": "Absolute headroom" }, + "adaptive_cc_pressure_frac": { "label": "Pressure fraction" }, + "adaptive_cc_slack_frac": { "label": "Slack fraction", "detail": "Slack band relative to the start point; inactive while hold-when-idle is enabled." }, + "adaptive_cc_ttl": { "label": "Start-point TTL", "detail": "Lifetime of the idle start-point state; inactive while hold-when-idle is enabled." }, + "adaptive_cc_interval": { "label": "Step interval" }, + "rt_limit_factor": { "label": "Runtime limit factor", "detail": "Scales the pool's runtime concurrency limit, not each peer's maximum." }, + "health_check_interval": { "label": "Probe interval" }, + "health_ban_ttl": { "label": "Ban TTL" }, + "cluster_avg_interval": { "label": "Cluster average interval" }, + "ttft_ewma_alpha": { "label": "TTFT EWMA alpha" }, + "ttft_window": { "label": "TTFT window" }, + "ttft_ttl": { "label": "TTFT TTL" }, + "tps_ewma_alpha": { "label": "TPS EWMA alpha" }, + "tps_window": { "label": "TPS window" }, + "tps_ttl": { "label": "TPS TTL" }, + "tps_min_tokens": { "label": "Minimum tokens" }, + "tps_min_decode_s": { "label": "Minimum decode seconds" }, + "ttft_probe_window": { "label": "TTFT probe window" }, + "ttft_probe_per_window": { "label": "TTFT probes per window", "detail": "Probe requests allowed through while TTFT limiting is active, per probe window." }, + "tps_probe_window": { "label": "TPS probe window" }, + "tps_probe_per_window": { "label": "TPS probes per window", "detail": "Probe requests allowed through while TPS limiting is active, per probe window." }, + "reject_rules_status": { "label": "Rejection status" } + } + }, "fields": { "model": "Model", "catalog": "Catalog", @@ -397,7 +463,7 @@ "adaptiveCc": "Adaptive concurrency", "adaptiveCcHint": "AIMD on the effective TPS signal; on by default", "adaptiveCcMinFrac": "Adaptive floor fraction", - "adaptiveCcMinFracHint": "Of the static peer max, 0 < value ≤ 1; empty leaves openresty's 0.4. Unused when the absolute floor is set", + "adaptiveCcMinFracHint": "Of the static maximum, 0 < value ≤ 1; empty uses the engine default. Unused when the absolute floor is set", "fracRange": "Must be over 0 and at most 1, or openresty ignores it", "monitor": "Monitor", "monitorHint": "Writes the backends into monitor's config", diff --git a/web/src/modules/inferences/locales/zh-CN.json b/web/src/modules/inferences/locales/zh-CN.json index 4739ccd..cab3e6e 100644 --- a/web/src/modules/inferences/locales/zh-CN.json +++ b/web/src/modules/inferences/locales/zh-CN.json @@ -319,6 +319,72 @@ "routing": "路由", "advanced": "高级" }, + "routeOptions": { + "advanced": "高级路由参数", + "inherit": "使用引擎默认值", + "on": "开启", + "off": "关闭", + "reset": "重置{label}", + "current": "当前：{value} · {source}", + "configured": "路由配置", + "inherited": "引擎默认值或派生值", + "error": "无法读取路由参数：{error}", + "groups": { + "Adaptive concurrency": "自适应并发", + "Thresholds": "阈值", + "Capacity & failover": "容量与故障切换", + "Capacity": "容量与故障切换", + "Routing": "路由", + "Observability": "可观测性", + "Health checks": "健康检查", + "TTFT measurement": "TTFT 测量", + "TPS measurement": "TPS 测量", + "Rejection probes": "限流探测", + "Other": "其他" + }, + "options": { + "adaptive_cc": { "label": "自适应并发" }, + "adaptive_cc_min": { "label": "自适应下限" }, + "adaptive_cc_min_frac": { "label": "下限比例" }, + "adaptive_cc_init": { "label": "起始并发" }, + "adaptive_cc_init_frac": { "label": "起始比例" }, + "adaptive_cc_hold_when_idle": { "label": "空闲时保持并发" }, + "adaptive_cc_use_ttft": { "label": "同时依据 TTFT 调整" }, + "tps_limit_tps": { "label": "解码速率阈值" }, + "ttft_limit_ms": { "label": "TTFT 阈值" }, + "default_max": { "label": "默认节点并发上限" }, + "max_more_tries": { "label": "重试次数" }, + "cross_tier_fallback": { "label": "跨层故障切换" }, + "health_probe_path": { "label": "探测路径" }, + "session_affinity_enabled": { "label": "会话亲和性" }, + "expose_routed_peer": { "label": "暴露路由后端" }, + "bodylog_default_pct": { "label": "请求体日志采样比例" }, + "adaptive_cc_inc": { "label": "增大倍率" }, + "adaptive_cc_dec": { "label": "缩小倍率" }, + "adaptive_cc_abs": { "label": "绝对余量" }, + "adaptive_cc_pressure_frac": { "label": "压力比例" }, + "adaptive_cc_slack_frac": { "label": "松弛比例", "detail": "相对起始并发的松弛区间；开启空闲保持时不生效。" }, + "adaptive_cc_ttl": { "label": "起始状态 TTL", "detail": "空闲起始状态的保留时间；开启空闲保持时不生效。" }, + "adaptive_cc_interval": { "label": "调整间隔" }, + "rt_limit_factor": { "label": "运行时并发倍率", "detail": "缩放整个池的运行时并发上限，不是每个节点的上限。" }, + "health_check_interval": { "label": "探测间隔" }, + "health_ban_ttl": { "label": "封禁 TTL" }, + "cluster_avg_interval": { "label": "集群均值计算间隔" }, + "ttft_ewma_alpha": { "label": "TTFT 平滑系数" }, + "ttft_window": { "label": "TTFT 时间窗口" }, + "ttft_ttl": { "label": "TTFT TTL" }, + "tps_ewma_alpha": { "label": "TPS 平滑系数" }, + "tps_window": { "label": "TPS 时间窗口" }, + "tps_ttl": { "label": "TPS TTL" }, + "tps_min_tokens": { "label": "最少 token 数" }, + "tps_min_decode_s": { "label": "最短解码时间" }, + "ttft_probe_window": { "label": "TTFT 探测窗口" }, + "ttft_probe_per_window": { "label": "每窗口 TTFT 探测请求数", "detail": "TTFT 限流期间每个探测窗口放行的请求数，不是采样率。" }, + "tps_probe_window": { "label": "TPS 探测窗口" }, + "tps_probe_per_window": { "label": "每窗口 TPS 探测请求数", "detail": "TPS 限流期间每个探测窗口放行的请求数，不是采样率。" }, + "reject_rules_status": { "label": "拒绝状态码" } + } + }, "fields": { "model": "模型", "catalog": "目录", @@ -386,7 +452,7 @@ "adaptiveCc": "自适应并发", "adaptiveCcHint": "按有效 TPS 做 AIMD 调整，默认开启", "adaptiveCcMinFrac": "自适应下限比例", - "adaptiveCcMinFracHint": "占静态并发上限的比例，0 < 值 ≤ 1，留空为 openresty 默认 0.4；设置了绝对下限时不使用", + "adaptiveCcMinFracHint": "占静态并发上限的比例，0 < 值 ≤ 1，留空使用引擎默认值；设置了绝对下限时不使用", "fracRange": "必须大于 0 且不超过 1，否则 openresty 会忽略", "monitor": "监控", "monitorHint": "把后端写入 monitor 的配置", diff --git a/web/src/modules/inferences/routeOptions.test.ts b/web/src/modules/inferences/routeOptions.test.ts new file mode 100644 index 0000000..3adcc98 --- /dev/null +++ b/web/src/modules/inferences/routeOptions.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { FALLBACK_ROUTE_OPTIONS, routeDirectives, runtimeValue } from "./routeOptions"; + +describe("legacy catalog", () => { + it("does not infer legacy support from pending or failed query data", () => { + expect(routeDirectives()).toEqual([]); + expect(routeDirectives(undefined)).toEqual([]); + expect(routeDirectives(null)).toEqual([...FALLBACK_ROUTE_OPTIONS.directives, ...FALLBACK_ROUTE_OPTIONS.advanced]); + }); + + it("keeps a successful empty catalog distinct from legacy fallback", () => { + expect(routeDirectives({ route: "model", directives: [], advanced: [] })).toEqual([]); + }); + + it("contains all 16 common and 24 advanced options without asserting engine defaults or route values", () => { + expect(FALLBACK_ROUTE_OPTIONS.directives).toHaveLength(16); + expect(FALLBACK_ROUTE_OPTIONS.advanced).toHaveLength(24); + for (const d of routeDirectives(null)) { + expect(d.fallback).toBe(""); + expect(d.value).toBeUndefined(); + expect(d.set).toBeUndefined(); + } + }); + it("keeps null, false and zero distinct", () => { + expect([null, false, true, 0, "the floor"].map(runtimeValue)).toEqual(["-", "off", "on", "0", "the floor"]); + }); +}); diff --git a/web/src/modules/inferences/routeOptions.ts b/web/src/modules/inferences/routeOptions.ts new file mode 100644 index 0000000..20186e1 --- /dev/null +++ b/web/src/modules/inferences/routeOptions.ts @@ -0,0 +1,55 @@ +import { DIRECTIVES, type Directive } from "@swiss/lib/routeDirectives"; + +export type RouteValue = string | number | boolean | null; +export interface RouteDirective extends Directive { + value?: RouteValue; + set?: boolean; +} +export interface RouteOptions { + route: string; + directives: RouteDirective[]; + advanced: RouteDirective[]; + peers?: { count: number }; + adaptive_cc_derived?: Record; +} + +const common = new Set([ + "adaptive_cc", "adaptive_cc_min", "adaptive_cc_min_frac", "adaptive_cc_init", "adaptive_cc_init_frac", "adaptive_cc_hold_when_idle", "adaptive_cc_use_ttft", + "tps_limit_tps", "ttft_limit_ms", "default_max", "max_more_tries", "cross_tier_fallback", "health_probe_path", "session_affinity_enabled", "expose_routed_peer", "bodylog_default_pct", +]); +const additions: Directive[] = [ + { key: "adaptive_cc", label: "Adaptive concurrency", group: "Adaptive concurrency", kind: "boolean", fallback: "", detail: "Adjust the pool limit using measured performance." }, + { key: "adaptive_cc_min_frac", label: "Floor fraction", group: "Adaptive concurrency", kind: "number", fallback: "", detail: "Fraction of the static maximum used as the floor; greater than zero and at most one.", needsAdaptive: true }, + { key: "adaptive_cc_init", label: "Start point", group: "Adaptive concurrency", kind: "number", fallback: "", detail: "Absolute initial pool limit.", needsAdaptive: true }, + { key: "adaptive_cc_init_frac", label: "Start fraction", group: "Adaptive concurrency", kind: "number", fallback: "", detail: "Fraction of the static maximum used as the initial limit.", needsAdaptive: true }, + { key: "adaptive_cc_hold_when_idle", label: "Hold when idle", group: "Adaptive concurrency", kind: "boolean", fallback: "", detail: "Keep the learned limit when idle instead of returning to the start point.", needsAdaptive: true }, + { key: "tps_limit_tps", label: "Decode rate threshold", group: "Thresholds", kind: "number", fallback: "", detail: "Static decode-rate threshold, used when no TPS metrics are declared." }, + { key: "ttft_limit_ms", label: "TTFT threshold", group: "Thresholds", kind: "number", fallback: "", detail: "Static time-to-first-token threshold in milliseconds, used when no TTFT metrics are declared." }, + { key: "session_affinity_enabled", label: "Session affinity", group: "Routing", kind: "boolean", fallback: "", detail: "Keep requests in a session on the same peer." }, + { key: "expose_routed_peer", label: "Expose routed peer", group: "Observability", kind: "boolean", fallback: "", detail: "Return the selected peer in the X-Routed-Peer response header." }, +]; +const corrections: Record = { + adaptive_cc_ttl: "Lifetime of the idle start-point state; has no effect while hold-when-idle is enabled.", + adaptive_cc_slack_frac: "Slack band relative to the start point; has no effect while hold-when-idle is enabled.", + rt_limit_factor: "Scales the pool's runtime concurrency limit, not each peer's maximum.", + max_more_tries: "Upper bound on balancer retries for one request.", + ttft_probe_per_window: "Probe requests allowed through while TTFT limiting is active, per probe window.", + tps_probe_per_window: "Probe requests allowed through while TPS limiting is active, per probe window.", +}; +const bundled = [...DIRECTIVES.map((d) => ({ ...d, fallback: "", detail: corrections[d.key] ?? d.detail })), ...additions]; +export const FALLBACK_ROUTE_OPTIONS: RouteOptions = { + route: "", + directives: bundled.filter((d) => common.has(d.key)), + advanced: bundled.filter((d) => !common.has(d.key)), +}; + +export function routeDirectives(options?: RouteOptions | null): RouteDirective[] { + const catalog = options === null ? FALLBACK_ROUTE_OPTIONS : options; + return catalog ? [...catalog.directives, ...catalog.advanced] : []; +} + +export function runtimeValue(value: RouteValue | undefined): string { + if (value === null || value === undefined) return "-"; + if (typeof value === "boolean") return value ? "on" : "off"; + return String(value); +}