From 2afb60a9d6108d6db312cf2c951c105f50929512 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 29 Sep 2026 09:34:35 +0000 Subject: [PATCH] Restore model probe for service routers Since #4270, a router replica of a service with `model` and no `probes` was probed with `GET /health` instead of the chat completions request. The latter deadlocked: the router only answers it once workers are registered, and the worker sync skipped routers that were not ready. #4313 dropped the readiness check from the sync, so this deadlock is gone. `/health` passes before the router has any workers. In a rolling deployment, the replacement router was considered ready, and the old one was scaled down, before the replacement could serve requests. Now that #4320 syncs workers with every running router, the replacement passes the chat completions probe once it has workers. Workers behind a router still get no default probe, as they may not serve HTTP at all (gRPC workers). With a Dynamo router, a router rolling deployment now waits for the replacement indefinitely instead of scaling down the old router, as workers stay attached to the old router's etcd/NATS. Such deployments were broken before as well and are to be fixed separately. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../_internal/core/models/configurations.py | 1 - .../services/jobs/configurators/base.py | 36 ++++++------------- .../jobs/configurators/test_service.py | 13 ++++--- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 13150805f..81d5d6226 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -72,7 +72,6 @@ DEFAULT_PROBE_READY_AFTER = 1 DEFAULT_PROBE_METHOD = "get" DEFAULT_PROBE_UNTIL_READY = False -ROUTER_HEALTH_PROBE_URL = "/health" MAX_PROBE_URL_LEN = 2048 DEFAULT_REPLICA_GROUP_NAME = "0" OPENAI_MODEL_PROBE_TIMEOUT = 30 diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index c4c5bfd60..1a1fadf37 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -27,7 +27,6 @@ DEFAULT_REPLICA_GROUP_NAME, LEGACY_REPO_DIR, OPENAI_MODEL_PROBE_TIMEOUT, - ROUTER_HEALTH_PROBE_URL, HTTPHeaderSpec, NodeGroup, PortMapping, @@ -513,17 +512,17 @@ def _probes(self) -> list[ProbeSpec]: model = conf.model if not isinstance(model, OpenAIChatModel): return [] - if all(group.router is None for group in conf.replica_groups): - # No router: every replica serves the model itself, so a chat completions - # request is a genuine end-to-end readiness check. - return [_openai_model_probe_spec(model.name, model.prefix)] - group = self._replica_group() - if group is not None and group.router is not None: - # Probe the router's own liveness endpoint, which does not depend on any - # worker. Workers get no default probe: they may not serve HTTP at all - # (gRPC workers), and they don't receive traffic directly. - return [_router_health_probe_spec()] - return [] + if any(group.router is not None for group in conf.replica_groups): + group = self._replica_group() + if group is None or group.router is None: + # Workers get no default probe: they may not serve HTTP at all (gRPC + # workers), and they don't receive traffic directly. + return [] + # The router answers chat completions only once it has workers, which it gets + # regardless of its own readiness. For SGLang routers, dstack registers workers + # via `ServiceRouterWorkerSyncWorker`, which doesn't check router readiness. + # Dynamo workers register themselves with the router via etcd/NATS. + return [_openai_model_probe_spec(model.name, model.prefix)] def interpolate_job_volumes( @@ -598,19 +597,6 @@ def _openai_model_probe_spec(model_name: str, prefix: str) -> ProbeSpec: ) -def _router_health_probe_spec() -> ProbeSpec: - # Both supported routers (SGLang/SMG and Dynamo) serve `/health` independently of - # whether any worker is registered. - return ProbeSpec( - type="http", - method=DEFAULT_PROBE_METHOD, - url=ROUTER_HEALTH_PROBE_URL, - timeout=DEFAULT_PROBE_TIMEOUT, - interval=DEFAULT_PROBE_INTERVAL, - ready_after=DEFAULT_PROBE_READY_AFTER, - ) - - def _join_shell_commands(commands: List[str]) -> str: for i, cmd in enumerate(commands): cmd = cmd.strip() diff --git a/src/tests/_internal/server/services/jobs/configurators/test_service.py b/src/tests/_internal/server/services/jobs/configurators/test_service.py index 8a962542a..3f0de1289 100644 --- a/src/tests/_internal/server/services/jobs/configurators/test_service.py +++ b/src/tests/_internal/server/services/jobs/configurators/test_service.py @@ -6,7 +6,6 @@ from dstack._internal import settings from dstack._internal.core.models.configurations import ( OPENAI_MODEL_PROBE_TIMEOUT, - ROUTER_HEALTH_PROBE_URL, ProbeConfig, PythonVersion, ReplicaGroup, @@ -116,9 +115,9 @@ def _router_worker_configuration() -> ServiceConfiguration: ], ) - async def test_router_group_gets_health_probe(self): - """The router must not be probed with chat completions: it only answers those once - dstack has registered workers, and registration requires the router to be ready.""" + async def test_router_group_gets_model_probe(self): + """The router is probed with chat completions like a replica serving the model + itself: worker registration doesn't wait for the router to be ready.""" run_spec = get_run_spec( run_name="run", repo_id="id", configuration=self._router_worker_configuration() ) @@ -128,9 +127,9 @@ async def test_router_group_gets_health_probe(self): probes = job_specs[0].probes assert len(probes) == 1 - assert probes[0].url == ROUTER_HEALTH_PROBE_URL - assert probes[0].method == "get" - assert probes[0].body is None + assert probes[0].method == "post" + assert probes[0].url == "/v1/chat/completions" + assert "meta-llama/Meta-Llama-3.1-8B-Instruct" in (probes[0].body or "") async def test_worker_group_gets_no_derived_probe(self): """Workers behind a router may speak gRPC, so no probe can be derived from `model`."""