diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cba1cbaa3..19e0a56c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,27 @@ jobs: mise run helm -- dependencies mise exec -- uv run --frozen --project . --no-sync pytest -q -m docker tools/ci/tests/test_collector_*privacy.py + bundle-resolution: + name: CI / Bundle resolution + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 45 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + persist-credentials: false + - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + version: 2026.5.5 + install_args: --jobs=1 + cache: false + - run: mise run sync + - name: Resolve release bundle constraints + env: + SIE_RUN_IMAGE_RESOLUTION: "1" + run: >- + mise exec -- uv run --frozen --project . --no-sync pytest -q + packages/sie_server/tests/core/test_bundle_image_resolution.py + typescript: name: CI / TypeScript runs-on: blacksmith-4vcpu-ubuntu-2404 @@ -292,7 +313,7 @@ jobs: required: name: CI / Required if: ${{ always() }} - needs: [policy, bootstrap, python, typescript, rust, contracts, helm, live-sdk, cpu-stack, python-distributions, npm-distributions] + needs: [policy, bootstrap, python, bundle-resolution, typescript, rust, contracts, helm, live-sdk, cpu-stack, python-distributions, npm-distributions] runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 5 steps: diff --git a/packages/sie_server/Dockerfile.cpu b/packages/sie_server/Dockerfile.cpu index b60b5fd10..94e6a2be5 100644 --- a/packages/sie_server/Dockerfile.cpu +++ b/packages/sie_server/Dockerfile.cpu @@ -129,13 +129,20 @@ FROM base AS builder ARG BUNDLE +# Ranged bundle deps are constrained to uv.lock. Exact pins and URL/VCS specs stay overrides. +COPY uv.lock /tmp/uv.lock + RUN .venv/bin/python -m sie_server.cli resolve-deps \ --bundle "${BUNDLE}" \ --models-dir models \ --cpu \ + --lock /tmp/uv.lock \ + --constraints-output /tmp/bundle-constraints.txt \ > /tmp/bundle-requirements.txt \ && echo "Bundle ${BUNDLE} requirements:" \ - && cat /tmp/bundle-requirements.txt + && cat /tmp/bundle-requirements.txt \ + && echo "Bundle ${BUNDLE} lock constraints:" \ + && cat /tmp/bundle-constraints.txt # Bundle-specific deps land in a separate site-packages tree so the shared # venv layer stays byte-identical across bundles of the same platform. @@ -149,6 +156,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ --extra-index-url https://pypi.org/simple \ --target=/app/bundle-libs \ --no-compile \ + -c /tmp/bundle-constraints.txt \ -r /tmp/bundle-requirements.txt; \ fi @@ -163,7 +171,9 @@ RUN set -eux; \ rm -rf torch/test torch/utils/tensorboard torch/utils/bottleneck 2>/dev/null || true; \ find torch -type f -name '*.a' -delete 2>/dev/null || true; \ fi; \ - find /app/bundle-libs -exec touch -h -d @0 {} + 2>/dev/null || true + find /app/bundle-libs -exec touch -h -d @0 {} + 2>/dev/null || true; \ + /app/.venv/bin/pip freeze --path /app/bundle-libs > /app/bundle-libs-freeze.txt; \ + touch -d @0 /app/bundle-libs-freeze.txt # ============================================================================= # Runtime image @@ -220,6 +230,7 @@ COPY --link --from=base --chown=1000:1000 /app/models /app/models COPY --link --from=base --chown=1000:1000 /app/bundles /app/bundles # Bundle-specific extras — last layer so shared layers above stay cached. COPY --link --from=builder --chown=1000:1000 /app/bundle-libs /app/bundle-libs +COPY --link --from=builder --chown=1000:1000 /app/bundle-libs-freeze.txt /app/bundle-libs-freeze.txt RUN mkdir -p /app/evals /app/.cache/huggingface /app/.cache/sie && chown -R sie:sie /app/evals /app/.cache \ && chmod 700 /app/.cache/sie diff --git a/packages/sie_server/Dockerfile.cuda12 b/packages/sie_server/Dockerfile.cuda12 index 1f58c81df..5393c3a28 100644 --- a/packages/sie_server/Dockerfile.cuda12 +++ b/packages/sie_server/Dockerfile.cuda12 @@ -148,12 +148,19 @@ FROM base AS builder ARG BUNDLE +# Ranged bundle deps are constrained to uv.lock. Exact pins and URL/VCS specs stay overrides. +COPY uv.lock /tmp/uv.lock + RUN .venv/bin/python -m sie_server.cli resolve-deps \ --bundle "${BUNDLE}" \ --models-dir models \ + --lock /tmp/uv.lock \ + --constraints-output /tmp/bundle-constraints.txt \ > /tmp/bundle-requirements.txt \ && echo "Bundle ${BUNDLE} requirements:" \ - && cat /tmp/bundle-requirements.txt + && cat /tmp/bundle-requirements.txt \ + && echo "Bundle ${BUNDLE} lock constraints:" \ + && cat /tmp/bundle-constraints.txt # Bundle-specific deps go into a separate site-packages tree (``bundle-libs``, # prepended to sys.path via a .pth) so the shared venv layer stays @@ -177,11 +184,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \ --target=/app/bundle-libs \ --no-compile \ --extra-index-url https://docs.sglang.ai/whl/cu129/ \ + -c /tmp/bundle-constraints.txt \ -r /tmp/bundle-requirements.txt ;; \ *) \ .venv/bin/pip install \ --target=/app/bundle-libs \ --no-compile \ + -c /tmp/bundle-constraints.txt \ -r /tmp/bundle-requirements.txt ;; \ esac; \ fi @@ -201,7 +210,9 @@ RUN set -eux; \ find nvidia -type f -name '*.a' -delete 2>/dev/null || true; \ fi; \ # Normalize mtimes so rebuilds of the same bundle produce identical layer bytes. - find /app/bundle-libs -exec touch -h -d @0 {} + 2>/dev/null || true + find /app/bundle-libs -exec touch -h -d @0 {} + 2>/dev/null || true; \ + /app/.venv/bin/pip freeze --path /app/bundle-libs > /app/bundle-libs-freeze.txt; \ + touch -d @0 /app/bundle-libs-freeze.txt # ============================================================================= # Runtime image @@ -307,6 +318,7 @@ COPY --link --from=base --chown=1000:1000 /app/bundles /app/bundles # Bundle-specific extras — last layer so shared layers above stay cached. # A `.pth` file in the shared venv puts /app/bundle-libs at sys.path[0] at startup. COPY --link --from=builder --chown=1000:1000 /app/bundle-libs /app/bundle-libs +COPY --link --from=builder --chown=1000:1000 /app/bundle-libs-freeze.txt /app/bundle-libs-freeze.txt RUN mkdir -p /app/evals /app/.cache/huggingface /app/.cache/sie-kernels /app/.cache/sie \ && chown -R sie:sie /app/evals /app/.cache \ diff --git a/packages/sie_server/Dockerfile.cuda13 b/packages/sie_server/Dockerfile.cuda13 index f885dfa88..9f1b7fb15 100644 --- a/packages/sie_server/Dockerfile.cuda13 +++ b/packages/sie_server/Dockerfile.cuda13 @@ -148,12 +148,19 @@ FROM base AS builder ARG BUNDLE +# Ranged bundle deps are constrained to uv.lock. Exact pins and URL/VCS specs stay overrides. +COPY uv.lock /tmp/uv.lock + RUN .venv/bin/python -m sie_server.cli resolve-deps \ --bundle "${BUNDLE}" \ --models-dir models \ + --lock /tmp/uv.lock \ + --constraints-output /tmp/bundle-constraints.txt \ > /tmp/bundle-requirements.txt \ && echo "Bundle ${BUNDLE} requirements:" \ - && cat /tmp/bundle-requirements.txt + && cat /tmp/bundle-requirements.txt \ + && echo "Bundle ${BUNDLE} lock constraints:" \ + && cat /tmp/bundle-constraints.txt # cu130 bundle deps go into a separate site-packages tree (``bundle-libs``, # prepended to sys.path via the .pth above). CUDA 13 engines need torch 2.11 @@ -191,6 +198,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ --prerelease=allow \ --extra-index-url https://download.pytorch.org/whl/cu130 \ --extra-index-url https://docs.sglang.ai/whl/cu130/ \ + --constraint /tmp/bundle-constraints.txt \ -r /tmp/bundle-requirements.txt ; \ fi @@ -206,7 +214,9 @@ RUN set -eux; \ find torch -type f -name '*.a' -delete 2>/dev/null || true; \ find nvidia -type f -name '*.a' -delete 2>/dev/null || true; \ fi; \ - find /app/bundle-libs -exec touch -h -d @0 {} + 2>/dev/null || true + find /app/bundle-libs -exec touch -h -d @0 {} + 2>/dev/null || true; \ + /app/.venv/bin/pip freeze --path /app/bundle-libs > /app/bundle-libs-freeze.txt; \ + touch -d @0 /app/bundle-libs-freeze.txt # ============================================================================= # Runtime image (CUDA 13 devel — nvcc for flashinfer/tvm_ffi first-decode JIT) @@ -272,6 +282,7 @@ COPY --link --from=base --chown=1000:1000 /app/models /app/models COPY --link --from=base --chown=1000:1000 /app/bundles /app/bundles # Bundle-specific extras — last layer so shared layers above stay cached. COPY --link --from=builder --chown=1000:1000 /app/bundle-libs /app/bundle-libs +COPY --link --from=builder --chown=1000:1000 /app/bundle-libs-freeze.txt /app/bundle-libs-freeze.txt RUN mkdir -p /app/evals /app/.cache/huggingface /app/.cache/sie-kernels /app/.cache/sie \ && chown -R sie:sie /app/evals /app/.cache \ diff --git a/packages/sie_server/src/sie_server/bundle_requirements.py b/packages/sie_server/src/sie_server/bundle_requirements.py index 4f329db9c..097aa7930 100644 --- a/packages/sie_server/src/sie_server/bundle_requirements.py +++ b/packages/sie_server/src/sie_server/bundle_requirements.py @@ -7,11 +7,23 @@ from __future__ import annotations import re -from collections.abc import Mapping +import tomllib +from collections.abc import Mapping, Sequence +from dataclasses import dataclass from hashlib import sha256 +from pathlib import Path from typing import cast +from packaging.requirements import Requirement +from packaging.version import InvalidVersion, Version + _CUDA_ONLY_PACKAGES = frozenset({"fla-core", "flash-attn", "xformers"}) +_EXACT_OPERATORS = frozenset({"==", "==="}) +_DOWNGRADE_EXAMPLES = ("sentence-transformers", "gliner") + + +def _normalize_package_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() def resolve_bundle_requirements( @@ -22,7 +34,7 @@ def resolve_bundle_requirements( """Convert a bundle ``deps`` mapping into canonical PEP 508 strings.""" requirements: list[str] = [] for package, constraint in bundle_deps.items(): - normalized = re.sub(r"[-_.]+", "-", package.lower()) + normalized = _normalize_package_name(package) if exclude_cuda and normalized in _CUDA_ONLY_PACKAGES: continue @@ -58,3 +70,377 @@ def bundle_requirements_sha256(bundle_deps: Mapping[str, object]) -> str: """Hash the exact normalized requirement payload baked by worker images.""" payload = "\n".join(normalized_bundle_requirements(bundle_deps)).encode() return sha256(payload).hexdigest() + + +@dataclass(frozen=True) +class LockEdge: + """One uv.lock dependency edge, including any requested extras.""" + + name: str + extras: frozenset[str] + + +@dataclass(frozen=True) +class UvLock: + """Package graph read from a uv.lock file. + + ``versions`` holds one constraint string per unambiguous package. Packages + with two public versions are listed in ``ambiguous`` and omitted from + ``versions`` so image builds skip them instead of aborting. ``dependencies`` + and ``optional_dependencies`` are the edges of the lock entries that the + constraint installs, keyed by normalized name. A shared public version is + emitted as ``2.9.1.*`` and takes edges from the public ``2.9.1`` entry + only, not from a local build such as ``2.9.1+cu129``. + """ + + versions: Mapping[str, str] + ambiguous: frozenset[str] + dependencies: Mapping[str, tuple[LockEdge, ...]] + optional_dependencies: Mapping[str, Mapping[str, tuple[LockEdge, ...]]] + names: Mapping[str, str] + + +def parse_uv_lock(source: str | Path) -> UvLock: + """Parse package tables from a uv.lock file into a dependency graph.""" + text = source.read_text(encoding="utf-8") if isinstance(source, Path) else source + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + msg = f"invalid uv.lock: {exc}" + raise ValueError(msg) from exc + packages = data.get("package", []) + if not isinstance(packages, list): + msg = "uv.lock package table is not a list" + raise ValueError(msg) + + grouped: dict[str, list[dict[str, object]]] = {} + names: dict[str, str] = {} + for package in packages: + if not isinstance(package, dict): + continue + raw_name = package.get("name") + raw_version = package.get("version") + if not isinstance(raw_name, str) or not isinstance(raw_version, str): + continue + normalized = _normalize_package_name(raw_name) + names.setdefault(normalized, raw_name) + grouped.setdefault(normalized, []).append(package) + + versions: dict[str, str] = {} + ambiguous: set[str] = set() + dependencies: dict[str, tuple[LockEdge, ...]] = {} + optional_dependencies: dict[str, dict[str, tuple[LockEdge, ...]]] = {} + for normalized, entries in grouped.items(): + found = {version for entry in entries if isinstance(version := entry.get("version"), str)} + resolved = _constraint_version(normalized, found) + if resolved is None: + ambiguous.add(normalized) + else: + versions[normalized] = resolved + base: set[LockEdge] = set() + optional: dict[str, set[LockEdge]] = {} + for entry in _entries_for_emitted_constraint(entries, resolved): + base.update(_lock_edges(entry.get("dependencies"))) + raw_optional = entry.get("optional-dependencies") + if not isinstance(raw_optional, dict): + continue + for extra, items in raw_optional.items(): + if isinstance(extra, str): + optional.setdefault(extra.lower(), set()).update(_lock_edges(items)) + dependencies[normalized] = _freeze_edges(base) + if optional: + optional_dependencies[normalized] = {extra: _freeze_edges(edges) for extra, edges in optional.items()} + + return UvLock( + versions=versions, + ambiguous=frozenset(ambiguous), + dependencies=dependencies, + optional_dependencies=optional_dependencies, + names=names, + ) + + +def locked_versions_from_uv_lock(source: str | Path) -> dict[str, str]: + """Return locked constraint versions keyed by normalized package name. + + One lock entry becomes that exact version string. Several entries that + share a public version (``torch`` ``2.9.1`` and ``2.9.1+cu129``) become a + prefix match, ``2.9.1.*``, so each locked build matches and a newer + release does not. Distinct public versions are omitted; the constraints + file records that skip instead of failing the image build. + """ + return dict(parse_uv_lock(source).versions) + + +def merge_locked_requirements(requirements: Sequence[str], locked_versions: Mapping[str, str]) -> list[str]: + """Pin ranged requirements to the lock unless the bundle spec overrides it. + + Exact ``==`` / ``===`` pins and URL or VCS specs are returned unchanged. + A range is rewritten only when a locked version still satisfies it, so a + bundle can deliberately move off the lock (transformers 5 versus a 4.x + pin) without an unsatisfiable constraint. Packages missing from the lock + stay as written. + """ + merged: list[str] = [] + for requirement in requirements: + parsed = Requirement(requirement) + if _is_bundle_override(parsed): + merged.append(requirement) + continue + pinned = locked_versions.get(_normalize_package_name(parsed.name)) + if pinned is None or not _locked_constraint_satisfies(parsed, pinned): + merged.append(requirement) + continue + merged.append(_format_locked_requirement(requirement, parsed, pinned)) + return merged + + +def lock_constraint_lines(requirements: Sequence[str], lock: UvLock) -> list[str]: + """Return constraints for the locked transitive closure of ``requirements``. + + Each pin is ``name==version`` with no extras and no markers. Pip rejects + extras in a constraints file, and a marker would leave the pin inactive on + other platforms. The file starts with a comment that ranged dependencies + follow ``uv.lock`` and can therefore be older than an unconstrained install. + + Exact ``==`` pins and URL or VCS specs stay out of the file. An exact pin + contributes lock neighbors only when it matches the locked version. A pin + that differs (``torch==2.11.0`` against a ``2.9.1`` lock) is an untrusted + root, same as a range that does not contain the lock: those edges describe + a distribution the bundle will not install. A neighbor an untrusted root + can reach is not pinned, even when a constrained root reaches it too. + Pinning that shared package to the lock can contradict the untrusted + distribution (``huggingface-hub`` 0.36 from ``sentence-transformers`` + versus ``huggingface-hub>=1`` from transformers 5). The constrained + requirement itself stays pinned. A package with two public versions is + skipped with a comment instead of aborting. Its combined outgoing edges + cannot establish trusted pins, but still exclude pins on untrusted paths. + """ + grouped = _requirement_groups(requirements) + constrained: list[tuple[str, frozenset[str]]] = [] + exact: list[tuple[str, frozenset[str]]] = [] + untrusted: list[tuple[str, frozenset[str]]] = [] + for name, reqs in grouped.items(): + kind = _root_kind(name, reqs, lock) + if kind is None: + continue + root = (name, _root_extras(reqs)) + if kind == "constrained": + constrained.append(root) + elif kind == "exact": + exact.append(root) + else: + untrusted.append(root) + + constrained_reach = _reachable(constrained, lock, follow_ambiguous=False) + exact_reach = _reachable(exact, lock, follow_ambiguous=False) + untrusted_reach = _reachable(untrusted, lock) + # A transitive package an untrusted root can reach stays unpinned, even + # when a constrained root reaches it too. The image installs the untrusted + # distribution, and the lock pin can make that solve fail. Direct + # constrained requirements stay pinned: dropping ``torch`` because + # transformers 5 also depends on it would float the bundle's own torch. + constrained_names = {name for name, _extras in constrained} + trusted = ((constrained_reach | exact_reach) - untrusted_reach) | constrained_names + + pins: list[str] = [] + skips: list[str] = [] + pinned_names: set[str] = set() + for name in sorted(trusted): + reqs = grouped.get(name, []) + if _pin_blocked(name, reqs, lock): + continue + display = lock.names.get(name, name) + if name in lock.ambiguous: + skips.append(f"# {display} skipped: uv.lock has multiple public versions") + continue + version = lock.versions.get(name) + if version is None: + continue + pins.append(f"{display}=={version}") + pinned_names.add(name) + return [*_constraint_header(pinned_names), *skips, *pins] + + +def _constraint_version(name: str, versions: set[str]) -> str | None: + if len(versions) == 1: + return next(iter(versions)) + try: + public_versions = {Version(version).public for version in versions} + except InvalidVersion as exc: + msg = f"uv.lock pins {name} to an invalid version" + raise ValueError(msg) from exc + if len(public_versions) != 1: + return None + return f"{next(iter(public_versions))}.*" + + +def _entries_for_emitted_constraint( + entries: Sequence[Mapping[str, object]], + constraint: str | None, +) -> list[Mapping[str, object]]: + """Edges for the build ``constraint`` installs, not every lock entry. + + A shared public version is emitted as ``2.9.1.*``. That installs the public + wheel (``2.9.1``), so a local build's edges (``2.9.1+cu129``) do not apply. + An exact version uses that build only. Ambiguous names keep every entry. + """ + if constraint is None: + return list(entries) + if constraint.endswith(".*"): + public = constraint.removesuffix(".*") + return [entry for entry in entries if entry.get("version") == public] + return [entry for entry in entries if entry.get("version") == constraint] + + +def _lock_edges(items: object) -> tuple[LockEdge, ...]: + if not isinstance(items, list): + return () + edges: list[LockEdge] = [] + for item in items: + if isinstance(item, str): + edges.append(LockEdge(_normalize_package_name(item), frozenset())) + continue + if not isinstance(item, dict): + continue + fields = cast("Mapping[str, object]", item) + raw_name = fields.get("name") + if not isinstance(raw_name, str): + continue + extras = _lock_extras(fields.get("extra", fields.get("extras"))) + edges.append(LockEdge(_normalize_package_name(raw_name), extras)) + return tuple(edges) + + +def _lock_extras(value: object) -> frozenset[str]: + if isinstance(value, str): + return frozenset({value.lower()}) + if isinstance(value, list): + return frozenset(item.lower() for item in value if isinstance(item, str)) + return frozenset() + + +def _freeze_edges(edges: set[LockEdge]) -> tuple[LockEdge, ...]: + return tuple(sorted(edges, key=lambda edge: (edge.name, sorted(edge.extras)))) + + +def _requirement_groups(requirements: Sequence[str]) -> dict[str, list[Requirement]]: + grouped: dict[str, list[Requirement]] = {} + for requirement in requirements: + parsed = Requirement(requirement) + grouped.setdefault(_normalize_package_name(parsed.name), []).append(parsed) + return grouped + + +def _root_kind(name: str, reqs: Sequence[Requirement], lock: UvLock) -> str | None: + """Classify a direct requirement as constrained, exact override, or untrusted. + + Untrusted roots are URL/VCS specs, ranges that do not contain the locked + version, and exact pins of a different version. Their lock edges belong to + a distribution the bundle will not install, so those edges must not become + constraints. An exact pin that matches the locked version still contributes + its edges. + """ + if name not in lock.versions and name not in lock.ambiguous: + return None + if any(req.url is not None for req in reqs): + return "untrusted" + version = lock.versions.get(name) + overrides = [req for req in reqs if _is_bundle_override(req)] + if overrides: + if version is not None and all(_locked_constraint_satisfies(req, version) for req in overrides): + return "exact" + return "untrusted" + if version is not None and any(not _locked_constraint_satisfies(req, version) for req in reqs): + return "untrusted" + return "constrained" + + +def _root_extras(reqs: Sequence[Requirement]) -> frozenset[str]: + extras: set[str] = set() + for req in reqs: + extras.update(extra.lower() for extra in req.extras) + return frozenset(extras) + + +def _reachable(roots: Sequence[tuple[str, frozenset[str]]], lock: UvLock, *, follow_ambiguous: bool = True) -> set[str]: + seen_packages: set[str] = set() + seen_extras: set[tuple[str, str]] = set() + stack = list(roots) + while stack: + name, extras = stack.pop() + if name not in lock.versions and name not in lock.ambiguous: + continue + if not follow_ambiguous and name in lock.ambiguous: + # Keep the skip diagnostic, but not edges from an unknown version. + seen_packages.add(name) + continue + if name not in seen_packages: + seen_packages.add(name) + for edge in lock.dependencies.get(name, ()): + stack.append((edge.name, edge.extras)) + for extra in extras: + key = (name, extra) + if key in seen_extras: + continue + seen_extras.add(key) + for edge in lock.optional_dependencies.get(name, {}).get(extra, ()): + stack.append((edge.name, edge.extras)) + return seen_packages + + +def _pin_blocked(name: str, reqs: Sequence[Requirement], lock: UvLock) -> bool: + if not reqs: + return False + if any(_is_bundle_override(req) for req in reqs): + return True + version = lock.versions.get(name) + if version is None: + return False + return any(not _locked_constraint_satisfies(req, version) for req in reqs) + + +def _constraint_header(pinned_names: set[str]) -> list[str]: + lines = [ + "# Ranged dependencies are pinned to uv.lock. This can install an older", + "# release than an unconstrained build of the same commit.", + ] + present = [name for name in _DOWNGRADE_EXAMPLES if name in pinned_names] + if not present: + return lines + shown = " and ".join(present) + verb = "follow" if len(present) > 1 else "follows" + lines.append( + f"# For example, {shown} {verb} uv.lock rather than the newest release that still matches the bundle range." + ) + return lines + + +def _locked_constraint_satisfies(requirement: Requirement, constraint: str) -> bool: + """A prefix pin is usable when its public version is inside the bundle spec.""" + candidate = constraint.removesuffix(".*") if constraint.endswith(".*") else constraint + try: + version = Version(candidate) + except InvalidVersion: + return False + return requirement.specifier.contains(version, prereleases=True) + + +def _is_bundle_override(requirement: Requirement) -> bool: + """Exact pins and direct URL/VCS references are deliberate bundle overrides.""" + if requirement.url is not None: + return True + specs = list(requirement.specifier) + if len(specs) != 1: + return False + spec = specs[0] + return spec.operator in _EXACT_OPERATORS and not spec.version.endswith(".*") + + +def _format_locked_requirement(raw: str, requirement: Requirement, version: str) -> str: + extras = f"[{','.join(sorted(requirement.extras))}]" if requirement.extras else "" + pinned = f"{requirement.name}{extras}=={version}" + if requirement.marker is None: + return pinned + marker = raw.split(";", maxsplit=1)[1].strip() + return f"{pinned} ; {marker}" diff --git a/packages/sie_server/src/sie_server/cli.py b/packages/sie_server/src/sie_server/cli.py index ad9b2b1d7..e9afa39c5 100644 --- a/packages/sie_server/src/sie_server/cli.py +++ b/packages/sie_server/src/sie_server/cli.py @@ -339,6 +339,12 @@ def resolve_deps( models_dir: str = typer.Option(DEFAULT_MODELS_DIR, "--models-dir", help="Models directory"), output_json: bool = typer.Option(False, "--json", help="Output as JSON"), cpu: bool = typer.Option(False, "--cpu", help="Exclude CUDA-only dependencies (flash-attn)"), + lock: Path | None = typer.Option(None, "--lock", help="uv.lock used to pin ranged bundle dependencies"), # noqa: B008 + constraints_output: Path | None = typer.Option( # noqa: B008 + None, + "--constraints-output", + help="Write uv.lock constraints for the bundle's locked transitive closure", + ), ) -> None: """Resolve and print dependencies for a bundle or model list. @@ -347,9 +353,25 @@ def resolve_deps( Use --cpu flag when building CPU-only images to exclude flash-attn and other CUDA-only dependencies. + + ``--lock`` and ``--constraints-output`` together write a constraints file + that pins ranged dependencies and their locked transitive closure to + ``uv.lock``. Lines are ``name==version`` only (pip rejects extras). Exact + pins, URL/VCS specs, and ranges that do not contain the locked version are + not tightened. The file starts with a comment that the pins can install an + older release than an unconstrained build of the same commit. Stdout stays + the bundle requirements. """ + from sie_server.bundle_requirements import lock_constraint_lines, parse_uv_lock from sie_server.core.deps import collect_bundle_deps + if (lock is None) != (constraints_output is None): + typer.echo("Error: --lock and --constraints-output must be used together", err=True) + raise typer.Exit(1) + if output_json and constraints_output is not None: + typer.echo("Error: --constraints-output cannot be combined with --json", err=True) + raise typer.Exit(1) + models_path = Path(models_dir).resolve() bundles_dir = _DEFAULT_BUNDLES_DIR pool_name = os.environ.get(ENV_POOL) or None @@ -386,6 +408,15 @@ def resolve_deps( typer.echo(f" - {conflict}", err=True) raise typer.Exit(1) + if constraints_output is not None and lock is not None: + try: + parsed_lock = parse_uv_lock(lock) + except (OSError, ValueError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(1) from exc + body = "\n".join(lock_constraint_lines(result.requirements, parsed_lock)) + constraints_output.write_text(f"{body}\n" if body else "", encoding="utf-8") + if output_json: import json diff --git a/packages/sie_server/tests/core/test_bundle_image_resolution.py b/packages/sie_server/tests/core/test_bundle_image_resolution.py new file mode 100644 index 000000000..1a8a5e5b1 --- /dev/null +++ b/packages/sie_server/tests/core/test_bundle_image_resolution.py @@ -0,0 +1,188 @@ +"""Resolve each release-matrix bundle the way its image install does. + +The image build runs on linux/amd64. ``uv pip install --dry-run`` applies that +marker environment on any host, with the same ``--target``, constraint, and +requirements arguments as the Dockerfiles. pip on macOS evaluates markers for +Darwin and misses the linux nvidia conflict those constraints used to create. +``--no-config`` keeps the repo's uv torch index from standing in for PyPI. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest +import yaml +from sie_server.bundle_requirements import lock_constraint_lines, parse_uv_lock, resolve_bundle_requirements + +_REPO_ROOT = Path(__file__).resolve().parents[4] +_BUNDLES = _REPO_ROOT / "packages" / "sie_server" / "bundles" +# CUDA 12 and 13 images are Ubuntu 22.04 (glibc 2.35). manylinux_2_28 rejects +# wheels such as sglang 0.5.20, which publish only manylinux_2_34 tags. +_LINUX_PLATFORM = "x86_64-manylinux_2_35" +_PYTHON_VERSION = "3.12" +# Dockerfile.cuda12's non-default index branch. Embedding is not in the +# release matrix today; keep the same case split as the image. +_SGLANG_CUDA12_BUNDLES = frozenset({"sglang", "sglang-embedding", "sglang-vision-extract"}) +_SCRUBBED_ENV = ( + "UV_INDEX", + "UV_INDEX_URL", + "UV_EXTRA_INDEX_URL", + "UV_DEFAULT_INDEX", + "UV_INDEX_STRATEGY", + "UV_TORCH_BACKEND", + "UV_CONSTRAINT", + "UV_OVERRIDE", + "PIP_INDEX_URL", + "PIP_EXTRA_INDEX_URL", + "PIP_CONSTRAINT", +) + + +def _release_targets() -> list[tuple[str, str]]: + data = json.loads((_REPO_ROOT / ".github" / "release-matrix.json").read_text()) + targets = [(str(platform), str(bundle)) for platform in data["platforms"] for bundle in data["bundles"]] + targets.extend((str(item["platform"]), str(item["bundle"])) for item in data.get("include", [])) + return targets + + +_TARGETS = _release_targets() + + +def _uv_executable() -> str: + sibling = Path(sys.executable).resolve().with_name("uv") + if sibling.is_file() and os.access(sibling, os.X_OK): + return str(sibling) + found = shutil.which("uv") + if found: + return found + pytest.fail( + "SIE_RUN_IMAGE_RESOLUTION=1 but uv is not installed; the resolution job must not pass without resolving" + ) + + +def _image_requirements(platform: str, bundle: str) -> list[str]: + bundle_path = _BUNDLES / f"{bundle}.yaml" + deps = yaml.safe_load(bundle_path.read_text())["deps"] + # Dockerfile.cpu passes --cpu, which drops flash-attn, xformers, and fla-core. + return resolve_bundle_requirements(deps, exclude_cuda=platform == "cpu") + + +def _resolve_command(platform: str, bundle: str, target: Path, requirements: Path, constraints: Path) -> list[str]: + """Arguments that affect resolution, matched to the platform Dockerfile.""" + command = [ + _uv_executable(), + "pip", + "install", + "--dry-run", + "--no-config", + "--python-platform", + _LINUX_PLATFORM, + "--python-version", + _PYTHON_VERSION, + "--target", + str(target), + ] + if platform == "cuda13": + # packages/sie_server/Dockerfile.cuda13 + command.extend( + [ + "--index-strategy", + "unsafe-best-match", + "--prerelease=allow", + "--extra-index-url", + "https://download.pytorch.org/whl/cu130", + "--extra-index-url", + "https://docs.sglang.ai/whl/cu130/", + "--constraint", + str(constraints), + ] + ) + elif platform == "cpu": + # packages/sie_server/Dockerfile.cpu. pip searches every configured index; + # uv's default first-index strategy would not. + command.extend( + [ + "--index-url", + "https://download.pytorch.org/whl/cpu", + "--extra-index-url", + "https://pypi.org/simple", + "--index-strategy", + "unsafe-best-match", + "-c", + str(constraints), + ] + ) + else: + # packages/sie_server/Dockerfile.cuda12 + if bundle in _SGLANG_CUDA12_BUNDLES: + command.extend( + [ + "--extra-index-url", + "https://docs.sglang.ai/whl/cu129/", + "--index-strategy", + "unsafe-best-match", + ] + ) + command.extend(["-c", str(constraints)]) + command.extend(["-r", str(requirements)]) + return command + + +def _resolver_env() -> dict[str, str]: + env = os.environ.copy() + for key in _SCRUBBED_ENV: + env.pop(key, None) + return env + + +def test_missing_uv_fails_the_opt_in(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(sys, "executable", str(tmp_path / "python")) + monkeypatch.setattr(shutil, "which", lambda _name: None) + with pytest.raises(pytest.fail.Exception, match="uv is not installed"): + _uv_executable() + + +@pytest.mark.skipif( + os.environ.get("SIE_RUN_IMAGE_RESOLUTION") != "1", + reason="needs network access to package indexes", +) +@pytest.mark.parametrize( + ("platform", "bundle"), + _TARGETS, + ids=[f"{platform}-{bundle}" for platform, bundle in _TARGETS], +) +def test_release_bundle_constraints_resolve(platform: str, bundle: str) -> None: + requirements = _image_requirements(platform, bundle) + if not requirements: + return + constraints = lock_constraint_lines(requirements, parse_uv_lock(_REPO_ROOT / "uv.lock")) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + requirements_path = root / "requirements.txt" + constraints_path = root / "constraints.txt" + target = root / "bundle-libs" + target.mkdir() + requirements_path.write_text("\n".join(requirements) + "\n", encoding="utf-8") + constraints_path.write_text("\n".join(constraints) + "\n", encoding="utf-8") + command = _resolve_command(platform, bundle, target, requirements_path, constraints_path) + assert str(target) in command + assert str(requirements_path) in command + assert str(constraints_path) in command + completed = subprocess.run( # noqa: S603 - fixed uv resolution command built above + command, + check=False, + capture_output=True, + text=True, + env=_resolver_env(), + cwd=tmp, + timeout=900, + ) + details = (completed.stderr or completed.stdout or "")[-4000:] + assert completed.returncode == 0, details diff --git a/packages/sie_server/tests/core/test_bundle_requirements.py b/packages/sie_server/tests/core/test_bundle_requirements.py index 6a1d8b2f9..5a445e2a9 100644 --- a/packages/sie_server/tests/core/test_bundle_requirements.py +++ b/packages/sie_server/tests/core/test_bundle_requirements.py @@ -1,11 +1,24 @@ from __future__ import annotations +from pathlib import Path + +import pytest +import yaml +from packaging.requirements import Requirement from sie_server.bundle_requirements import ( + UvLock, bundle_requirements_sha256, + lock_constraint_lines, + locked_versions_from_uv_lock, + merge_locked_requirements, normalized_bundle_requirements, + parse_uv_lock, resolve_bundle_requirements, ) +_REPO_ROOT = Path(__file__).resolve().parents[4] +_BUNDLES = _REPO_ROOT / "packages" / "sie_server" / "bundles" + def test_resolve_bundle_requirements_preserves_cli_semantics() -> None: assert resolve_bundle_requirements( @@ -38,3 +51,492 @@ def test_release_pin_normalization_is_sorted_and_marker_free() -> None: assert normalized_bundle_requirements(deps) == ["a-first==1", "z-last==2"] assert bundle_requirements_sha256(deps) == "d2ccdab8db5e0df55e2b47ae7c7a8f4c9b7623bd42fb7c09f028114ed3aba884" + + +def _bundle_requirements(name: str) -> list[str]: + bundle = yaml.safe_load((_BUNDLES / f"{name}.yaml").read_text()) + return resolve_bundle_requirements(bundle["deps"]) + + +def test_default_bundle_ranges_pin_to_uv_lock_versions() -> None: + locked = locked_versions_from_uv_lock(_REPO_ROOT / "uv.lock") + original = _bundle_requirements("default") + merged = merge_locked_requirements(original, locked) + + assert "gliner>=0.2.26,<1" in original + assert f"gliner=={locked['gliner']}" in merged + assert "gliner>=0.2.26,<1" not in merged + assert f"torch=={locked['torch']}" in merged + assert f"requests=={locked['requests']}" in merged + + +def test_exact_pins_and_url_specs_override_the_lock() -> None: + parsed = parse_uv_lock(_REPO_ROOT / "uv.lock") + locked = parsed.versions + original = _bundle_requirements("transformers5") + merged = merge_locked_requirements(original, locked) + url_specs = [requirement for requirement in original if " @" in requirement] + + assert url_specs + assert "gliner2==2.0.0" in merged + # Exact bundle pin, not the lock's ``0.24.1.*`` prefix constraint. + torchvision_lines = [requirement for requirement in merged if requirement.startswith("torchvision==")] + assert torchvision_lines == ["torchvision==0.24.1"] + for requirement in url_specs: + assert requirement in merged + # Ranges that do not contain the locked version stay as written. + declared = [raw for raw in original if Requirement(raw).name == "transformers"] + assert len(declared) == 1 + assert not Requirement(declared[0]).specifier.contains(locked["transformers"], prereleases=True) + assert declared[0] in merged + # An exact override remains a requirement even when it matches the lock. + assert f"sentence-transformers=={locked['sentence-transformers']}" in original + assert f"sentence-transformers=={locked['sentence-transformers']}" in merged + constraints = lock_constraint_lines(original, parsed) + assert "gliner2==2.0.0" not in constraints + assert f"gliner2=={locked['gliner2']}" not in constraints + assert not any(line.startswith("torchvision==") for line in constraints) + assert all(" @" not in line for line in constraints) + assert f"transformers=={locked['transformers']}" not in constraints + assert f"sentence-transformers=={locked['sentence-transformers']}" not in constraints + # The 4.x lock's huggingface-hub and safetensors do not satisfy transformers 5. + assert f"huggingface-hub=={locked['huggingface-hub']}" not in constraints + assert f"safetensors=={locked['safetensors']}" not in constraints + # The TF5 override also prevents pinning shared transitive dependencies. + # torch stays because the bundle range still contains the locked version. + assert f"peft=={locked['peft']}" not in constraints + assert f"torch=={locked['torch']}" in constraints + + +def test_merge_leaves_url_specs_and_unlocked_packages_unchanged() -> None: + locked = {"gliner": "0.2.26", "demo": "9.9.9"} + requirements = [ + "gliner>=0.2.26,<1 ; sys_platform == 'linux'", + "demo @ git+https://example.com/demo.git@abc123", + "wheel @ https://example.com/wheel.whl ; sys_platform == 'linux'", + "absent>=1,<2", + "pinned==3", + ] + + assert merge_locked_requirements(requirements, locked) == [ + "gliner==0.2.26 ; sys_platform == 'linux'", + "demo @ git+https://example.com/demo.git@abc123", + "wheel @ https://example.com/wheel.whl ; sys_platform == 'linux'", + "absent>=1,<2", + "pinned==3", + ] + + +def test_local_version_builds_share_one_prefix_constraint() -> None: + lock = """ +version = 1 +[[package]] +name = "torch" +version = "2.9.1" + +[[package]] +name = "torch" +version = "2.9.1+cu129" +""" + assert locked_versions_from_uv_lock(lock) == {"torch": "2.9.1.*"} + assert merge_locked_requirements(["torch>=2.9,<2.10"], {"torch": "2.9.1.*"}) == ["torch==2.9.1.*"] + constraints = lock_constraint_lines(["torch>=2.9,<2.10"], parse_uv_lock(lock)) + assert "torch==2.9.1.*" in constraints + assert not any("skipped:" in line for line in constraints) + + +def test_default_bundle_constraints_pin_transitive_deps_and_name_the_downgrade() -> None: + parsed = parse_uv_lock(_REPO_ROOT / "uv.lock") + constraints = lock_constraint_lines(_bundle_requirements("default"), parsed) + + assert constraints[0] == "# Ranged dependencies are pinned to uv.lock. This can install an older" + assert "older" in constraints[0] + assert "release than an unconstrained build" in constraints[1] + assert any(line.startswith("# For example, gliner follows ") for line in constraints) + assert f"onnxruntime=={parsed.versions['onnxruntime']}" in constraints + assert f"docling-core=={parsed.versions['docling-core']}" in constraints + assert f"gliner=={parsed.versions['gliner']}" in constraints + # Exact bundle pins stay requirements, not constraints. + assert not any(line.startswith("sentence-transformers==") for line in constraints) + assert not any(line.startswith("pillow==") for line in constraints) + assert not any(line.startswith("gliformer==") for line in constraints) + assert all("[" not in line for line in constraints) + assert f"chromadb=={parsed.versions['chromadb']}" not in constraints + # The public torch pin must not import the cu129 wheel's nvidia stack. + assert not any(line.startswith("nvidia-cublas-cu12==") for line in constraints) + + +def test_constraints_pin_transitive_closure_not_the_whole_lock() -> None: + lock = """ +version = 1 +[[package]] +name = "direct" +version = "1.2.0" +dependencies = [ + { name = "onnxruntime" }, + { name = "docling-core" }, +] + +[[package]] +name = "onnxruntime" +version = "1.25.0" + +[[package]] +name = "docling-core" +version = "2.79.0" +dependencies = [ + { name = "leaf" }, +] + +[[package]] +name = "leaf" +version = "8.0.0" + +[[package]] +name = "outside" +version = "9.0.0" +""" + constraints = lock_constraint_lines(["direct>=1,<2"], parse_uv_lock(lock)) + assert "direct==1.2.0" in constraints + assert "onnxruntime==1.25.0" in constraints + assert "docling-core==2.79.0" in constraints + assert "leaf==8.0.0" in constraints + assert "outside==9.0.0" not in constraints + + +def test_constraint_lines_strip_extras() -> None: + lock = """ +version = 1 +[[package]] +name = "demo" +version = "1.2.3" +dependencies = [ + { name = "base" }, +] + +[package.optional-dependencies] +extra = [ + { name = "optional-dep" }, +] + +[[package]] +name = "base" +version = "4.0.0" + +[[package]] +name = "optional-dep" +version = "5.0.0" + +[[package]] +name = "not-requested" +version = "6.0.0" +""" + constraints = lock_constraint_lines( + ["demo[extra]>=1,<2 ; sys_platform == 'linux'"], + parse_uv_lock(lock), + ) + assert "demo==1.2.3" in constraints + assert "base==4.0.0" in constraints + assert "optional-dep==5.0.0" in constraints + assert "not-requested==6.0.0" not in constraints + assert all("[" not in line for line in constraints) + assert all(";" not in line for line in constraints) + + +def test_exact_override_neighbors_stay_pinned_only_when_the_pin_matches_the_lock() -> None: + lock = """ +version = 1 +[[package]] +name = "kept" +version = "1.0.0" +dependencies = [ + { name = "shared" }, +] + +[[package]] +name = "matched" +version = "1.0.0" +dependencies = [ + { name = "matched-neighbor" }, + { name = "contested" }, +] + +[[package]] +name = "pinned-override" +version = "1.0.0" +dependencies = [ + { name = "neighbor" }, + { name = "contested" }, +] + +[[package]] +name = "moved" +version = "4.0.0" +dependencies = [ + { name = "old-only" }, + { name = "contested" }, +] + +[[package]] +name = "neighbor" +version = "2.0.0" + +[[package]] +name = "matched-neighbor" +version = "2.1.0" + +[[package]] +name = "shared" +version = "3.0.0" + +[[package]] +name = "old-only" +version = "5.0.0" + +[[package]] +name = "contested" +version = "7.0.0" +""" + constraints = lock_constraint_lines( + ["kept>=1,<2", "matched==1.0.0", "pinned-override==9.9.9", "moved>=5,<6"], + parse_uv_lock(lock), + ) + assert "kept==1.0.0" in constraints + assert "shared==3.0.0" in constraints + assert "matched-neighbor==2.1.0" in constraints + assert "neighbor==2.0.0" not in constraints + assert not any(line.startswith("matched==") for line in constraints) + assert not any(line.startswith("pinned-override==") for line in constraints) + assert not any(line.startswith("moved==") for line in constraints) + assert "old-only==5.0.0" not in constraints + assert "contested==7.0.0" not in constraints + + +def test_untrusted_root_unpins_a_transitive_also_reached_by_a_constrained_root() -> None: + lock = """ +version = 1 +[[package]] +name = "sentence-transformers" +version = "5.4.1" +dependencies = [ + { name = "huggingface-hub" }, +] + +[[package]] +name = "transformers" +version = "4.57.6" +dependencies = [ + { name = "huggingface-hub" }, +] + +[[package]] +name = "huggingface-hub" +version = "0.36.2" +""" + constraints = lock_constraint_lines( + ["sentence-transformers>=5.4.1,<6", "transformers>=5.14,<6"], + parse_uv_lock(lock), + ) + assert "sentence-transformers==5.4.1" in constraints + assert "huggingface-hub==0.36.2" not in constraints + assert not any(line.startswith("transformers==") for line in constraints) + + +def test_public_prefix_constraint_ignores_local_build_edges() -> None: + lock = """ +version = 1 +[[package]] +name = "torch" +version = "2.9.1" +dependencies = [ + { name = "filelock" }, +] + +[[package]] +name = "torch" +version = "2.9.1+cu129" +dependencies = [ + { name = "filelock" }, + { name = "nvidia-cublas-cu12" }, +] + +[[package]] +name = "filelock" +version = "3.0.0" + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.9.1.4" + +[[package]] +name = "torch-local-only" +version = "2.9.1+cu129" +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] + +[[package]] +name = "torch-local-only" +version = "2.9.1+cu130" +dependencies = [ + { name = "nvidia-cublas-cu13" }, +] + +[[package]] +name = "nvidia-cublas-cu13" +version = "13.0.0" +""" + parsed = parse_uv_lock(lock) + assert parsed.versions["torch"] == "2.9.1.*" + assert parsed.versions["torch-local-only"] == "2.9.1.*" + public = lock_constraint_lines(["torch>=2.9,<2.10"], parsed) + assert "torch==2.9.1.*" in public + assert "filelock==3.0.0" in public + assert "nvidia-cublas-cu12==12.9.1.4" not in public + local_only = lock_constraint_lines(["torch-local-only>=2.9,<2.10"], parsed) + assert "torch-local-only==2.9.1.*" in local_only + assert "nvidia-cublas-cu12==12.9.1.4" not in local_only + assert "nvidia-cublas-cu13==13.0.0" not in local_only + + +def test_sole_local_build_keeps_its_own_edges() -> None: + lock = """ +version = 1 +[[package]] +name = "torch" +version = "2.9.1+cu129" +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.9.1.4" +""" + constraints = lock_constraint_lines(["torch>=2.9,<2.10"], parse_uv_lock(lock)) + assert "torch==2.9.1+cu129" in constraints + assert "nvidia-cublas-cu12==12.9.1.4" in constraints + + +def test_moved_exact_pins_do_not_contribute_lock_edges() -> None: + parsed = parse_uv_lock(_REPO_ROOT / "uv.lock") + constraints = lock_constraint_lines(_bundle_requirements("tensorrt-llm"), parsed) + + assert f"huggingface-hub=={parsed.versions['huggingface-hub']}" not in constraints + assert f"setuptools=={parsed.versions['setuptools']}" not in constraints + assert not any(line.startswith("nvidia-cublas-cu12==") for line in constraints) + assert not any(line.startswith("torch==") for line in constraints) + assert not any(line.startswith("transformers==") for line in constraints) + + +def test_multiple_public_versions_are_skipped_without_raising() -> None: + lock = """ +version = 1 +[[package]] +name = "direct" +version = "1.0.0" +dependencies = [ + { name = "split" }, + { name = "stable" }, +] + +[[package]] +name = "split" +version = "1.0.0" +dependencies = [ + { name = "stable" }, +] + +[[package]] +name = "split" +version = "2.0.0" + +[[package]] +name = "stable" +version = "3.1.0" +""" + parsed = parse_uv_lock(lock) + assert locked_versions_from_uv_lock(lock) == {"direct": "1.0.0", "stable": "3.1.0"} + assert "split" in parsed.ambiguous + constraints = lock_constraint_lines(["direct>=1,<2"], parsed) + assert "direct==1.0.0" in constraints + assert "stable==3.1.0" in constraints + assert not any(line.startswith("split==") for line in constraints) + assert "# split skipped: uv.lock has multiple public versions" in constraints + + +@pytest.fixture +def ambiguous_dependency_lock() -> UvLock: + return parse_uv_lock(""" +version = 1 +[[package]] +name = "parent" +version = "1.0.0" +dependencies = [{ name = "split", extra = ["feature"] }] + +[[package]] +name = "split" +version = "1.0.0" +dependencies = [{ name = "old-only" }] +[package.optional-dependencies] +feature = [{ name = "optional-only" }] + +[[package]] +name = "split" +version = "2.0.0" +dependencies = [{ name = "new-only" }] + +[[package]] +name = "independent" +version = "1.0.0" +dependencies = [{ name = "old-only" }, { name = "optional-only" }] + +[[package]] +name = "untrusted" +version = "1.0.0" +dependencies = [{ name = "split", extra = ["feature"] }] + +[[package]] +name = "old-only" +version = "3.0.0" +dependencies = [{ name = "leaf" }] + +[[package]] +name = "leaf" +version = "4.0.0" + +[[package]] +name = "optional-only" +version = "5.0.0" + +[[package]] +name = "new-only" +version = "6.0.0" +""") + + +@pytest.mark.parametrize("root", ["split[feature]>=2", "parent>=1", "parent==1.0.0"]) +@pytest.mark.parametrize("independent", [False, True]) +def test_ambiguous_dependencies_need_an_independent_trusted_path( + ambiguous_dependency_lock: UvLock, root: str, independent: bool +) -> None: + requirements = [root, "independent>=1"] if independent else [root] + constraints = lock_constraint_lines(requirements, ambiguous_dependency_lock) + + assert "# split skipped: uv.lock has multiple public versions" in constraints + assert not any(line.startswith("split==") for line in constraints) + # Neither version's combined dependency graph describes a known install. + assert "new-only==6.0.0" not in constraints + for pin in ("old-only==3.0.0", "optional-only==5.0.0", "leaf==4.0.0"): + assert (pin in constraints) is independent + + +@pytest.mark.parametrize("untrusted", ["untrusted==9", "untrusted @ https://example.com/untrusted.whl"]) +def test_untrusted_paths_through_ambiguous_dependencies_still_unpin_shared_neighbors( + ambiguous_dependency_lock: UvLock, untrusted: str +) -> None: + constraints = lock_constraint_lines(["independent>=1", untrusted], ambiguous_dependency_lock) + + assert "independent==1.0.0" in constraints + for pin in ("old-only==3.0.0", "optional-only==5.0.0", "leaf==4.0.0"): + assert pin not in constraints diff --git a/tools/ci/required_ci.py b/tools/ci/required_ci.py index 47808ef75..294562b8f 100644 --- a/tools/ci/required_ci.py +++ b/tools/ci/required_ci.py @@ -10,6 +10,7 @@ "policy", "bootstrap", "python", + "bundle-resolution", "typescript", "rust", "contracts", diff --git a/tools/ci/tests/test_docker_task.py b/tools/ci/tests/test_docker_task.py index 2fe48173f..129737201 100644 --- a/tools/ci/tests/test_docker_task.py +++ b/tools/ci/tests/test_docker_task.py @@ -309,6 +309,52 @@ def fail_verification(*_args, **_kwargs): assert commands == [] +def _dockerfile_logical_lines(text: str) -> list[str]: + logical: list[str] = [] + buffer = "" + for line in text.splitlines(): + stripped = line.strip() + buffer = f"{buffer} {stripped}".strip() if buffer else stripped + if buffer.endswith("\\"): + buffer = buffer[:-1].rstrip() + continue + logical.append(buffer) + buffer = "" + if buffer: + logical.append(buffer) + return logical + + +@pytest.mark.parametrize( + "dockerfile", + ["Dockerfile.cpu", "Dockerfile.cuda12", "Dockerfile.cuda13"], +) +def test_bundle_image_installs_are_constrained_to_the_lock(dockerfile: str) -> None: + text = (Path(__file__).resolve().parents[3] / "packages" / "sie_server" / dockerfile).read_text() + assert "COPY uv.lock /tmp/uv.lock" in text + assert "--constraints-output /tmp/bundle-constraints.txt" in text + assert "pip freeze --path /app/bundle-libs > /app/bundle-libs-freeze.txt" in text + assert "/app/bundle-libs-freeze.txt /app/bundle-libs-freeze.txt" in text + installs = [ + line + for line in _dockerfile_logical_lines(text) + if "-r /tmp/bundle-requirements.txt" in line and ("pip install" in line or "uv pip install" in line) + ] + assert installs + constraints_file = "/tmp/bundle-constraints.txt" # noqa: S108 + # A case/if arm ends at `;;`. One logical RUN can hold both CUDA 12 + # branches, so each install command is checked on its own. + commands = [ + part.strip() + for line in installs + for part in line.split(";;") + if "-r /tmp/bundle-requirements.txt" in part and ("pip install" in part or "uv pip install" in part) + ] + assert commands + for command in commands: + assert f"-c {constraints_file}" in command or f"--constraint {constraints_file}" in command + + def test_expected_release_set_has_sixteen_tags_and_six_names(complete_source): images = docker_task.expected_versioned_images("ghcr.io/superlinked", VERSION, complete_source) assert len(images) == len(set(images)) == 16