From afc27c366a7385b4f966ff02318935f1237ada1e Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Wed, 4 Nov 2026 12:16:32 +0100 Subject: [PATCH 1/4] feat(sources): resolve a release through an ordered list of sources A build and a read must find each release in one ordered list of sources: registries and the installed connectors. This change adds the parts that answer that question. The build and the client do not use them yet. The new module timenet.sources defines ReleaseSource, the union of BaseRegistry and InstalledConnectors. A source has the method resolve(dataset_id, version), where version is a Version, or None for the newest version of that source. A source gives one of four answers, a ReleaseLookup. FoundInRegistry names the version and the registry that serves the release. FoundInConnector keeps the InstalledBuilder that the lookup read: the backend and the card that build the release. Found is the union of the two. NotFound means that the source does not know the release. NotServed means that the source lists the release but does not serve its files. NotServed keeps the access page of each card that is not open, and access_error() turns it into a TimeNetAccessError. resolve_release() parses the version once and asks the sources in order. An exact version comes from the first source that serves it. Without a version (None or an empty string), the first source that serves the dataset gives its newest version. So a copy in the first source answers, and no later source is asked. "latest" is the highest version that any source provides, and the order decides only which source serves it. NotFound and NotServed fall through, and any other error stops the resolution. A remote registry that cannot be reached raises TimeNetRegistryError, which names the registry, not a raw httpx error. This also holds for the parallel download of a release. An S3 registry that cannot reach its bucket raises TimeNetRegistryError, which names the bucket. Every registry is now a source. BaseRegistry.resolve() reads the manifest. If the registry requires open access, it answers NotServed for a release whose effective access is not open. LocalRegistry.resolve() serves every release that it holds. It looks only for the manifest of the release, so a miss does not read every manifest in the registry. InstalledConnectors() answers with the version that the card of the connector declares. FoundInRegistry and NotServed keep the manifest that the source read to answer, in an optional manifest field. A caller can use it and does not fetch the same manifest again. The field takes no part in equality or in the repr. In every registry method that takes a version string, None, an empty string and "latest" all mean the newest version of that registry. split_ref() now keeps "latest", so org/x@latest and org/x are two different requests. A build reads both as the version of the card, because a connector builds only that version. The API docs generator now names a page after its full module path if both packages have a module with that name, because timenet.sources and timenet_connectors.sources both mapped to sources.md. BREAKING CHANGE: A remote registry that cannot be reached raises TimeNetRegistryError, not an httpx error. An S3 registry that cannot reach its bucket raises TimeNetRegistryError, not a botocore error. split_ref() returns "latest" for a reference that ends with @latest, not None. So a TimeNet method that gets such a reference and also a version argument raises TimeFValidationError. Part of TNET-157. --- .../src/timenet_connectors/builder/cli.py | 4 +- .../tests/test_build_backend.py | 7 + .../timenet/src/timenet/coordinator/build.py | 5 +- packages/timenet/src/timenet/registry/base.py | 27 ++- .../timenet/src/timenet/registry/local.py | 30 ++- .../src/timenet/registry/remote/_http.py | 43 +++- .../src/timenet/registry/remote/registry.py | 9 +- packages/timenet/src/timenet/registry/s3.py | 56 +++-- packages/timenet/src/timenet/sources.py | 193 ++++++++++++++++++ .../timenet/src/timenet/types/references.py | 16 +- .../tests/registry/test_conformance.py | 13 ++ .../timenet/tests/registry/test_remote.py | 77 ++++++- packages/timenet/tests/registry/test_s3.py | 20 +- packages/timenet/tests/test_refs.py | 2 +- packages/timenet/tests/test_sources.py | 158 ++++++++++++++ scripts/gen_api_docs.py | 15 +- zensical.toml | 2 +- 17 files changed, 622 insertions(+), 55 deletions(-) create mode 100644 packages/timenet/src/timenet/sources.py create mode 100644 packages/timenet/tests/test_sources.py diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index 9207ee93b..7b43b2d7f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -33,7 +33,7 @@ open_writable_registry, ) from timenet.registry._paths import release_dir # ruff: ignore[import-private-name] -from timenet.types.references import DatasetRef, split_ref +from timenet.types.references import DatasetRef, is_latest, split_ref from timenet_connectors.builder.backend import ConnectorBuilder from timenet_connectors.discovery import known_ids @@ -141,7 +141,7 @@ def build( known = known_ids() if dataset_id not in known: raise typer.BadParameter(f"no connector for {dataset_id!r}; known: {', '.join(known) or '(none)'}") - if version is not None: + if not is_latest(version): declared(dataset_id, version) try: # The parent shows each copy and build step. An isolated child shows its own downloads and shards. diff --git a/packages/timenet-connectors/tests/test_build_backend.py b/packages/timenet-connectors/tests/test_build_backend.py index 8c3baedb7..ab179a74c 100644 --- a/packages/timenet-connectors/tests/test_build_backend.py +++ b/packages/timenet-connectors/tests/test_build_backend.py @@ -16,6 +16,8 @@ from timenet.progress import BytesMoved, ShardWritten from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version +from timenet.sources import FoundInConnector, InstalledConnectors +from timenet.types import Version from timenet.types.references import DatasetRef from timenet.values_backends import ValuesBackend from timenet.writer.progress import ProgressStage, WriteProgressEvent @@ -99,6 +101,11 @@ def test_a_card_that_cannot_be_read_is_skipped_with_a_warning(tmp_path, monkeypa found = installed_builder("timenet/hello-world") assert found is not None assert found.card.dataset_id == "timenet/hello-world" + # The installed connectors still answer for the other datasets. + with pytest.warns(ConnectorCardWarning, match=expected): + answer = InstalledConnectors().resolve("timenet/hello-world", None) + assert isinstance(answer, FoundInConnector) + assert answer.version == Version(1, 0, 0) def test_build_in_process_round_trips_the_connector_default_zarr_backend(tmp_path, monkeypatch): diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index a58b69b67..1fab16f92 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -13,7 +13,7 @@ from timenet.registry.dependencies import prepare_local_copy from timenet.registry.local import LocalRegistry from timenet.registry.shared import complete_copy -from timenet.types.references import DatasetRef, split_ref +from timenet.types.references import DatasetRef, is_latest, split_ref def build_release( # ruff: ignore[too-many-arguments] @@ -59,7 +59,8 @@ def build_release( # ruff: ignore[too-many-arguments] """ # ruff: ignore[docstring-extraneous-exception] - raised by the plan, the copy and the backends options = options or BuildOptions() dataset_id, version = split_ref(dataset) - found = declared(dataset_id, version) + # A connector builds only the version of its card, so "latest" asks for that version. + found = declared(dataset_id, None if is_latest(version) else version) card, requested = found.card, found.release steps = plan_parents( dataset_id, diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index afbea2b7e..50a265418 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -21,7 +21,9 @@ from timenet.registry.dependencies import prepare_local_copy, validate_manifest_reference from timenet.registry.shared import complete_copy from timenet.registry.version import DatasetVersion -from timenet.types import DatasetMetadata, Domain, License, Task +from timenet.sources import FoundInRegistry, NotFound, NotServed, ReleaseLookup +from timenet.types import Access, DatasetMetadata, Domain, License, Task, Version +from timenet.types.access import access_requirements, effective_access from timenet.types.references import DatasetRef @@ -127,6 +129,29 @@ def open_version(self, dataset_id: str, version: str | None = None) -> DatasetVe TimeNetDatasetNotFoundError: If the dataset id or version is unknown. """ + def resolve(self, dataset_id: str, version: Version | None) -> ReleaseLookup: + """Check whether the registry serves a dataset release. + + If :attr:`requires_open_access` is true, the registry serves a release only if its card and each card in its + lineage declare open access. + + Args: + dataset_id: The dataset ID. + version: An exact version, or ``None`` for the latest version. + + Returns: + ``FoundInRegistry`` if served, ``NotServed`` if access is restricted, or ``NotFound`` if absent. + """ + try: + manifest = self.get_manifest(dataset_id, None if version is None else str(version)) + except TimeNetDatasetNotFoundError: + return NotFound() + held = manifest.metadata.dataset_version + access = effective_access(manifest.metadata, manifest.lineage) + if access is Access.OPEN or not self.requires_open_access: + return FoundInRegistry(held, self, manifest) + return NotServed(held, access, access_requirements(manifest.metadata, manifest.lineage), manifest) + def download_version( # ruff: ignore[too-many-arguments] self, dataset_id: str, diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index 0376d8ab0..81a1b15e5 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -9,13 +9,14 @@ from timenet._fs import staged_directory from timenet.dataset import TimeFDataset -from timenet.errors import TimeFFormatError, TimeNetDatasetNotFoundError, TimeNetFileError +from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError from timenet.format.constants import MANIFEST_FILE from timenet.manifest import Manifest, PayloadSource, read_manifest from timenet.registry._paths import release_dir from timenet.registry.dependencies import check_file_sizes, open_local_release from timenet.registry.version import DatasetVersion from timenet.registry.writable import WritableRegistry +from timenet.sources import FoundInRegistry, NotFound, ReleaseLookup from timenet.types import DatasetMetadata, Version from timenet.types.metadata_fields import validate_dataset_id from timenet.types.references import DatasetRef, is_latest @@ -123,6 +124,33 @@ def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest: self._manifest_cache[dataset_id, resolved] = (mtime, manifest) return manifest + def resolve(self, dataset_id: str, version: Version | None) -> ReleaseLookup: + """Check whether the local registry holds a dataset release. + + Local releases are available at any access level. + + Args: + dataset_id: The dataset ID. + version: An exact version, or ``None`` for the latest version. + + Returns: + ``FoundInRegistry`` with the committed version, or ``NotFound`` if the release is absent. + + Raises: + TimeFValidationError: If ``dataset_id`` is invalid. + """ + try: + validate_dataset_id(dataset_id) + except ValueError as error: + raise TimeFValidationError(f"invalid dataset ID {dataset_id!r}: {error}") from error + resolved = self._latest_version(dataset_id) if version is None else str(version) + if resolved is None: + return NotFound() + reference = DatasetRef.parse(f"{dataset_id}@{resolved}") + if not (release_dir(self._root, reference) / MANIFEST_FILE).exists(): + return NotFound() + return FoundInRegistry(reference.version, self) + def open_file(self, dataset_id: str, version: str, relpath: str) -> BinaryIO: """Open one file of a dataset version for binary reading. diff --git a/packages/timenet/src/timenet/registry/remote/_http.py b/packages/timenet/src/timenet/registry/remote/_http.py index 4dd7ea6ad..87c9029c6 100644 --- a/packages/timenet/src/timenet/registry/remote/_http.py +++ b/packages/timenet/src/timenet/registry/remote/_http.py @@ -2,13 +2,14 @@ Owns how the SDK talks to the hosted registry. It prefixes ``/api/v1`` and attaches the bearer token and a ``timenet/`` User-Agent to API calls. It resolves presigned download URLs without -leaking the token to object storage. It maps HTTP status codes onto TimeNet errors. A sync -:class:`httpx.Client` serves the control plane. :meth:`new_async_client` hands out an +leaking the token to object storage. It maps HTTP status codes and network failures onto TimeNet +errors. A sync :class:`httpx.Client` serves the control plane. :meth:`new_async_client` hands out an :class:`httpx.AsyncClient` sharing the same configuration for the parallel downloader. """ import asyncio -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Generator, Mapping +from contextlib import contextmanager import importlib.metadata import logging import math @@ -124,6 +125,22 @@ async def _send_with_retry_async(send: Callable[[], Awaitable[httpx.Response]]) return await send() +@contextmanager +def _reaching(where: str) -> Generator[None]: + """Name ``where`` in the error of a request that fails on the network. + + Args: + where: The registry, or its storage, that the block sends requests to. + + Raises: + TimeNetRegistryError: If a request in the block cannot reach ``where``. + """ + try: + yield + except httpx.TransportError as error: + raise TimeNetRegistryError(f"cannot reach {where}: {error}") from error + + def timenet_user_agent() -> str: """Return the ``timenet/`` User-Agent used for backend tracing. @@ -261,7 +278,8 @@ def get_text(self, path: str) -> str: Returns: The response text. """ - response = _send_with_retry(lambda: self._client.get(f"{API_PREFIX}{path}", headers=self.api_headers())) + with _reaching(self._base_url): + response = _send_with_retry(lambda: self._client.get(f"{API_PREFIX}{path}", headers=self.api_headers())) _raise_for_status(response) return response.text @@ -276,9 +294,10 @@ def post(self, path: str, *, content: bytes | None = None, json: Any = None) -> Returns: The raw response (already status-checked). """ - response = _send_with_retry( - lambda: self._client.post(f"{API_PREFIX}{path}", content=content, json=json, headers=self.api_headers()) - ) + with _reaching(self._base_url): + response = _send_with_retry( + lambda: self._client.post(f"{API_PREFIX}{path}", content=content, json=json, headers=self.api_headers()) + ) _raise_for_status(response) return response @@ -297,7 +316,10 @@ def resolve_presigned(self, dataset_id: str, version: str, relpath: str) -> str: TimeNetRegistryError: If the download endpoint does not redirect. """ # ruff: ignore[docstring-extraneous-exception] - raised by _redirect_target path = _download_path(dataset_id, version, relpath) - response = _send_with_retry(lambda: self._client.get(path, headers=self.api_headers(), follow_redirects=False)) + with _reaching(self._base_url): + response = _send_with_retry( + lambda: self._client.get(path, headers=self.api_headers(), follow_redirects=False) + ) return _redirect_target(response, path) def stream_to(self, url: str, sink: BinaryIO) -> None: @@ -307,7 +329,7 @@ def stream_to(self, url: str, sink: BinaryIO) -> None: url: The presigned URL. sink: A writable binary file object. """ - with self._client.stream("GET", url) as response: + with _reaching(f"the storage of {self._base_url}"), self._client.stream("GET", url) as response: if response.is_error: response.read() # a streaming response has no body until read; _raise_for_status needs it _raise_for_status(response) @@ -322,7 +344,8 @@ def put(self, url: str, *, content: bytes, headers: Mapping[str, str]) -> None: content: The request body. headers: Headers the presign requires (e.g. the checksum binding). """ - response = self._client.put(url, content=content, headers=dict(headers)) + with _reaching(f"the storage of {self._base_url}"): + response = self._client.put(url, content=content, headers=dict(headers)) _raise_for_status(response) def new_async_client(self) -> httpx.AsyncClient: diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index 258363079..d33cece8f 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -18,7 +18,7 @@ from timenet.manifest import Manifest, parse_manifest from timenet.registry.lazy_read import LazyReadRegistry from timenet.registry.remote._download import ProgressCallback, download_version_files -from timenet.registry.remote._http import RegistryHttpClient +from timenet.registry.remote._http import RegistryHttpClient, _reaching from timenet.registry.remote.filesystem import RegistryFileSystem from timenet.registry.remote.models import ( DatasetDetailModel, @@ -151,9 +151,10 @@ def download_release( dataset_id, version = reference.dataset_id, str(reference.version) if manifest is None: manifest = self.get_manifest(dataset_id, version) - download_version_files( - self._http, manifest, dataset_id, version, Path(directory), force=force, progress_cb=progress_cb - ) + with _reaching(self._base_url): + download_version_files( + self._http, manifest, dataset_id, version, Path(directory), force=force, progress_cb=progress_cb + ) def put_release(self, from_directory: Path) -> DatasetRef: """Publish a release directory to the remote registry. diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 4a7d894e5..4ec31d639 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -11,6 +11,8 @@ boto3 is optional. Install the ``timenet[s3]`` extra to use this backend. """ +from collections.abc import Generator +from contextlib import contextmanager import os from pathlib import Path from typing import Any, BinaryIO @@ -132,7 +134,8 @@ def open_file(self, dataset_id: str, version: str, relpath: str) -> BinaryIO: client = self._client() try: - response = client.get_object(Bucket=self._bucket, Key=self._key(dataset_id, version, relpath)) + with self._reaching(): + response = client.get_object(Bucket=self._bucket, Key=self._key(dataset_id, version, relpath)) except botocore.exceptions.ClientError as exc: raise TimeNetDatasetNotFoundError(f"no object {relpath!r} for {dataset_id!r} version {version!r}") from exc return response["Body"] @@ -167,16 +170,17 @@ def put_release(self, from_directory: Path) -> DatasetRef: client = self._client() final_base = self._key(manifest.dataset_id, str(manifest.metadata.dataset_version)) temp_base = f"{final_base}.tmp-{uuid.uuid4().hex}" - try: - self._delete_prefix(client, f"{final_base}.tmp-") # sweep a prior crashed publish's temp prefix - data_relpaths = sorted({part.path for part in manifest.files.all_files()}) - for relpath in (*data_relpaths, MANIFEST_FILE): - client.upload_file(str(from_directory / relpath), self._bucket, f"{temp_base}/{relpath}") - for relpath in data_relpaths: - self._move(client, f"{temp_base}/{relpath}", f"{final_base}/{relpath}") - self._move(client, f"{temp_base}/{MANIFEST_FILE}", f"{final_base}/{MANIFEST_FILE}") - finally: - self._delete_prefix(client, f"{temp_base}/") # remove any objects left staged in the temp prefix + with self._reaching(): + try: + self._delete_prefix(client, f"{final_base}.tmp-") # sweep a prior crashed publish's temp prefix + data_relpaths = sorted({part.path for part in manifest.files.all_files()}) + for relpath in (*data_relpaths, MANIFEST_FILE): + client.upload_file(str(from_directory / relpath), self._bucket, f"{temp_base}/{relpath}") + for relpath in data_relpaths: + self._move(client, f"{temp_base}/{relpath}", f"{final_base}/{relpath}") + self._move(client, f"{temp_base}/{MANIFEST_FILE}", f"{final_base}/{MANIFEST_FILE}") + finally: + self._delete_prefix(client, f"{temp_base}/") # remove any objects left staged in the temp prefix return DatasetRef(dataset_id=manifest.dataset_id, version=manifest.metadata.dataset_version) @staticmethod @@ -218,6 +222,20 @@ def _client() -> Any: return boto3.client("s3") + @contextmanager + def _reaching(self) -> Generator[None]: + """Name the bucket in the error of a request that fails on the network. + + Raises: + TimeNetRegistryError: If a request fails to reach the bucket. + """ + import botocore.exceptions # ruff: ignore[import-outside-top-level] - optional S3 dependency + + try: + yield + except (botocore.exceptions.ConnectionError, botocore.exceptions.HTTPClientError) as error: + raise TimeNetRegistryError(f"cannot reach {self._display_root()}: {error}") from error + def _key(self, *parts: str) -> str: # ruff: ignore[docstring-missing-returns] """Build a bucket-relative key: the prefix, then ``datasets/``, then the path parts.""" @@ -251,7 +269,8 @@ def _get_bytes(self, key: str) -> bytes | None: client = self._client() try: - response = client.get_object(Bucket=self._bucket, Key=key) + with self._reaching(): + response = client.get_object(Bucket=self._bucket, Key=key) except botocore.exceptions.ClientError as exc: if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: return None # a genuinely absent object @@ -263,12 +282,13 @@ def _latest_version(self, dataset_id: str) -> str | None: prefix = f"{self._key(dataset_id)}/" client = self._client() versions: list[str] = [] - paginator = client.get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix, Delimiter="/"): - for entry in page.get("CommonPrefixes", []): - name = entry["Prefix"][len(prefix) :].strip("/") - if _is_version(name): - versions.append(name) + with self._reaching(): + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix, Delimiter="/"): + for entry in page.get("CommonPrefixes", []): + name = entry["Prefix"][len(prefix) :].strip("/") + if _is_version(name): + versions.append(name) return max(versions, key=Version.parse) if versions else None diff --git a/packages/timenet/src/timenet/sources.py b/packages/timenet/src/timenet/sources.py new file mode 100644 index 000000000..a72aaaea0 --- /dev/null +++ b/packages/timenet/src/timenet/sources.py @@ -0,0 +1,193 @@ +"""Find dataset releases across registries and installed connectors.""" + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypeAlias, Union + +from timenet import builders +from timenet.errors import TimeNetAccessError +from timenet.types.access import Access, AccessRequirement +from timenet.types.references import LATEST_VERSION, DatasetRef +from timenet.types.version import Version + + +if TYPE_CHECKING: + from timenet.manifest import Manifest + from timenet.registry.base import BaseRegistry + + +# The registry module imports this module, so the alias names the registry as a forward reference. +ReleaseSource: TypeAlias = Union["BaseRegistry", "InstalledConnectors"] +"""A source that can serve or build a dataset release: a registry or the installed connectors.""" + + +@dataclass(frozen=True) +class FoundInRegistry: + """A result for a release that a registry serves. + + Attributes: + version: The exact version. + registry: The registry that serves the release. + manifest: The release manifest that the registry read, or ``None``. + """ + + version: Version + registry: "BaseRegistry" + manifest: "Manifest | None" = field(default=None, compare=False, repr=False) + + +@dataclass(frozen=True) +class FoundInConnector: + """A result for a release that an installed connector builds. + + Attributes: + builder: The backend and card that the lookup read. + """ + + builder: builders.InstalledBuilder + + @property + def version(self) -> Version: + """The version that the card declares.""" + return self.builder.card.dataset_version + + +Found: TypeAlias = FoundInRegistry | FoundInConnector +"""A result for a release that a source serves or builds.""" + + +@dataclass(frozen=True) +class NotFound: + """A result for a release that the source cannot find.""" + + +@dataclass(frozen=True) +class NotServed: + """A result for a listed release whose files the source cannot serve. + + Attributes: + version: The exact version that the source lists. + access: The strictest access level of the release's card and the cards in its lineage. + requirements: Each card that is not open access, with its access level and access page. + manifest: The release manifest, or ``None``. + """ + + version: Version + access: Access + requirements: tuple[AccessRequirement, ...] + manifest: "Manifest | None" = field(default=None, compare=False, repr=False) + + def access_error(self, dataset_id: str) -> TimeNetAccessError: + """Create an error with the release's access requirements. + + Args: + dataset_id: The dataset ID. + + Returns: + An error with the release, its access level, and its access pages. + """ + release = DatasetRef(dataset_id=dataset_id, version=self.version) + return TimeNetAccessError(release, self.access, self.requirements) + + +ReleaseLookup: TypeAlias = Found | NotFound | NotServed +"""The result of looking up a release in a source.""" + + +@dataclass(frozen=True) +class InstalledConnectors: + """A release source backed by installed dataset connectors. + + Each connector provides only the version in its card. + All instances compare equal. + """ + + def __str__(self) -> str: + if not tuple(builders.entry_points(group=builders.ENTRY_POINT_GROUP)): + return "installed connectors (no connector package is installed)" + return "installed connectors" + + def resolve(self, dataset_id: str, version: Version | None) -> ReleaseLookup: # ruff: ignore[no-self-use] (every source has it) + """Check whether an installed connector can build a release. + + The lookup uses the first entry point that claims the dataset. + + Args: + dataset_id: The dataset ID. + version: An exact version, or ``None`` for the latest version. + + Returns: + ``FoundInConnector`` if a connector builds the requested version, or ``NotFound``. + + Raises: + TimeNetInvalidCardError: If the connector's card cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend + builder = builders.installed_builder(dataset_id) + if builder is None or (version is not None and version != builder.card.dataset_version): + return NotFound() + return FoundInConnector(builder) + + +def resolve_release(sources: Sequence[ReleaseSource], dataset_id: str, version: str | None) -> ReleaseLookup: + """Find a source that provides the requested dataset release. + + An exact version comes from the first source that serves it, and no version means the newest version of the first + source that serves the dataset. ``"latest"`` is the highest version of any source, source order breaks ties, and + source errors stop the search. + + Args: + sources: The sources, in order. + dataset_id: The dataset ID. + version: An exact version, ``"latest"``, or ``None`` or ``""`` for no version. + + Returns: + A ``Found`` result if available, otherwise the first ``NotServed`` result or ``NotFound``. + + Raises: + TimeFValidationError: If ``version`` is neither ``"latest"`` nor a ``major.minor.patch`` version. + """ # ruff: ignore[docstring-extraneous-exception] - raised by Version.parse + if version == LATEST_VERSION: + return _resolve_latest(sources, dataset_id) + return _resolve_first(sources, dataset_id, Version.parse(version) if version else None) + + +def _resolve_first(sources: Sequence[ReleaseSource], dataset_id: str, version: Version | None) -> ReleaseLookup: + """Return the answer of the first source that serves ``version``, or its newest for ``None``, else the first listing.""" + listed: NotServed | None = None + for source in sources: + answer = source.resolve(dataset_id, version) + if isinstance(answer, Found): + return answer + if isinstance(answer, NotServed) and listed is None: + listed = answer + return NotFound() if listed is None else listed + + +def _resolve_latest(sources: Sequence[ReleaseSource], dataset_id: str) -> ReleaseLookup: + """Return the highest version that any source provides, from the first source that provides it.""" + best: Found | None = None + listed: list[tuple[ReleaseSource, NotServed]] = [] + before_best = 0 + for source in sources: + answer = source.resolve(dataset_id, None) + if isinstance(answer, NotServed): + listed.append((source, answer)) + elif isinstance(answer, Found) and (best is None or answer.version > best.version): + best, before_best = answer, len(listed) + if best is None: + return listed[0][1] if listed else NotFound() + return _ask_again(listed[:before_best], dataset_id, best.version) or best + + +def _ask_again(listed: Sequence[tuple[ReleaseSource, NotServed]], dataset_id: str, version: Version) -> Found | None: + """Ask each source that lists a higher version without serving it whether it serves ``version``. + + Returns: + The answer of the first such source that serves ``version``, or ``None``. + """ + for source, answer in listed: + if answer.version > version: + again = source.resolve(dataset_id, version) + if isinstance(again, Found): + return again + return None diff --git a/packages/timenet/src/timenet/types/references.py b/packages/timenet/src/timenet/types/references.py index fd68c447c..c79c71d1a 100644 --- a/packages/timenet/src/timenet/types/references.py +++ b/packages/timenet/src/timenet/types/references.py @@ -9,7 +9,7 @@ LATEST_VERSION = "latest" -"""Reference identifier that selects the newest committed dataset version.""" +"""Reference identifier that selects the newest committed dataset version across all sources.""" def is_latest(version: str | None) -> bool: @@ -24,15 +24,16 @@ def is_latest(version: str | None) -> bool: def split_ref(ref: str) -> tuple[str, str | None]: - """Split a dataset reference into its id and pinned version. + """Split a dataset reference into its id and requested version. - ``"org/id@1.0.0"`` -> ``("org/id", "1.0.0")``. ``"org/id@latest"`` or ``"org/id"`` -> ``("org/id", None)``. + ``"org/id@1.0.0"`` -> ``("org/id", "1.0.0")``. ``"org/id@latest"`` -> ``("org/id", "latest")``. ``"org/id"`` -> + ``("org/id", None)``. Args: ref: A dataset id, optionally suffixed with ``@`` or ``@latest``. Returns: - The bare dataset id and the pinned version string, or ``None`` for latest. + The bare dataset id and the version string, ``"latest"``, or ``None`` without a version. Raises: TimeFValidationError: If ``ref`` has more than one ``@``, an empty id or version, or a version @@ -43,11 +44,12 @@ def split_ref(ref: str) -> tuple[str, str | None]: dataset_id, separator, version = ref.partition("@") if not dataset_id: raise TimeFValidationError(f"dataset ref has an empty id: {ref!r}") - if not separator or version == LATEST_VERSION: + if not separator: return dataset_id, None if not version: raise TimeFValidationError(f"dataset ref has an empty version: {ref!r}") - Version.parse(version) # Validate the pin. This raises TimeFValidationError on a malformed version. + if version != LATEST_VERSION: + Version.parse(version) # Validate the pin. This raises TimeFValidationError on a malformed version. return dataset_id, version @@ -71,7 +73,7 @@ def parse(cls, value: str) -> "DatasetRef": TimeFValidationError: If the reference is malformed or does not contain an exact version. """ dataset_id, version = split_ref(value) - if version is None: + if is_latest(version): raise TimeFValidationError(f"an exact dataset release is required: {value!r}") try: return cls.model_validate({"dataset_id": dataset_id, "version": version}) diff --git a/packages/timenet/tests/registry/test_conformance.py b/packages/timenet/tests/registry/test_conformance.py index f365f56ca..e22356e5d 100644 --- a/packages/timenet/tests/registry/test_conformance.py +++ b/packages/timenet/tests/registry/test_conformance.py @@ -16,6 +16,7 @@ from timenet.dataset.axis import RegularAxis from timenet.errors import TimeNetDatasetNotFoundError, TimeNetFileError from timenet.registry import LocalRegistry, RemoteRegistry, S3Registry, WritableRegistry +from timenet.sources import FoundInRegistry, NotFound from timenet.testing import make_dataset, sine_loader from timenet.types import ( Annotation, @@ -154,6 +155,18 @@ def test_get_manifest_unknown_version_raises(backend): backend.registry.get_manifest("timenet/hello-world", "9.9.9") +# ---- resolve ---------------------------------------------------------------------------------- + + +def test_a_registry_resolves_a_held_release_and_not_an_unknown_one(backend): + registry = backend.registry + assert registry.resolve("timenet/hello-world", None) == FoundInRegistry(Version(1, 1, 0), registry) + assert registry.resolve("timenet/hello-world", Version(1, 0, 0)) == FoundInRegistry(Version(1, 0, 0), registry) + assert registry.resolve("no/such", None) == NotFound() + assert registry.resolve("no/such", Version(1, 0, 0)) == NotFound() + assert registry.resolve("timenet/hello-world", Version(9, 9, 9)) == NotFound() + + # ---- open_file -------------------------------------------------------------------------------- diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 8ae6242c4..c803ce8c2 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -23,8 +23,9 @@ from timenet.manifest import Manifest from timenet.reader import LoadPlan, TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version +from timenet.sources import FoundInRegistry, NotServed from timenet.testing import assert_datasets_equal, make_dataset -from timenet.types import Access, DatasetSchema, Domain, License +from timenet.types import Access, DatasetSchema, Domain, License, Version from timenet.types.access import AccessRequirement from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.reference_keys import ParentAlias @@ -558,8 +559,8 @@ def handler(request): assert requests == prepared_requests -@pytest.mark.parametrize("page", ["own", "ancestor"]) -def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tmp_path, page): +def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: + """Write a release that is not open into ``origin``; return its ID and the release whose card is credentialed.""" dataset = make_dataset() gated = {"access": Access.CREDENTIALED, "access_url": "https://example.org/dua"} release = DatasetRef(dataset_id=dataset.metadata.dataset_id, version=dataset.metadata.dataset_version) @@ -571,14 +572,80 @@ def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tm dataset._import_state.lineage["test/ancestor@1.0.0"] = ParentDatasetCard(metadata=ancestor) release = DatasetRef.parse("test/ancestor@1.0.0") dataset.derive_schema() - with TimeFWriter(tmp_path / "origin", dataset) as writer: + with TimeFWriter(origin, dataset) as writer: writer.write() + return dataset.metadata.dataset_id, release + + +@pytest.mark.parametrize("page", ["own", "ancestor"]) +def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tmp_path, page): + dataset_id, release = _gated_release(tmp_path / "origin", page) transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") with pytest.raises(TimeNetAccessError) as caught: - TimeNet(tmp_path / "local").open(dataset.metadata.dataset_id, auto_build=False, fetch_from=remote) + TimeNet(tmp_path / "local").open(dataset_id, auto_build=False, fetch_from=remote) assert caught.value.requirements == (AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua"),) assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) assert not (tmp_path / "local").exists() + + +@pytest.mark.parametrize("page", ["own", "ancestor"]) +def test_a_remote_registry_lists_but_does_not_serve_a_release_that_is_not_open(tmp_path, page): + dataset_id, release = _gated_release(tmp_path / "origin", page) + transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) + remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + requirement = AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua") + + for version in (None, Version(1, 0, 0)): + assert remote.resolve(dataset_id, version) == NotServed(Version(1, 0, 0), Access.CREDENTIALED, (requirement,)) + assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) + # A local registry serves every release that it holds. + local = LocalRegistry(tmp_path / "origin") + assert local.resolve(dataset_id, None) == FoundInRegistry(Version(1, 0, 0), local) + + +@pytest.mark.parametrize("version", [None, Version(1, 0, 0)]) +def test_resolve_keeps_the_manifest_that_it_fetched(version_dir, tmp_path, version): + registry, requests = _remote(version_dir, tmp_path) + _, manifest = version_dir + dataset_id = manifest.metadata.dataset_id + + answer = registry.resolve(dataset_id, version) + + assert isinstance(answer, FoundInRegistry) + assert answer.manifest == manifest + assert sum(request.url.path.endswith("/manifest") for request in requests) == 1 + + +def test_resolve_keeps_the_manifest_of_a_release_that_is_not_served(tmp_path): + dataset_id, _ = _gated_release(tmp_path / "origin", "own") + transport, _ = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) + remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + + answer = remote.resolve(dataset_id, None) + + assert isinstance(answer, NotServed) + assert answer.manifest is not None + assert answer.manifest.metadata.dataset_id == dataset_id + + +def _offline(request): + raise httpx.ConnectError("the network is down", request=request) + + +def test_an_unreachable_registry_raises_a_registry_error_that_names_it(tmp_path): + remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(_offline), cache_dir=tmp_path) + + with pytest.raises(TimeNetRegistryError, match=r"^cannot reach http://api\.local: the network is down$"): + remote.resolve("timenet/hello-world", None) + + +def test_a_network_drop_during_the_parallel_download_raises_a_registry_error(version_dir, tmp_path): + _, manifest = version_dir + remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(_offline)) + release = DatasetRef(dataset_id=manifest.metadata.dataset_id, version=manifest.metadata.dataset_version) + + with pytest.raises(TimeNetRegistryError, match=r"^cannot reach http://api\.local: the network is down$"): + remote.download_release(release, tmp_path / "copy", manifest=manifest) diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index e7857904b..62d5f95f5 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -4,6 +4,7 @@ boto3 = pytest.importorskip("boto3") pytest.importorskip("moto") +import botocore.exceptions # ruff: ignore[module-import-not-at-top-of-file] from moto.server import ThreadedMotoServer # ruff: ignore[module-import-not-at-top-of-file] import pyarrow.fs as pafs # ruff: ignore[module-import-not-at-top-of-file] @@ -16,7 +17,7 @@ from timenet.reader import TimeFReader # ruff: ignore[module-import-not-at-top-of-file] from timenet.registry import LocalRegistry, S3Registry # ruff: ignore[module-import-not-at-top-of-file] from timenet.testing import assert_datasets_equal, make_dataset # ruff: ignore[module-import-not-at-top-of-file] -from timenet.types import DatasetSchema # ruff: ignore[module-import-not-at-top-of-file] +from timenet.types import DatasetSchema, Version # ruff: ignore[module-import-not-at-top-of-file] from timenet.types.references import DatasetRef # ruff: ignore[module-import-not-at-top-of-file] @@ -135,3 +136,20 @@ def test_list_and_search_are_unsupported(tmp_path): def test_rejects_a_non_s3_uri(tmp_path): with pytest.raises(TimeNetRegistryError, match="s3://"): S3Registry("https://not-s3", cache_dir=tmp_path) + + +class _Unreachable: + """An S3 client whose every request fails to reach the endpoint.""" + + def __getattr__(self, name): + def request(*args, **kwargs): + raise botocore.exceptions.EndpointConnectionError(endpoint_url="https://s3.example") + + return request + + +def test_an_unreachable_bucket_raises_a_registry_error_that_names_it(monkeypatch): + registry = S3Registry("s3://tn-test/registry") + monkeypatch.setattr(registry, "_client", _Unreachable) + with pytest.raises(TimeNetRegistryError, match=r"^cannot reach s3://tn-test/registry: Could not connect"): + registry.resolve("timenet/hello-world", Version(1, 0, 0)) diff --git a/packages/timenet/tests/test_refs.py b/packages/timenet/tests/test_refs.py index 01186ea60..8a144cd50 100644 --- a/packages/timenet/tests/test_refs.py +++ b/packages/timenet/tests/test_refs.py @@ -9,7 +9,7 @@ [ ("timenet/hello-world", ("timenet/hello-world", None)), ("org/name", ("org/name", None)), - ("org/name@latest", ("org/name", None)), + ("org/name@latest", ("org/name", "latest")), ("org/name@1.2.3", ("org/name", "1.2.3")), ("timenet/hello-world@0.0.1", ("timenet/hello-world", "0.0.1")), ], diff --git a/packages/timenet/tests/test_sources.py b/packages/timenet/tests/test_sources.py new file mode 100644 index 000000000..b274b4315 --- /dev/null +++ b/packages/timenet/tests/test_sources.py @@ -0,0 +1,158 @@ +from pathlib import Path + +import pytest + +from timenet import builders +from timenet.errors import ( + TimeFValidationError, + TimeNetRegistryError, +) +from timenet.registry import LocalRegistry +from timenet.sources import ( + FoundInConnector, + FoundInRegistry, + InstalledConnectors, + NotFound, + NotServed, + resolve_release, +) +from timenet.testing import install_backends, make_dataset +from timenet.types import Access, Version + + +V1, V11, V2 = Version(1, 0, 0), Version(1, 1, 0), Version(2, 0, 0) + + +class _Source(LocalRegistry): + """A registry that serves the versions in ``served`` and lists the versions in ``listed`` without serving them.""" + + def __init__(self, *served: str, listed: tuple[str, ...] = ()): + super().__init__(Path("unused")) + self.served, self.listed = set(served), set(listed) + + def resolve(self, dataset_id, version): + known = sorted(self.served | self.listed, key=Version.parse) + wanted = (known[-1] if known else None) if version is None else str(version) + if wanted in self.served: + return FoundInRegistry(Version.parse(wanted), self) + if wanted in self.listed: + return NotServed(Version.parse(wanted), Access.CREDENTIALED, ()) + return NotFound() + + +class _Broken(_Source): + def resolve(self, dataset_id, version): + raise TimeNetRegistryError("the registry does not answer") + + +class _Backend: + def __init__(self, card): + self._card = card + + def dataset_ids(self): + return [self._card.dataset_id] + + def card(self, dataset_id): + return self._card if dataset_id == self._card.dataset_id else None + + +def _fail(*args, **kwargs): + raise AssertionError("resolve() must not read every manifest") + + +def test_an_exact_version_comes_from_the_first_source_that_provides_it(): + second = _Source("1.0.0") + assert resolve_release([_Source(), second, _Source("1.0.0")], "a/b", "1.0.0") == FoundInRegistry(V1, second) + + +def test_without_a_version_the_first_source_that_serves_the_dataset_gives_its_newest_version(): + first = _Source("1.0.0") + sources = [_Source(), first, _Source("1.1.0")] + assert resolve_release(sources, "a/b", None) == FoundInRegistry(V1, first) + assert resolve_release(sources, "a/b", "") == FoundInRegistry(V1, first) + + +def test_without_a_version_a_source_that_does_not_serve_the_dataset_falls_through(): + second = _Source("1.0.0") + listed = _Source(listed=("2.0.0",)) + assert resolve_release([listed, second, _Source("1.1.0")], "a/b", None) == FoundInRegistry(V1, second) + # With no source that serves it, the access error of the first listing comes last. + assert resolve_release([listed, _Source()], "a/b", None) == NotServed(V2, Access.CREDENTIALED, ()) + + +def test_latest_is_the_highest_version_across_all_sources(): + remote = _Source("1.0.0", "1.1.0") + assert resolve_release([_Source("1.0.0"), remote], "a/b", "latest") == FoundInRegistry(V11, remote) + + +def test_latest_comes_from_the_first_source_that_holds_the_highest_version(): + store = _Source("1.1.0") + assert resolve_release([store, _Source("1.1.0")], "a/b", "latest") == FoundInRegistry(V11, store) + + +def test_a_release_that_is_not_served_falls_through_and_does_not_count_for_latest(): + connectors = _Source("1.0.0") + assert resolve_release([_Source(listed=("2.0.0",)), connectors], "a/b", "latest") == FoundInRegistry(V1, connectors) + assert resolve_release([_Source(listed=("2.0.0",))], "a/b", "2.0.0") == NotServed(V2, Access.CREDENTIALED, ()) + + +def test_latest_asks_a_source_that_did_not_serve_a_higher_version_for_the_chosen_one(): + hosted = _Source("1.0.0", listed=("2.0.0",)) + assert resolve_release([hosted, _Source("1.0.0")], "a/b", "latest") == FoundInRegistry(V1, hosted) + + +def test_no_source_gives_not_found(): + assert resolve_release([_Source(), _Source()], "a/b", None) == NotFound() + assert resolve_release([_Source("1.0.0")], "a/b", "2.0.0") == NotFound() + + +def test_an_error_of_a_source_stops_the_resolution(): + with pytest.raises(TimeNetRegistryError): + resolve_release([_Broken(), _Source("1.0.0")], "a/b", "1.0.0") + with pytest.raises(TimeNetRegistryError): + resolve_release([_Source("1.0.0"), _Broken()], "a/b", "latest") + + +def test_installed_connectors_answer_with_the_builder_of_the_declared_version(monkeypatch): + card = make_dataset().metadata + backend = _Backend(card) + install_backends(monkeypatch, backend) + latest, exact = ( + InstalledConnectors().resolve(card.dataset_id, None), + InstalledConnectors().resolve(card.dataset_id, V1), + ) + assert isinstance(latest, FoundInConnector) and isinstance(exact, FoundInConnector) + assert (latest.builder.backend, latest.version) == (exact.builder.backend, exact.version) == (backend, V1) + + +def test_installed_connectors_answer_not_found_for_another_version(monkeypatch): + card = make_dataset().metadata + install_backends(monkeypatch, _Backend(card)) + assert InstalledConnectors().resolve(card.dataset_id, V2) == NotFound() + assert InstalledConnectors().resolve("x/y", None) == NotFound() + + +def test_installed_connectors_without_a_package_answer_not_found(monkeypatch): + monkeypatch.setattr(builders, "entry_points", lambda group: []) + assert InstalledConnectors().resolve("timenet/hello-world", None) == NotFound() + assert "no connector package is installed" in str(InstalledConnectors()) + + +def test_installed_connectors_are_equal(): + assert InstalledConnectors() == InstalledConnectors() + assert len({InstalledConnectors(), InstalledConnectors()}) == 1 + + +def test_a_local_registry_answers_not_found_without_reading_every_manifest(tmp_path, monkeypatch): + registry = LocalRegistry(tmp_path) + registry.put_dataset(make_dataset()) + monkeypatch.setattr(LocalRegistry, "list_datasets", _fail) + assert registry.resolve("test/unknown", None) == NotFound() + assert registry.resolve("test/unknown", V1) == NotFound() + assert registry.resolve("timenet/hello-world", V2) == NotFound() + + +@pytest.mark.parametrize("version", [None, V1]) +def test_a_local_registry_rejects_an_invalid_dataset_id(tmp_path, version): + with pytest.raises(TimeFValidationError): + LocalRegistry(tmp_path).resolve("../evil", version) diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.py index 127eb1cad..ee4ffa704 100644 --- a/scripts/gen_api_docs.py +++ b/scripts/gen_api_docs.py @@ -19,7 +19,8 @@ from __future__ import annotations import ast -from dataclasses import dataclass +from collections import Counter +from dataclasses import dataclass, replace from pathlib import Path import re @@ -62,7 +63,8 @@ class ApiModule: Attributes: dotted: Importable dotted path, e.g. ``timenet.registry``. - slug: Output filename stem under ``docs/api/`` (the last path component). + slug: Output filename stem under ``docs/api/``: the last path component, or the + dotted path if a module of the other package has the same last component. summary: Cleaned first line of the module docstring, or ``""``. show_submodules: Render submodules recursively (packages without ``__all__``). """ @@ -264,6 +266,15 @@ def main() -> None: """ OUTPUT_DIR.mkdir(parents=True, exist_ok=True) by_package = {pkg: _discover(pkg, root) for pkg, root in PACKAGE_ROOTS.items()} + # Both packages can have a module with the same name, such as timenet.sources and + # timenet_connectors.sources. The page of each such module takes its full dotted path. + shared = { + slug for slug, count in Counter(mod.slug for mods in by_package.values() for mod in mods).items() if count > 1 + } + by_package = { + pkg: [replace(mod, slug=mod.dotted) if mod.slug in shared else mod for mod in mods] + for pkg, mods in by_package.items() + } all_mods = [mod for mods in by_package.values() for mod in mods] # "index" is reserved for the landing page written by _write_index below. diff --git a/zensical.toml b/zensical.toml index a10b4762d..5e0b129e0 100644 --- a/zensical.toml +++ b/zensical.toml @@ -22,7 +22,7 @@ nav = [ { "Datasets" = [{ "Use datasets" = [{ "The client (SDK)" = "client.md" }, { "Registries" = "registry.md" }, { "Load into your stack" = "usage.md" }] }, { "Build datasets" = [{ "Connectors" = "connectors.md" }, { "Build & publish" = "build.md" }] }, { "Catalog" = [{ "Datasets" = "catalog/datasets.md" }, { "Benchmarks" = "catalog/benchmarks.md" }, { "Licensing" = "catalog/licensing.md" }] }] }, { "CLI" = [{ "Overview" = "cli/index.md" }, { "timenet (consumer)" = "cli/timenet.md" }, { "timenet-build (producer)" = "cli/build.md" }] }, # >>> api-nav: generated by scripts/gen_api_docs.py — do not edit >>> - { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, + { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.sources" = "api/timenet.sources.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/timenet_connectors.sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, # <<< api-nav <<< ] From f69282dcb93cc73ef09b72ee911afbe30eeab258 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Wed, 4 Nov 2026 18:57:03 +0100 Subject: [PATCH 2/4] docs: shorten the docstrings and docs of the release sources Say only what a caller of a release source and of resolve_release() needs to know. Part of TNET-157. --- scripts/gen_api_docs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.py index ee4ffa704..25f19a58d 100644 --- a/scripts/gen_api_docs.py +++ b/scripts/gen_api_docs.py @@ -63,8 +63,7 @@ class ApiModule: Attributes: dotted: Importable dotted path, e.g. ``timenet.registry``. - slug: Output filename stem under ``docs/api/``: the last path component, or the - dotted path if a module of the other package has the same last component. + slug: The page name under ``docs/api/``: the last component, or the dotted path on a clash. summary: Cleaned first line of the module docstring, or ``""``. show_submodules: Render submodules recursively (packages without ``__all__``). """ From aa1d2ed4d5b261fc18d2ef0d911b82e3a54546e4 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 00:20:02 +0100 Subject: [PATCH 3/4] docs: shorten the docstrings of the release sources Shorter docstrings for the code that this change adds or changes. Part of TNET-157. --- packages/timenet/src/timenet/registry/base.py | 2 +- packages/timenet/src/timenet/registry/local.py | 2 +- .../timenet/src/timenet/registry/remote/_http.py | 12 ++++++------ scripts/gen_api_docs.py | 16 +++++++++------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index 50a265418..eb9791782 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -35,7 +35,7 @@ class BaseRegistry(ABC): - """A registry of dataset manifests, files, and searchable metadata.""" + """A registry that provides TimeF dataset metadata, manifests, and files.""" def __str__(self) -> str: return type(self).__name__ diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index 81a1b15e5..e0a077263 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -24,7 +24,7 @@ class LocalRegistry(WritableRegistry): - """Read and publish datasets in a local directory.""" + """A registry that reads and writes datasets in a local directory.""" def __init__(self, root: Path) -> None: """Open a local registry rooted at a directory. diff --git a/packages/timenet/src/timenet/registry/remote/_http.py b/packages/timenet/src/timenet/registry/remote/_http.py index 87c9029c6..bfbdbf721 100644 --- a/packages/timenet/src/timenet/registry/remote/_http.py +++ b/packages/timenet/src/timenet/registry/remote/_http.py @@ -130,10 +130,10 @@ def _reaching(where: str) -> Generator[None]: """Name ``where`` in the error of a request that fails on the network. Args: - where: The registry, or its storage, that the block sends requests to. + where: The registry or storage location to name in the error. Raises: - TimeNetRegistryError: If a request in the block cannot reach ``where``. + TimeNetRegistryError: If a request fails to reach ``where``. """ try: yield @@ -209,7 +209,7 @@ def _redirect_target(response: httpx.Response, path: str) -> str: class RegistryHttpClient: - """A thin httpx wrapper for the registry API and its presigned object URLs.""" + """An HTTP client for the registry API and signed file URLs.""" @property def token(self) -> str | None: @@ -270,7 +270,7 @@ def api_headers(self) -> dict[str, str]: return headers def get_text(self, path: str) -> str: - """GET an API path and return the raw text body (used for ``manifest.json``). + """Fetch the text body from a registry API path. Args: path: A path under ``/api/v1``. @@ -288,7 +288,7 @@ def post(self, path: str, *, content: bytes | None = None, json: Any = None) -> Args: path: A path under ``/api/v1``. - content: Raw request body bytes (e.g. the manifest), mutually exclusive with ``json``. + content: The request bytes, mutually exclusive with ``json``. json: A JSON-serializable body. Returns: @@ -302,7 +302,7 @@ def post(self, path: str, *, content: bytes | None = None, json: Any = None) -> return response def resolve_presigned(self, dataset_id: str, version: str, relpath: str) -> str: - """Resolve a file's presigned URL via the download redirect, without following it. + """Get a signed download URL for a dataset file. Args: dataset_id: The ``org/name`` id. diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.py index 25f19a58d..ae2f66a01 100644 --- a/scripts/gen_api_docs.py +++ b/scripts/gen_api_docs.py @@ -59,13 +59,13 @@ @dataclass(frozen=True) class ApiModule: - """One module to document. + """A module's API page settings. Attributes: - dotted: Importable dotted path, e.g. ``timenet.registry``. - slug: The page name under ``docs/api/``: the last component, or the dotted path on a clash. - summary: Cleaned first line of the module docstring, or ``""``. - show_submodules: Render submodules recursively (packages without ``__all__``). + dotted: The import path, such as ``timenet.registry``. + slug: The page name under ``docs/api/``: the last part of the import path, or the full path on a clash. + summary: The cleaned first line of the module docstring, or ``""``. + show_submodules: Whether to include submodules recursively. True for a package without ``__all__``. """ dotted: str @@ -258,10 +258,12 @@ def _sub(match: re.Match[str]) -> str: def main() -> None: - """Generate the API pages and landing page, and rewrite the nav region. + """Generate the API reference pages and landing page, and rewrite the nav region of ``zensical.toml``. + + It also deletes each page in ``docs/api/`` that no current module maps to. Raises: - SystemExit: On a slug collision or a missing nav marker region. + SystemExit: If page names clash or the nav markers are missing or duplicated. """ OUTPUT_DIR.mkdir(parents=True, exist_ok=True) by_package = {pkg: _discover(pkg, root) for pkg, root in PACKAGE_ROOTS.items()} From 3117782c6590016dd60312516dc698182935be81 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 10:17:07 +0100 Subject: [PATCH 4/4] test: trim the tests of the release sources Drop tests that cannot fail or repeat other tests, and sharpen the rest. Part of TNET-157. --- .../timenet/tests/registry/test_remote.py | 32 +++++++------------ packages/timenet/tests/test_sources.py | 10 ++---- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index c803ce8c2..c00426a6a 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -591,46 +591,38 @@ def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tm assert not (tmp_path / "local").exists() -@pytest.mark.parametrize("page", ["own", "ancestor"]) -def test_a_remote_registry_lists_but_does_not_serve_a_release_that_is_not_open(tmp_path, page): - dataset_id, release = _gated_release(tmp_path / "origin", page) +def test_a_remote_registry_lists_but_does_not_serve_a_release_that_is_not_open(tmp_path): + # The card of the release is open. A card in its lineage is not. + dataset_id, release = _gated_release(tmp_path / "origin", "ancestor") transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") requirement = AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua") - for version in (None, Version(1, 0, 0)): - assert remote.resolve(dataset_id, version) == NotServed(Version(1, 0, 0), Access.CREDENTIALED, (requirement,)) + answer = remote.resolve(dataset_id, None) + + assert answer == NotServed(Version(1, 0, 0), Access.CREDENTIALED, (requirement,)) + # The answer keeps the manifest that the lookup fetched. + assert isinstance(answer, NotServed) + assert answer.manifest is not None + assert answer.manifest.metadata.dataset_id == dataset_id assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) # A local registry serves every release that it holds. local = LocalRegistry(tmp_path / "origin") assert local.resolve(dataset_id, None) == FoundInRegistry(Version(1, 0, 0), local) -@pytest.mark.parametrize("version", [None, Version(1, 0, 0)]) -def test_resolve_keeps_the_manifest_that_it_fetched(version_dir, tmp_path, version): +def test_resolve_keeps_the_manifest_that_it_fetched(version_dir, tmp_path): registry, requests = _remote(version_dir, tmp_path) _, manifest = version_dir dataset_id = manifest.metadata.dataset_id - answer = registry.resolve(dataset_id, version) + answer = registry.resolve(dataset_id, None) assert isinstance(answer, FoundInRegistry) assert answer.manifest == manifest assert sum(request.url.path.endswith("/manifest") for request in requests) == 1 -def test_resolve_keeps_the_manifest_of_a_release_that_is_not_served(tmp_path): - dataset_id, _ = _gated_release(tmp_path / "origin", "own") - transport, _ = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) - remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") - - answer = remote.resolve(dataset_id, None) - - assert isinstance(answer, NotServed) - assert answer.manifest is not None - assert answer.manifest.metadata.dataset_id == dataset_id - - def _offline(request): raise httpx.ConnectError("the network is down", request=request) diff --git a/packages/timenet/tests/test_sources.py b/packages/timenet/tests/test_sources.py index b274b4315..904d3863a 100644 --- a/packages/timenet/tests/test_sources.py +++ b/packages/timenet/tests/test_sources.py @@ -138,11 +138,6 @@ def test_installed_connectors_without_a_package_answer_not_found(monkeypatch): assert "no connector package is installed" in str(InstalledConnectors()) -def test_installed_connectors_are_equal(): - assert InstalledConnectors() == InstalledConnectors() - assert len({InstalledConnectors(), InstalledConnectors()}) == 1 - - def test_a_local_registry_answers_not_found_without_reading_every_manifest(tmp_path, monkeypatch): registry = LocalRegistry(tmp_path) registry.put_dataset(make_dataset()) @@ -152,7 +147,6 @@ def test_a_local_registry_answers_not_found_without_reading_every_manifest(tmp_p assert registry.resolve("timenet/hello-world", V2) == NotFound() -@pytest.mark.parametrize("version", [None, V1]) -def test_a_local_registry_rejects_an_invalid_dataset_id(tmp_path, version): +def test_a_local_registry_rejects_an_invalid_dataset_id(tmp_path): with pytest.raises(TimeFValidationError): - LocalRegistry(tmp_path).resolve("../evil", version) + LocalRegistry(tmp_path).resolve("../evil", None)