diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index 9207ee93b..7b43b2d7f 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -33,7 +33,7 @@ open_writable_registry, ) from timenet.registry._paths import release_dir # ruff: ignore[import-private-name] -from timenet.types.references import DatasetRef, split_ref +from timenet.types.references import DatasetRef, is_latest, split_ref from timenet_connectors.builder.backend import ConnectorBuilder from timenet_connectors.discovery import known_ids @@ -141,7 +141,7 @@ def build( known = known_ids() if dataset_id not in known: raise typer.BadParameter(f"no connector for {dataset_id!r}; known: {', '.join(known) or '(none)'}") - if version is not None: + if not is_latest(version): declared(dataset_id, version) try: # The parent shows each copy and build step. An isolated child shows its own downloads and shards. diff --git a/packages/timenet-connectors/tests/test_build_backend.py b/packages/timenet-connectors/tests/test_build_backend.py index 8c3baedb7..ab179a74c 100644 --- a/packages/timenet-connectors/tests/test_build_backend.py +++ b/packages/timenet-connectors/tests/test_build_backend.py @@ -16,6 +16,8 @@ from timenet.progress import BytesMoved, ShardWritten from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version +from timenet.sources import FoundInConnector, InstalledConnectors +from timenet.types import Version from timenet.types.references import DatasetRef from timenet.values_backends import ValuesBackend from timenet.writer.progress import ProgressStage, WriteProgressEvent @@ -99,6 +101,11 @@ def test_a_card_that_cannot_be_read_is_skipped_with_a_warning(tmp_path, monkeypa found = installed_builder("timenet/hello-world") assert found is not None assert found.card.dataset_id == "timenet/hello-world" + # The installed connectors still answer for the other datasets. + with pytest.warns(ConnectorCardWarning, match=expected): + answer = InstalledConnectors().resolve("timenet/hello-world", None) + assert isinstance(answer, FoundInConnector) + assert answer.version == Version(1, 0, 0) def test_build_in_process_round_trips_the_connector_default_zarr_backend(tmp_path, monkeypatch): diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index a58b69b67..1fab16f92 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -13,7 +13,7 @@ from timenet.registry.dependencies import prepare_local_copy from timenet.registry.local import LocalRegistry from timenet.registry.shared import complete_copy -from timenet.types.references import DatasetRef, split_ref +from timenet.types.references import DatasetRef, is_latest, split_ref def build_release( # ruff: ignore[too-many-arguments] @@ -59,7 +59,8 @@ def build_release( # ruff: ignore[too-many-arguments] """ # ruff: ignore[docstring-extraneous-exception] - raised by the plan, the copy and the backends options = options or BuildOptions() dataset_id, version = split_ref(dataset) - found = declared(dataset_id, version) + # A connector builds only the version of its card, so "latest" asks for that version. + found = declared(dataset_id, None if is_latest(version) else version) card, requested = found.card, found.release steps = plan_parents( dataset_id, diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index afbea2b7e..eb9791782 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -21,7 +21,9 @@ from timenet.registry.dependencies import prepare_local_copy, validate_manifest_reference from timenet.registry.shared import complete_copy from timenet.registry.version import DatasetVersion -from timenet.types import DatasetMetadata, Domain, License, Task +from timenet.sources import FoundInRegistry, NotFound, NotServed, ReleaseLookup +from timenet.types import Access, DatasetMetadata, Domain, License, Task, Version +from timenet.types.access import access_requirements, effective_access from timenet.types.references import DatasetRef @@ -33,7 +35,7 @@ class BaseRegistry(ABC): - """A registry of dataset manifests, files, and searchable metadata.""" + """A registry that provides TimeF dataset metadata, manifests, and files.""" def __str__(self) -> str: return type(self).__name__ @@ -127,6 +129,29 @@ def open_version(self, dataset_id: str, version: str | None = None) -> DatasetVe TimeNetDatasetNotFoundError: If the dataset id or version is unknown. """ + def resolve(self, dataset_id: str, version: Version | None) -> ReleaseLookup: + """Check whether the registry serves a dataset release. + + If :attr:`requires_open_access` is true, the registry serves a release only if its card and each card in its + lineage declare open access. + + Args: + dataset_id: The dataset ID. + version: An exact version, or ``None`` for the latest version. + + Returns: + ``FoundInRegistry`` if served, ``NotServed`` if access is restricted, or ``NotFound`` if absent. + """ + try: + manifest = self.get_manifest(dataset_id, None if version is None else str(version)) + except TimeNetDatasetNotFoundError: + return NotFound() + held = manifest.metadata.dataset_version + access = effective_access(manifest.metadata, manifest.lineage) + if access is Access.OPEN or not self.requires_open_access: + return FoundInRegistry(held, self, manifest) + return NotServed(held, access, access_requirements(manifest.metadata, manifest.lineage), manifest) + def download_version( # ruff: ignore[too-many-arguments] self, dataset_id: str, diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index 0376d8ab0..e0a077263 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -9,13 +9,14 @@ from timenet._fs import staged_directory from timenet.dataset import TimeFDataset -from timenet.errors import TimeFFormatError, TimeNetDatasetNotFoundError, TimeNetFileError +from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError from timenet.format.constants import MANIFEST_FILE from timenet.manifest import Manifest, PayloadSource, read_manifest from timenet.registry._paths import release_dir from timenet.registry.dependencies import check_file_sizes, open_local_release from timenet.registry.version import DatasetVersion from timenet.registry.writable import WritableRegistry +from timenet.sources import FoundInRegistry, NotFound, ReleaseLookup from timenet.types import DatasetMetadata, Version from timenet.types.metadata_fields import validate_dataset_id from timenet.types.references import DatasetRef, is_latest @@ -23,7 +24,7 @@ class LocalRegistry(WritableRegistry): - """Read and publish datasets in a local directory.""" + """A registry that reads and writes datasets in a local directory.""" def __init__(self, root: Path) -> None: """Open a local registry rooted at a directory. @@ -123,6 +124,33 @@ def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest: self._manifest_cache[dataset_id, resolved] = (mtime, manifest) return manifest + def resolve(self, dataset_id: str, version: Version | None) -> ReleaseLookup: + """Check whether the local registry holds a dataset release. + + Local releases are available at any access level. + + Args: + dataset_id: The dataset ID. + version: An exact version, or ``None`` for the latest version. + + Returns: + ``FoundInRegistry`` with the committed version, or ``NotFound`` if the release is absent. + + Raises: + TimeFValidationError: If ``dataset_id`` is invalid. + """ + try: + validate_dataset_id(dataset_id) + except ValueError as error: + raise TimeFValidationError(f"invalid dataset ID {dataset_id!r}: {error}") from error + resolved = self._latest_version(dataset_id) if version is None else str(version) + if resolved is None: + return NotFound() + reference = DatasetRef.parse(f"{dataset_id}@{resolved}") + if not (release_dir(self._root, reference) / MANIFEST_FILE).exists(): + return NotFound() + return FoundInRegistry(reference.version, self) + def open_file(self, dataset_id: str, version: str, relpath: str) -> BinaryIO: """Open one file of a dataset version for binary reading. diff --git a/packages/timenet/src/timenet/registry/remote/_http.py b/packages/timenet/src/timenet/registry/remote/_http.py index 4dd7ea6ad..bfbdbf721 100644 --- a/packages/timenet/src/timenet/registry/remote/_http.py +++ b/packages/timenet/src/timenet/registry/remote/_http.py @@ -2,13 +2,14 @@ Owns how the SDK talks to the hosted registry. It prefixes ``/api/v1`` and attaches the bearer token and a ``timenet/`` User-Agent to API calls. It resolves presigned download URLs without -leaking the token to object storage. It maps HTTP status codes onto TimeNet errors. A sync -:class:`httpx.Client` serves the control plane. :meth:`new_async_client` hands out an +leaking the token to object storage. It maps HTTP status codes and network failures onto TimeNet +errors. A sync :class:`httpx.Client` serves the control plane. :meth:`new_async_client` hands out an :class:`httpx.AsyncClient` sharing the same configuration for the parallel downloader. """ import asyncio -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Generator, Mapping +from contextlib import contextmanager import importlib.metadata import logging import math @@ -124,6 +125,22 @@ async def _send_with_retry_async(send: Callable[[], Awaitable[httpx.Response]]) return await send() +@contextmanager +def _reaching(where: str) -> Generator[None]: + """Name ``where`` in the error of a request that fails on the network. + + Args: + where: The registry or storage location to name in the error. + + Raises: + TimeNetRegistryError: If a request fails to reach ``where``. + """ + try: + yield + except httpx.TransportError as error: + raise TimeNetRegistryError(f"cannot reach {where}: {error}") from error + + def timenet_user_agent() -> str: """Return the ``timenet/`` User-Agent used for backend tracing. @@ -192,7 +209,7 @@ def _redirect_target(response: httpx.Response, path: str) -> str: class RegistryHttpClient: - """A thin httpx wrapper for the registry API and its presigned object URLs.""" + """An HTTP client for the registry API and signed file URLs.""" @property def token(self) -> str | None: @@ -253,7 +270,7 @@ def api_headers(self) -> dict[str, str]: return headers def get_text(self, path: str) -> str: - """GET an API path and return the raw text body (used for ``manifest.json``). + """Fetch the text body from a registry API path. Args: path: A path under ``/api/v1``. @@ -261,7 +278,8 @@ def get_text(self, path: str) -> str: Returns: The response text. """ - response = _send_with_retry(lambda: self._client.get(f"{API_PREFIX}{path}", headers=self.api_headers())) + with _reaching(self._base_url): + response = _send_with_retry(lambda: self._client.get(f"{API_PREFIX}{path}", headers=self.api_headers())) _raise_for_status(response) return response.text @@ -270,20 +288,21 @@ def post(self, path: str, *, content: bytes | None = None, json: Any = None) -> Args: path: A path under ``/api/v1``. - content: Raw request body bytes (e.g. the manifest), mutually exclusive with ``json``. + content: The request bytes, mutually exclusive with ``json``. json: A JSON-serializable body. Returns: The raw response (already status-checked). """ - response = _send_with_retry( - lambda: self._client.post(f"{API_PREFIX}{path}", content=content, json=json, headers=self.api_headers()) - ) + with _reaching(self._base_url): + response = _send_with_retry( + lambda: self._client.post(f"{API_PREFIX}{path}", content=content, json=json, headers=self.api_headers()) + ) _raise_for_status(response) return response def resolve_presigned(self, dataset_id: str, version: str, relpath: str) -> str: - """Resolve a file's presigned URL via the download redirect, without following it. + """Get a signed download URL for a dataset file. Args: dataset_id: The ``org/name`` id. @@ -297,7 +316,10 @@ def resolve_presigned(self, dataset_id: str, version: str, relpath: str) -> str: TimeNetRegistryError: If the download endpoint does not redirect. """ # ruff: ignore[docstring-extraneous-exception] - raised by _redirect_target path = _download_path(dataset_id, version, relpath) - response = _send_with_retry(lambda: self._client.get(path, headers=self.api_headers(), follow_redirects=False)) + with _reaching(self._base_url): + response = _send_with_retry( + lambda: self._client.get(path, headers=self.api_headers(), follow_redirects=False) + ) return _redirect_target(response, path) def stream_to(self, url: str, sink: BinaryIO) -> None: @@ -307,7 +329,7 @@ def stream_to(self, url: str, sink: BinaryIO) -> None: url: The presigned URL. sink: A writable binary file object. """ - with self._client.stream("GET", url) as response: + with _reaching(f"the storage of {self._base_url}"), self._client.stream("GET", url) as response: if response.is_error: response.read() # a streaming response has no body until read; _raise_for_status needs it _raise_for_status(response) @@ -322,7 +344,8 @@ def put(self, url: str, *, content: bytes, headers: Mapping[str, str]) -> None: content: The request body. headers: Headers the presign requires (e.g. the checksum binding). """ - response = self._client.put(url, content=content, headers=dict(headers)) + with _reaching(f"the storage of {self._base_url}"): + response = self._client.put(url, content=content, headers=dict(headers)) _raise_for_status(response) def new_async_client(self) -> httpx.AsyncClient: diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index 258363079..d33cece8f 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -18,7 +18,7 @@ from timenet.manifest import Manifest, parse_manifest from timenet.registry.lazy_read import LazyReadRegistry from timenet.registry.remote._download import ProgressCallback, download_version_files -from timenet.registry.remote._http import RegistryHttpClient +from timenet.registry.remote._http import RegistryHttpClient, _reaching from timenet.registry.remote.filesystem import RegistryFileSystem from timenet.registry.remote.models import ( DatasetDetailModel, @@ -151,9 +151,10 @@ def download_release( dataset_id, version = reference.dataset_id, str(reference.version) if manifest is None: manifest = self.get_manifest(dataset_id, version) - download_version_files( - self._http, manifest, dataset_id, version, Path(directory), force=force, progress_cb=progress_cb - ) + with _reaching(self._base_url): + download_version_files( + self._http, manifest, dataset_id, version, Path(directory), force=force, progress_cb=progress_cb + ) def put_release(self, from_directory: Path) -> DatasetRef: """Publish a release directory to the remote registry. diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 4a7d894e5..4ec31d639 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -11,6 +11,8 @@ boto3 is optional. Install the ``timenet[s3]`` extra to use this backend. """ +from collections.abc import Generator +from contextlib import contextmanager import os from pathlib import Path from typing import Any, BinaryIO @@ -132,7 +134,8 @@ def open_file(self, dataset_id: str, version: str, relpath: str) -> BinaryIO: client = self._client() try: - response = client.get_object(Bucket=self._bucket, Key=self._key(dataset_id, version, relpath)) + with self._reaching(): + response = client.get_object(Bucket=self._bucket, Key=self._key(dataset_id, version, relpath)) except botocore.exceptions.ClientError as exc: raise TimeNetDatasetNotFoundError(f"no object {relpath!r} for {dataset_id!r} version {version!r}") from exc return response["Body"] @@ -167,16 +170,17 @@ def put_release(self, from_directory: Path) -> DatasetRef: client = self._client() final_base = self._key(manifest.dataset_id, str(manifest.metadata.dataset_version)) temp_base = f"{final_base}.tmp-{uuid.uuid4().hex}" - try: - self._delete_prefix(client, f"{final_base}.tmp-") # sweep a prior crashed publish's temp prefix - data_relpaths = sorted({part.path for part in manifest.files.all_files()}) - for relpath in (*data_relpaths, MANIFEST_FILE): - client.upload_file(str(from_directory / relpath), self._bucket, f"{temp_base}/{relpath}") - for relpath in data_relpaths: - self._move(client, f"{temp_base}/{relpath}", f"{final_base}/{relpath}") - self._move(client, f"{temp_base}/{MANIFEST_FILE}", f"{final_base}/{MANIFEST_FILE}") - finally: - self._delete_prefix(client, f"{temp_base}/") # remove any objects left staged in the temp prefix + with self._reaching(): + try: + self._delete_prefix(client, f"{final_base}.tmp-") # sweep a prior crashed publish's temp prefix + data_relpaths = sorted({part.path for part in manifest.files.all_files()}) + for relpath in (*data_relpaths, MANIFEST_FILE): + client.upload_file(str(from_directory / relpath), self._bucket, f"{temp_base}/{relpath}") + for relpath in data_relpaths: + self._move(client, f"{temp_base}/{relpath}", f"{final_base}/{relpath}") + self._move(client, f"{temp_base}/{MANIFEST_FILE}", f"{final_base}/{MANIFEST_FILE}") + finally: + self._delete_prefix(client, f"{temp_base}/") # remove any objects left staged in the temp prefix return DatasetRef(dataset_id=manifest.dataset_id, version=manifest.metadata.dataset_version) @staticmethod @@ -218,6 +222,20 @@ def _client() -> Any: return boto3.client("s3") + @contextmanager + def _reaching(self) -> Generator[None]: + """Name the bucket in the error of a request that fails on the network. + + Raises: + TimeNetRegistryError: If a request fails to reach the bucket. + """ + import botocore.exceptions # ruff: ignore[import-outside-top-level] - optional S3 dependency + + try: + yield + except (botocore.exceptions.ConnectionError, botocore.exceptions.HTTPClientError) as error: + raise TimeNetRegistryError(f"cannot reach {self._display_root()}: {error}") from error + def _key(self, *parts: str) -> str: # ruff: ignore[docstring-missing-returns] """Build a bucket-relative key: the prefix, then ``datasets/``, then the path parts.""" @@ -251,7 +269,8 @@ def _get_bytes(self, key: str) -> bytes | None: client = self._client() try: - response = client.get_object(Bucket=self._bucket, Key=key) + with self._reaching(): + response = client.get_object(Bucket=self._bucket, Key=key) except botocore.exceptions.ClientError as exc: if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: return None # a genuinely absent object @@ -263,12 +282,13 @@ def _latest_version(self, dataset_id: str) -> str | None: prefix = f"{self._key(dataset_id)}/" client = self._client() versions: list[str] = [] - paginator = client.get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix, Delimiter="/"): - for entry in page.get("CommonPrefixes", []): - name = entry["Prefix"][len(prefix) :].strip("/") - if _is_version(name): - versions.append(name) + with self._reaching(): + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix, Delimiter="/"): + for entry in page.get("CommonPrefixes", []): + name = entry["Prefix"][len(prefix) :].strip("/") + if _is_version(name): + versions.append(name) return max(versions, key=Version.parse) if versions else None diff --git a/packages/timenet/src/timenet/sources.py b/packages/timenet/src/timenet/sources.py new file mode 100644 index 000000000..a72aaaea0 --- /dev/null +++ b/packages/timenet/src/timenet/sources.py @@ -0,0 +1,193 @@ +"""Find dataset releases across registries and installed connectors.""" + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypeAlias, Union + +from timenet import builders +from timenet.errors import TimeNetAccessError +from timenet.types.access import Access, AccessRequirement +from timenet.types.references import LATEST_VERSION, DatasetRef +from timenet.types.version import Version + + +if TYPE_CHECKING: + from timenet.manifest import Manifest + from timenet.registry.base import BaseRegistry + + +# The registry module imports this module, so the alias names the registry as a forward reference. +ReleaseSource: TypeAlias = Union["BaseRegistry", "InstalledConnectors"] +"""A source that can serve or build a dataset release: a registry or the installed connectors.""" + + +@dataclass(frozen=True) +class FoundInRegistry: + """A result for a release that a registry serves. + + Attributes: + version: The exact version. + registry: The registry that serves the release. + manifest: The release manifest that the registry read, or ``None``. + """ + + version: Version + registry: "BaseRegistry" + manifest: "Manifest | None" = field(default=None, compare=False, repr=False) + + +@dataclass(frozen=True) +class FoundInConnector: + """A result for a release that an installed connector builds. + + Attributes: + builder: The backend and card that the lookup read. + """ + + builder: builders.InstalledBuilder + + @property + def version(self) -> Version: + """The version that the card declares.""" + return self.builder.card.dataset_version + + +Found: TypeAlias = FoundInRegistry | FoundInConnector +"""A result for a release that a source serves or builds.""" + + +@dataclass(frozen=True) +class NotFound: + """A result for a release that the source cannot find.""" + + +@dataclass(frozen=True) +class NotServed: + """A result for a listed release whose files the source cannot serve. + + Attributes: + version: The exact version that the source lists. + access: The strictest access level of the release's card and the cards in its lineage. + requirements: Each card that is not open access, with its access level and access page. + manifest: The release manifest, or ``None``. + """ + + version: Version + access: Access + requirements: tuple[AccessRequirement, ...] + manifest: "Manifest | None" = field(default=None, compare=False, repr=False) + + def access_error(self, dataset_id: str) -> TimeNetAccessError: + """Create an error with the release's access requirements. + + Args: + dataset_id: The dataset ID. + + Returns: + An error with the release, its access level, and its access pages. + """ + release = DatasetRef(dataset_id=dataset_id, version=self.version) + return TimeNetAccessError(release, self.access, self.requirements) + + +ReleaseLookup: TypeAlias = Found | NotFound | NotServed +"""The result of looking up a release in a source.""" + + +@dataclass(frozen=True) +class InstalledConnectors: + """A release source backed by installed dataset connectors. + + Each connector provides only the version in its card. + All instances compare equal. + """ + + def __str__(self) -> str: + if not tuple(builders.entry_points(group=builders.ENTRY_POINT_GROUP)): + return "installed connectors (no connector package is installed)" + return "installed connectors" + + def resolve(self, dataset_id: str, version: Version | None) -> ReleaseLookup: # ruff: ignore[no-self-use] (every source has it) + """Check whether an installed connector can build a release. + + The lookup uses the first entry point that claims the dataset. + + Args: + dataset_id: The dataset ID. + version: An exact version, or ``None`` for the latest version. + + Returns: + ``FoundInConnector`` if a connector builds the requested version, or ``NotFound``. + + Raises: + TimeNetInvalidCardError: If the connector's card cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend + builder = builders.installed_builder(dataset_id) + if builder is None or (version is not None and version != builder.card.dataset_version): + return NotFound() + return FoundInConnector(builder) + + +def resolve_release(sources: Sequence[ReleaseSource], dataset_id: str, version: str | None) -> ReleaseLookup: + """Find a source that provides the requested dataset release. + + An exact version comes from the first source that serves it, and no version means the newest version of the first + source that serves the dataset. ``"latest"`` is the highest version of any source, source order breaks ties, and + source errors stop the search. + + Args: + sources: The sources, in order. + dataset_id: The dataset ID. + version: An exact version, ``"latest"``, or ``None`` or ``""`` for no version. + + Returns: + A ``Found`` result if available, otherwise the first ``NotServed`` result or ``NotFound``. + + Raises: + TimeFValidationError: If ``version`` is neither ``"latest"`` nor a ``major.minor.patch`` version. + """ # ruff: ignore[docstring-extraneous-exception] - raised by Version.parse + if version == LATEST_VERSION: + return _resolve_latest(sources, dataset_id) + return _resolve_first(sources, dataset_id, Version.parse(version) if version else None) + + +def _resolve_first(sources: Sequence[ReleaseSource], dataset_id: str, version: Version | None) -> ReleaseLookup: + """Return the answer of the first source that serves ``version``, or its newest for ``None``, else the first listing.""" + listed: NotServed | None = None + for source in sources: + answer = source.resolve(dataset_id, version) + if isinstance(answer, Found): + return answer + if isinstance(answer, NotServed) and listed is None: + listed = answer + return NotFound() if listed is None else listed + + +def _resolve_latest(sources: Sequence[ReleaseSource], dataset_id: str) -> ReleaseLookup: + """Return the highest version that any source provides, from the first source that provides it.""" + best: Found | None = None + listed: list[tuple[ReleaseSource, NotServed]] = [] + before_best = 0 + for source in sources: + answer = source.resolve(dataset_id, None) + if isinstance(answer, NotServed): + listed.append((source, answer)) + elif isinstance(answer, Found) and (best is None or answer.version > best.version): + best, before_best = answer, len(listed) + if best is None: + return listed[0][1] if listed else NotFound() + return _ask_again(listed[:before_best], dataset_id, best.version) or best + + +def _ask_again(listed: Sequence[tuple[ReleaseSource, NotServed]], dataset_id: str, version: Version) -> Found | None: + """Ask each source that lists a higher version without serving it whether it serves ``version``. + + Returns: + The answer of the first such source that serves ``version``, or ``None``. + """ + for source, answer in listed: + if answer.version > version: + again = source.resolve(dataset_id, version) + if isinstance(again, Found): + return again + return None diff --git a/packages/timenet/src/timenet/types/references.py b/packages/timenet/src/timenet/types/references.py index fd68c447c..c79c71d1a 100644 --- a/packages/timenet/src/timenet/types/references.py +++ b/packages/timenet/src/timenet/types/references.py @@ -9,7 +9,7 @@ LATEST_VERSION = "latest" -"""Reference identifier that selects the newest committed dataset version.""" +"""Reference identifier that selects the newest committed dataset version across all sources.""" def is_latest(version: str | None) -> bool: @@ -24,15 +24,16 @@ def is_latest(version: str | None) -> bool: def split_ref(ref: str) -> tuple[str, str | None]: - """Split a dataset reference into its id and pinned version. + """Split a dataset reference into its id and requested version. - ``"org/id@1.0.0"`` -> ``("org/id", "1.0.0")``. ``"org/id@latest"`` or ``"org/id"`` -> ``("org/id", None)``. + ``"org/id@1.0.0"`` -> ``("org/id", "1.0.0")``. ``"org/id@latest"`` -> ``("org/id", "latest")``. ``"org/id"`` -> + ``("org/id", None)``. Args: ref: A dataset id, optionally suffixed with ``@`` or ``@latest``. Returns: - The bare dataset id and the pinned version string, or ``None`` for latest. + The bare dataset id and the version string, ``"latest"``, or ``None`` without a version. Raises: TimeFValidationError: If ``ref`` has more than one ``@``, an empty id or version, or a version @@ -43,11 +44,12 @@ def split_ref(ref: str) -> tuple[str, str | None]: dataset_id, separator, version = ref.partition("@") if not dataset_id: raise TimeFValidationError(f"dataset ref has an empty id: {ref!r}") - if not separator or version == LATEST_VERSION: + if not separator: return dataset_id, None if not version: raise TimeFValidationError(f"dataset ref has an empty version: {ref!r}") - Version.parse(version) # Validate the pin. This raises TimeFValidationError on a malformed version. + if version != LATEST_VERSION: + Version.parse(version) # Validate the pin. This raises TimeFValidationError on a malformed version. return dataset_id, version @@ -71,7 +73,7 @@ def parse(cls, value: str) -> "DatasetRef": TimeFValidationError: If the reference is malformed or does not contain an exact version. """ dataset_id, version = split_ref(value) - if version is None: + if is_latest(version): raise TimeFValidationError(f"an exact dataset release is required: {value!r}") try: return cls.model_validate({"dataset_id": dataset_id, "version": version}) diff --git a/packages/timenet/tests/registry/test_conformance.py b/packages/timenet/tests/registry/test_conformance.py index f365f56ca..e22356e5d 100644 --- a/packages/timenet/tests/registry/test_conformance.py +++ b/packages/timenet/tests/registry/test_conformance.py @@ -16,6 +16,7 @@ from timenet.dataset.axis import RegularAxis from timenet.errors import TimeNetDatasetNotFoundError, TimeNetFileError from timenet.registry import LocalRegistry, RemoteRegistry, S3Registry, WritableRegistry +from timenet.sources import FoundInRegistry, NotFound from timenet.testing import make_dataset, sine_loader from timenet.types import ( Annotation, @@ -154,6 +155,18 @@ def test_get_manifest_unknown_version_raises(backend): backend.registry.get_manifest("timenet/hello-world", "9.9.9") +# ---- resolve ---------------------------------------------------------------------------------- + + +def test_a_registry_resolves_a_held_release_and_not_an_unknown_one(backend): + registry = backend.registry + assert registry.resolve("timenet/hello-world", None) == FoundInRegistry(Version(1, 1, 0), registry) + assert registry.resolve("timenet/hello-world", Version(1, 0, 0)) == FoundInRegistry(Version(1, 0, 0), registry) + assert registry.resolve("no/such", None) == NotFound() + assert registry.resolve("no/such", Version(1, 0, 0)) == NotFound() + assert registry.resolve("timenet/hello-world", Version(9, 9, 9)) == NotFound() + + # ---- open_file -------------------------------------------------------------------------------- diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 8ae6242c4..c00426a6a 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -23,8 +23,9 @@ from timenet.manifest import Manifest from timenet.reader import LoadPlan, TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version +from timenet.sources import FoundInRegistry, NotServed from timenet.testing import assert_datasets_equal, make_dataset -from timenet.types import Access, DatasetSchema, Domain, License +from timenet.types import Access, DatasetSchema, Domain, License, Version from timenet.types.access import AccessRequirement from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.reference_keys import ParentAlias @@ -558,8 +559,8 @@ def handler(request): assert requests == prepared_requests -@pytest.mark.parametrize("page", ["own", "ancestor"]) -def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tmp_path, page): +def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: + """Write a release that is not open into ``origin``; return its ID and the release whose card is credentialed.""" dataset = make_dataset() gated = {"access": Access.CREDENTIALED, "access_url": "https://example.org/dua"} release = DatasetRef(dataset_id=dataset.metadata.dataset_id, version=dataset.metadata.dataset_version) @@ -571,14 +572,72 @@ def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tm dataset._import_state.lineage["test/ancestor@1.0.0"] = ParentDatasetCard(metadata=ancestor) release = DatasetRef.parse("test/ancestor@1.0.0") dataset.derive_schema() - with TimeFWriter(tmp_path / "origin", dataset) as writer: + with TimeFWriter(origin, dataset) as writer: writer.write() + return dataset.metadata.dataset_id, release + + +@pytest.mark.parametrize("page", ["own", "ancestor"]) +def test_a_local_client_checks_access_before_it_copies_from_a_remote_registry(tmp_path, page): + dataset_id, release = _gated_release(tmp_path / "origin", page) transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") with pytest.raises(TimeNetAccessError) as caught: - TimeNet(tmp_path / "local").open(dataset.metadata.dataset_id, auto_build=False, fetch_from=remote) + TimeNet(tmp_path / "local").open(dataset_id, auto_build=False, fetch_from=remote) assert caught.value.requirements == (AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua"),) assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) assert not (tmp_path / "local").exists() + + +def test_a_remote_registry_lists_but_does_not_serve_a_release_that_is_not_open(tmp_path): + # The card of the release is open. A card in its lineage is not. + dataset_id, release = _gated_release(tmp_path / "origin", "ancestor") + transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) + remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + requirement = AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua") + + answer = remote.resolve(dataset_id, None) + + assert answer == NotServed(Version(1, 0, 0), Access.CREDENTIALED, (requirement,)) + # The answer keeps the manifest that the lookup fetched. + assert isinstance(answer, NotServed) + assert answer.manifest is not None + assert answer.manifest.metadata.dataset_id == dataset_id + assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) + # A local registry serves every release that it holds. + local = LocalRegistry(tmp_path / "origin") + assert local.resolve(dataset_id, None) == FoundInRegistry(Version(1, 0, 0), local) + + +def test_resolve_keeps_the_manifest_that_it_fetched(version_dir, tmp_path): + registry, requests = _remote(version_dir, tmp_path) + _, manifest = version_dir + dataset_id = manifest.metadata.dataset_id + + answer = registry.resolve(dataset_id, None) + + assert isinstance(answer, FoundInRegistry) + assert answer.manifest == manifest + assert sum(request.url.path.endswith("/manifest") for request in requests) == 1 + + +def _offline(request): + raise httpx.ConnectError("the network is down", request=request) + + +def test_an_unreachable_registry_raises_a_registry_error_that_names_it(tmp_path): + remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(_offline), cache_dir=tmp_path) + + with pytest.raises(TimeNetRegistryError, match=r"^cannot reach http://api\.local: the network is down$"): + remote.resolve("timenet/hello-world", None) + + +def test_a_network_drop_during_the_parallel_download_raises_a_registry_error(version_dir, tmp_path): + _, manifest = version_dir + remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(_offline)) + release = DatasetRef(dataset_id=manifest.metadata.dataset_id, version=manifest.metadata.dataset_version) + + with pytest.raises(TimeNetRegistryError, match=r"^cannot reach http://api\.local: the network is down$"): + remote.download_release(release, tmp_path / "copy", manifest=manifest) diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index e7857904b..62d5f95f5 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -4,6 +4,7 @@ boto3 = pytest.importorskip("boto3") pytest.importorskip("moto") +import botocore.exceptions # ruff: ignore[module-import-not-at-top-of-file] from moto.server import ThreadedMotoServer # ruff: ignore[module-import-not-at-top-of-file] import pyarrow.fs as pafs # ruff: ignore[module-import-not-at-top-of-file] @@ -16,7 +17,7 @@ from timenet.reader import TimeFReader # ruff: ignore[module-import-not-at-top-of-file] from timenet.registry import LocalRegistry, S3Registry # ruff: ignore[module-import-not-at-top-of-file] from timenet.testing import assert_datasets_equal, make_dataset # ruff: ignore[module-import-not-at-top-of-file] -from timenet.types import DatasetSchema # ruff: ignore[module-import-not-at-top-of-file] +from timenet.types import DatasetSchema, Version # ruff: ignore[module-import-not-at-top-of-file] from timenet.types.references import DatasetRef # ruff: ignore[module-import-not-at-top-of-file] @@ -135,3 +136,20 @@ def test_list_and_search_are_unsupported(tmp_path): def test_rejects_a_non_s3_uri(tmp_path): with pytest.raises(TimeNetRegistryError, match="s3://"): S3Registry("https://not-s3", cache_dir=tmp_path) + + +class _Unreachable: + """An S3 client whose every request fails to reach the endpoint.""" + + def __getattr__(self, name): + def request(*args, **kwargs): + raise botocore.exceptions.EndpointConnectionError(endpoint_url="https://s3.example") + + return request + + +def test_an_unreachable_bucket_raises_a_registry_error_that_names_it(monkeypatch): + registry = S3Registry("s3://tn-test/registry") + monkeypatch.setattr(registry, "_client", _Unreachable) + with pytest.raises(TimeNetRegistryError, match=r"^cannot reach s3://tn-test/registry: Could not connect"): + registry.resolve("timenet/hello-world", Version(1, 0, 0)) diff --git a/packages/timenet/tests/test_refs.py b/packages/timenet/tests/test_refs.py index 01186ea60..8a144cd50 100644 --- a/packages/timenet/tests/test_refs.py +++ b/packages/timenet/tests/test_refs.py @@ -9,7 +9,7 @@ [ ("timenet/hello-world", ("timenet/hello-world", None)), ("org/name", ("org/name", None)), - ("org/name@latest", ("org/name", None)), + ("org/name@latest", ("org/name", "latest")), ("org/name@1.2.3", ("org/name", "1.2.3")), ("timenet/hello-world@0.0.1", ("timenet/hello-world", "0.0.1")), ], diff --git a/packages/timenet/tests/test_sources.py b/packages/timenet/tests/test_sources.py new file mode 100644 index 000000000..904d3863a --- /dev/null +++ b/packages/timenet/tests/test_sources.py @@ -0,0 +1,152 @@ +from pathlib import Path + +import pytest + +from timenet import builders +from timenet.errors import ( + TimeFValidationError, + TimeNetRegistryError, +) +from timenet.registry import LocalRegistry +from timenet.sources import ( + FoundInConnector, + FoundInRegistry, + InstalledConnectors, + NotFound, + NotServed, + resolve_release, +) +from timenet.testing import install_backends, make_dataset +from timenet.types import Access, Version + + +V1, V11, V2 = Version(1, 0, 0), Version(1, 1, 0), Version(2, 0, 0) + + +class _Source(LocalRegistry): + """A registry that serves the versions in ``served`` and lists the versions in ``listed`` without serving them.""" + + def __init__(self, *served: str, listed: tuple[str, ...] = ()): + super().__init__(Path("unused")) + self.served, self.listed = set(served), set(listed) + + def resolve(self, dataset_id, version): + known = sorted(self.served | self.listed, key=Version.parse) + wanted = (known[-1] if known else None) if version is None else str(version) + if wanted in self.served: + return FoundInRegistry(Version.parse(wanted), self) + if wanted in self.listed: + return NotServed(Version.parse(wanted), Access.CREDENTIALED, ()) + return NotFound() + + +class _Broken(_Source): + def resolve(self, dataset_id, version): + raise TimeNetRegistryError("the registry does not answer") + + +class _Backend: + def __init__(self, card): + self._card = card + + def dataset_ids(self): + return [self._card.dataset_id] + + def card(self, dataset_id): + return self._card if dataset_id == self._card.dataset_id else None + + +def _fail(*args, **kwargs): + raise AssertionError("resolve() must not read every manifest") + + +def test_an_exact_version_comes_from_the_first_source_that_provides_it(): + second = _Source("1.0.0") + assert resolve_release([_Source(), second, _Source("1.0.0")], "a/b", "1.0.0") == FoundInRegistry(V1, second) + + +def test_without_a_version_the_first_source_that_serves_the_dataset_gives_its_newest_version(): + first = _Source("1.0.0") + sources = [_Source(), first, _Source("1.1.0")] + assert resolve_release(sources, "a/b", None) == FoundInRegistry(V1, first) + assert resolve_release(sources, "a/b", "") == FoundInRegistry(V1, first) + + +def test_without_a_version_a_source_that_does_not_serve_the_dataset_falls_through(): + second = _Source("1.0.0") + listed = _Source(listed=("2.0.0",)) + assert resolve_release([listed, second, _Source("1.1.0")], "a/b", None) == FoundInRegistry(V1, second) + # With no source that serves it, the access error of the first listing comes last. + assert resolve_release([listed, _Source()], "a/b", None) == NotServed(V2, Access.CREDENTIALED, ()) + + +def test_latest_is_the_highest_version_across_all_sources(): + remote = _Source("1.0.0", "1.1.0") + assert resolve_release([_Source("1.0.0"), remote], "a/b", "latest") == FoundInRegistry(V11, remote) + + +def test_latest_comes_from_the_first_source_that_holds_the_highest_version(): + store = _Source("1.1.0") + assert resolve_release([store, _Source("1.1.0")], "a/b", "latest") == FoundInRegistry(V11, store) + + +def test_a_release_that_is_not_served_falls_through_and_does_not_count_for_latest(): + connectors = _Source("1.0.0") + assert resolve_release([_Source(listed=("2.0.0",)), connectors], "a/b", "latest") == FoundInRegistry(V1, connectors) + assert resolve_release([_Source(listed=("2.0.0",))], "a/b", "2.0.0") == NotServed(V2, Access.CREDENTIALED, ()) + + +def test_latest_asks_a_source_that_did_not_serve_a_higher_version_for_the_chosen_one(): + hosted = _Source("1.0.0", listed=("2.0.0",)) + assert resolve_release([hosted, _Source("1.0.0")], "a/b", "latest") == FoundInRegistry(V1, hosted) + + +def test_no_source_gives_not_found(): + assert resolve_release([_Source(), _Source()], "a/b", None) == NotFound() + assert resolve_release([_Source("1.0.0")], "a/b", "2.0.0") == NotFound() + + +def test_an_error_of_a_source_stops_the_resolution(): + with pytest.raises(TimeNetRegistryError): + resolve_release([_Broken(), _Source("1.0.0")], "a/b", "1.0.0") + with pytest.raises(TimeNetRegistryError): + resolve_release([_Source("1.0.0"), _Broken()], "a/b", "latest") + + +def test_installed_connectors_answer_with_the_builder_of_the_declared_version(monkeypatch): + card = make_dataset().metadata + backend = _Backend(card) + install_backends(monkeypatch, backend) + latest, exact = ( + InstalledConnectors().resolve(card.dataset_id, None), + InstalledConnectors().resolve(card.dataset_id, V1), + ) + assert isinstance(latest, FoundInConnector) and isinstance(exact, FoundInConnector) + assert (latest.builder.backend, latest.version) == (exact.builder.backend, exact.version) == (backend, V1) + + +def test_installed_connectors_answer_not_found_for_another_version(monkeypatch): + card = make_dataset().metadata + install_backends(monkeypatch, _Backend(card)) + assert InstalledConnectors().resolve(card.dataset_id, V2) == NotFound() + assert InstalledConnectors().resolve("x/y", None) == NotFound() + + +def test_installed_connectors_without_a_package_answer_not_found(monkeypatch): + monkeypatch.setattr(builders, "entry_points", lambda group: []) + assert InstalledConnectors().resolve("timenet/hello-world", None) == NotFound() + assert "no connector package is installed" in str(InstalledConnectors()) + + +def test_a_local_registry_answers_not_found_without_reading_every_manifest(tmp_path, monkeypatch): + registry = LocalRegistry(tmp_path) + registry.put_dataset(make_dataset()) + monkeypatch.setattr(LocalRegistry, "list_datasets", _fail) + assert registry.resolve("test/unknown", None) == NotFound() + assert registry.resolve("test/unknown", V1) == NotFound() + assert registry.resolve("timenet/hello-world", V2) == NotFound() + + +def test_a_local_registry_rejects_an_invalid_dataset_id(tmp_path): + with pytest.raises(TimeFValidationError): + LocalRegistry(tmp_path).resolve("../evil", None) diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.py index 127eb1cad..ae2f66a01 100644 --- a/scripts/gen_api_docs.py +++ b/scripts/gen_api_docs.py @@ -19,7 +19,8 @@ from __future__ import annotations import ast -from dataclasses import dataclass +from collections import Counter +from dataclasses import dataclass, replace from pathlib import Path import re @@ -58,13 +59,13 @@ @dataclass(frozen=True) class ApiModule: - """One module to document. + """A module's API page settings. Attributes: - dotted: Importable dotted path, e.g. ``timenet.registry``. - slug: Output filename stem under ``docs/api/`` (the last path component). - summary: Cleaned first line of the module docstring, or ``""``. - show_submodules: Render submodules recursively (packages without ``__all__``). + dotted: The import path, such as ``timenet.registry``. + slug: The page name under ``docs/api/``: the last part of the import path, or the full path on a clash. + summary: The cleaned first line of the module docstring, or ``""``. + show_submodules: Whether to include submodules recursively. True for a package without ``__all__``. """ dotted: str @@ -257,13 +258,24 @@ def _sub(match: re.Match[str]) -> str: def main() -> None: - """Generate the API pages and landing page, and rewrite the nav region. + """Generate the API reference pages and landing page, and rewrite the nav region of ``zensical.toml``. + + It also deletes each page in ``docs/api/`` that no current module maps to. Raises: - SystemExit: On a slug collision or a missing nav marker region. + SystemExit: If page names clash or the nav markers are missing or duplicated. """ OUTPUT_DIR.mkdir(parents=True, exist_ok=True) by_package = {pkg: _discover(pkg, root) for pkg, root in PACKAGE_ROOTS.items()} + # Both packages can have a module with the same name, such as timenet.sources and + # timenet_connectors.sources. The page of each such module takes its full dotted path. + shared = { + slug for slug, count in Counter(mod.slug for mods in by_package.values() for mod in mods).items() if count > 1 + } + by_package = { + pkg: [replace(mod, slug=mod.dotted) if mod.slug in shared else mod for mod in mods] + for pkg, mods in by_package.items() + } all_mods = [mod for mods in by_package.values() for mod in mods] # "index" is reserved for the landing page written by _write_index below. diff --git a/zensical.toml b/zensical.toml index a10b4762d..5e0b129e0 100644 --- a/zensical.toml +++ b/zensical.toml @@ -22,7 +22,7 @@ nav = [ { "Datasets" = [{ "Use datasets" = [{ "The client (SDK)" = "client.md" }, { "Registries" = "registry.md" }, { "Load into your stack" = "usage.md" }] }, { "Build datasets" = [{ "Connectors" = "connectors.md" }, { "Build & publish" = "build.md" }] }, { "Catalog" = [{ "Datasets" = "catalog/datasets.md" }, { "Benchmarks" = "catalog/benchmarks.md" }, { "Licensing" = "catalog/licensing.md" }] }] }, { "CLI" = [{ "Overview" = "cli/index.md" }, { "timenet (consumer)" = "cli/timenet.md" }, { "timenet-build (producer)" = "cli/build.md" }] }, # >>> api-nav: generated by scripts/gen_api_docs.py — do not edit >>> - { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, + { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.sources" = "api/timenet.sources.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/timenet_connectors.sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, # <<< api-nav <<< ]