From 50c5f40eb731d089c3fbcafcd3855b5383fe7c1b Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Wed, 4 Nov 2026 14:20:10 +0100 Subject: [PATCH 1/5] feat(client)!: take an ordered list of sources A client now takes one ordered list of sources as TimeNet(sources=[...]). An item is a registry, InstalledConnectors(), or a string or Path that open_registry() opens. A bare string or Path, an empty list and an item of another type raise TimeFValidationError. A string that names no registry raises TimeNetRegistryError, which names the item, such as sources[0]. Every registry takes read_only=False, and the property read_only reads the flag. A read-only registry refuses each write with TimeNetRegistryError: put_dataset(), put_release(), a copy into it, and a run of a plan that copies or builds into it. Builds land in the first LocalRegistry of the list that is not read-only. A string or Path in the list opens a writable LocalRegistry. The client only reads a later writable local registry. A list with InstalledConnectors() needs a writable local registry, else TimeNet() raises TimeFValidationError. default_sources() makes the list of TimeNet(): the local registry at /registry, then timenet://, then InstalledConnectors(). A registry URL in TIMENET_REGISTRY takes the place of timenet://. A local path in TIMENET_REGISTRY is the local registry, and the list then has no other registry. The remote_registry argument puts a registry after the local registry. open(), load() and open_torch() pass the list and that local registry to read_release(). The fetch_from= and auto_build= keywords go away, and so do _read_sources() and _missing_hint(). If no source provides a release, the hint asks for a registry that holds it. If the list has no InstalledConnectors(), the hint also names it. An access error names InstalledConnectors() and timenet-build build as the ways to build the release locally. get(), list(), search() and download() still ask one registry. _catalog() picks the first registry of the list that is not that local registry, else that local registry. download() still copies into TIMENET_STORAGE. Each command of the consumer CLI builds its client from default_sources(), and --registry becomes the remote_registry argument. timenet_connectors.load() reads with TimeNet().load(), because its build already put each parent and payload source into the local registry of the default list. open_registry() is now the one parser of a registry value. It raises TimeNetRegistryError, not ValueError, and it rejects a blank value instead of opening the current directory. Its label argument names the value in the error. open_writable_registry() goes away, because every registry is writable. timenet info and timenet download drop --version, because org/name@version names a version. The examples show the list of sources. BREAKING CHANGE: TimeNet takes sources= instead of registry= and storage_path=. open(), load() and open_torch() no longer take fetch_from= and auto_build=. The default client also reads the local registry and builds with installed connectors. timenet download no longer takes --storage. open_registry() raises TimeNetRegistryError, not ValueError, and rejects a blank value. open_writable_registry() is gone. timenet info and timenet download no longer take --version: use org/name@version. Part of TNET-157. --- examples/load_tsqa.py | 10 +- examples/test_mean_classifier.py | 2 +- .../src/timenet_connectors/api.py | 11 +- .../src/timenet_connectors/builder/cli.py | 3 +- .../tests/test_build_plan.py | 12 +- .../tests/test_end_to_end.py | 11 +- packages/timenet/src/timenet/cli/app.py | 59 ++- packages/timenet/src/timenet/client.py | 353 +++++++++------- .../timenet/src/timenet/coordinator/build.py | 3 + .../timenet/src/timenet/registry/__init__.py | 2 - .../timenet/src/timenet/registry/_copy.py | 2 + packages/timenet/src/timenet/registry/base.py | 23 +- .../timenet/src/timenet/registry/factory.py | 63 +-- .../timenet/src/timenet/registry/lazy_read.py | 4 +- .../timenet/src/timenet/registry/local.py | 8 +- .../src/timenet/registry/remote/registry.py | 7 +- packages/timenet/src/timenet/registry/s3.py | 6 +- .../timenet/tests/coordinator/test_build.py | 11 +- packages/timenet/tests/reader/test_reader.py | 2 +- .../timenet/tests/registry/test_registry.py | 31 +- .../timenet/tests/registry/test_remote.py | 32 +- packages/timenet/tests/registry/test_s3.py | 7 +- packages/timenet/tests/test_cli.py | 19 +- packages/timenet/tests/test_client.py | 383 ++++++++++++------ 24 files changed, 662 insertions(+), 402 deletions(-) diff --git a/examples/load_tsqa.py b/examples/load_tsqa.py index 77a98e418..f361dbc51 100644 --- a/examples/load_tsqa.py +++ b/examples/load_tsqa.py @@ -5,10 +5,10 @@ uv run timenet-build build "chengsenwang/tsqa" --out .timenet-registry uv run python examples/load_tsqa.py -``TimeNet(registry).open(...)`` returns a reader, and its ``read()`` returns a ``TimeFDataset``. -``describe()`` prints its identity, counts, per-spec columns, and a record preview. Signal values -stay in storage until you ask for them. Pass a dataset ID as the first argument and a registry path -as the second argument. +``TimeNet(sources=[registry]).open(...)`` returns a reader, and its ``read()`` returns a +``TimeFDataset``. ``describe()`` prints its identity, counts, per-spec columns, and a record +preview. Signal values stay in storage until you ask for them. Pass a dataset ID as the first +argument and a registry path as the second argument. """ from pathlib import Path @@ -22,7 +22,7 @@ def main() -> None: arguments = sys.argv[1:] dataset_id = arguments[0] if arguments else "chengsenwang/tsqa" registry = Path(arguments[1]) if len(arguments) > 1 else Path(".timenet-registry") - with TimeNet(registry).open(dataset_id) as reader: + with TimeNet(sources=[registry]).open(dataset_id) as reader: dataset = reader.read() dataset.describe() diff --git a/examples/test_mean_classifier.py b/examples/test_mean_classifier.py index 0090cac99..e99f382d8 100644 --- a/examples/test_mean_classifier.py +++ b/examples/test_mean_classifier.py @@ -24,7 +24,7 @@ # Build and load through the same explicit local registry. registry = Path(".timenet-registry") timenet_connectors.build("timenet/test-mean", out=registry) -dataset = TimeNet(registry).load("timenet/test-mean") +dataset = TimeNet(sources=[registry]).load("timenet/test-mean") # A quick tour of what we loaded: identity, counts, specs, and a record preview. dataset.describe() diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py index f57b12dcf..88c5e03bc 100644 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ b/packages/timenet-connectors/src/timenet_connectors/api.py @@ -109,11 +109,16 @@ def load( ) -> "TimeFDataset": """Build a dataset if needed and load it into memory. + The build lands in the store of the default list of sources + (:func:`timenet.client.default_sources`), with each parent and payload source that it needs. The + read then reads it through that list. + Args: dataset_id: The dataset ID (``org/name``). version: The version to load, which the card must declare, or ``None`` for the latest. - fetch_from: The registry or URI that supplies missing parents and payload sources. - build_parents: Build missing parent releases with installed connectors. + fetch_from: The registry, or its URI, that the build copies missing parent releases and + payload sources from. + build_parents: Build missing parent releases from their installed connectors. Returns: The loaded dataset. @@ -121,4 +126,4 @@ def load( from timenet.client import TimeNet # ruff: ignore[import-outside-top-level] build(dataset_id, version=version, fetch_from=fetch_from, build_parents=build_parents) - return TimeNet().load(dataset_id, version, fetch_from=fetch_from) + return TimeNet().load(dataset_id, version) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index a49ac0511..c2ab2775c 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -30,7 +30,6 @@ copy_release, local_registry_path, open_registry, - open_writable_registry, ) from timenet.registry._paths import release_dir # ruff: ignore[import-private-name] from timenet.sources import InstalledConnectors @@ -68,7 +67,7 @@ def _resolve_target(out: str | None) -> WritableRegistry | Path: try: return local_registry_path(target) except TimeNetRegistryError: - return open_writable_registry(target) + return open_registry(target) @app.command() diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index 48acf8bf2..89111b6a4 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -31,7 +31,6 @@ from timenet.types.references import DatasetRef from timenet.values_backends import ValuesBackend import timenet_connectors -from timenet_connectors import api as api_module from timenet_connectors.builder import backend as backend_module, cli as cli_module, env as env_module from timenet_connectors.builder.backend import ConnectorBuilder from timenet_connectors.builder.cli import app @@ -567,23 +566,20 @@ def test_load_obtains_a_missing_parent_before_it_builds_and_reads(tmp_path, monk assert calls == (builds if obtain == "fetch_from" else [(PARENT.dataset_id, False), *builds]) -def test_load_reads_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): +def test_load_copies_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): + _install_cards(monkeypatch, parent={"parent": GRAND}) out = LocalRegistry(tmp_path / "out") _parent_without_its_source(out) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) monkeypatch.setenv("TIMENET_REGISTRY", str(out.root)) - # The build has nothing to do, and no builder backend can build the payload source, so only the - # read can get it. - monkeypatch.setattr(api_module, "build", lambda dataset_id, **options: None) - monkeypatch.setattr(builders, "entry_points", lambda group: []) dataset = timenet_connectors.load(PARENT.dataset_id, fetch_from=origin) assert len(dataset.records) == 2 - # The read opens the payload source in origin, where it is, and copies nothing. - assert not out.exists(GRAND.dataset_id, str(GRAND.version)) + # The build copies the payload source into the store, where the read finds it. + assert out.exists(GRAND.dataset_id, str(GRAND.version)) def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch): diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 64dc2a5d6..44fef0388 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -9,7 +9,8 @@ from timenet.engine import run_pipeline from timenet.errors import TimeFValidationError from timenet.provenance import build_env -from timenet.registry import RemoteRegistry +from timenet.registry import LocalRegistry, RemoteRegistry +from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal from timenet_connectors import build, load from timenet_connectors.builder.cli import _resolve_target, app as build_app @@ -53,7 +54,7 @@ def test_build_then_load_round_trips(converted, tmp_path): assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists() # CONSUME: load it back through the SDK and compare to the dataset the build wrote. - restored = TimeNet(registry, storage_path=tmp_path / "store").load("timenet/hello-world") + restored = TimeNet(sources=[str(registry)]).load("timenet/hello-world") [original] = converted assert_datasets_equal(original, restored) @@ -97,7 +98,7 @@ def test_build_honors_timenet_registry(clean_env, monkeypatch, tmp_path): assert runner.invoke(build_app, ["build", "timenet/hello-world", "--no-isolation"]).exit_code == 0 assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists() # The SDK resolves $TIMENET_REGISTRY the same way, so it reads back what the build just wrote. - assert TimeNet(storage_path=tmp_path / "store").list()[0].dataset_id == "timenet/hello-world" + assert TimeNet().list()[0].dataset_id == "timenet/hello-world" def test_build_out_overrides_timenet_registry(clean_env, monkeypatch, tmp_path): @@ -167,7 +168,7 @@ def test_build_cleans_cache_but_keep_flag_retains(tmp_path, monkeypatch): @pytest.mark.slow def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monkeypatch): monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - client = TimeNet(tmp_path / "registry") + client = TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]) dataset = client.load("timenet/hello-world") @@ -178,7 +179,7 @@ def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monk @pytest.mark.slow def test_an_isolated_build_records_its_environment(tmp_path, monkeypatch): monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - TimeNet(tmp_path / "registry").load("timenet/hello-world") + TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]).load("timenet/hello-world") manifest = json.loads((tmp_path / "registry/timenet/hello-world/1.0.0/manifest.json").read_text()) packages = manifest["build_env"]["packages"] diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 0dab74b7c..ca411fd93 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -13,7 +13,7 @@ from timenet.cache import CachedDataset, cached_datasets, clear_cache, human_bytes, raw_cache_size from timenet.cli.runner import run_cli from timenet.cli.ui import console as ui -from timenet.client import TimeNet +from timenet.client import TimeNet, default_sources from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access from timenet.types.tasks import TASKS @@ -22,7 +22,12 @@ app = typer.Typer(help="Browse a TimeNet registry and fetch datasets.", no_args_is_help=True) console = Console() -_registry_option = typer.Option(None, "--registry", "-r", help="Registry URL or path (else $TIMENET_REGISTRY).") +_registry_option = typer.Option( + None, + "--registry", + "-r", + help="Registry URL or path that this command reads (else $TIMENET_REGISTRY, then timenet://).", +) T = TypeVar("T") @@ -55,6 +60,19 @@ def _enum_list(values: list[str], factory: Callable[[str], T], flag: str, choice return parsed or None +def _client(registry: str | None) -> TimeNet: + """Create the client of a command, with the default list of sources. + + Args: + registry: The value of ``--registry``: the registry after the store, or ``None``. + + Returns: + The client. + """ + # Glue: --registry maps onto default_sources() until the CLI takes the flags of the defaults. + return TimeNet(sources=default_sources(remote_registry=registry)) + + def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table: """Build a table of dataset metadata (id, name, license, domains). @@ -83,7 +101,7 @@ def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table: @app.command("list") def list_datasets(registry: str | None = _registry_option) -> None: """List every dataset in the registry.""" - rows = TimeNet(registry).list() + rows = _client(registry).list() if not rows: typer.echo("No datasets in the registry.") return @@ -103,7 +121,7 @@ def search( # ruff: ignore[too-many-arguments, too-many-positional-arguments] limit: int = typer.Option(100, "--limit", min=0, help="Maximum results."), ) -> None: """Search datasets by filter. Repeat a flag to pass several values.""" - results = TimeNet(registry).search( + results = _client(registry).search( query=query or None, domain=_enum_list(domain, Domain, "--domain", [d.value for d in Domain]), task=_enum_list(task, lambda value: TASKS[TaskType(value)], "--task", [t.value for t in TaskType]), @@ -120,9 +138,9 @@ def search( # ruff: ignore[too-many-arguments, too-many-positional-arguments] @app.command() -def info(dataset_id: str, version: str | None = None, registry: str | None = _registry_option) -> None: - """Print a dataset's manifest summary: metadata, schema, and counts.""" - manifest = TimeNet(registry).get(dataset_id, version) +def info(dataset_id: str, registry: str | None = _registry_option) -> None: + """Display a dataset's metadata, schema, and counts.""" + manifest = _client(registry).get(dataset_id) counts = manifest.counts table = Table(title=manifest.dataset_id, title_justify="left", title_style="bold", show_header=False) table.add_column("Field", style="cyan") @@ -162,37 +180,34 @@ def info(dataset_id: str, version: str | None = None, registry: str | None = _re @app.command() -def download( - dataset_id: str, - version: str | None = None, - registry: str | None = _registry_option, - storage: str | None = typer.Option(None, "--storage", help="Local storage dir (else $TIMENET_STORAGE)."), -) -> None: - """Fetch a complete TimeF version to local storage. Prints its directory to stdout.""" - client = TimeNet(registry, storage_path=storage) - manifest = client.get(dataset_id, version) +def download(dataset_id: str, registry: str | None = _registry_option) -> None: + """Download a dataset version and print its local directory to stdout. + + Files go to ``$TIMENET_STORAGE`` or ``/storage``. + """ + client = _client(registry) + manifest = client.get(dataset_id) files = list(manifest.files.all_files()) total = sum(part.size for part in files) ui.status("⬇️", f"Downloading '{dataset_id}' ({len(files)} files, {human_bytes(total)})…") - target = _download(client, dataset_id, version, total) + target = _download(client, dataset_id, total) ui.success(f"Downloaded '{dataset_id}' ({target.name})") typer.echo(str(target)) -def _download(client: TimeNet, dataset_id: str, version: str | None, total: int) -> Path: +def _download(client: TimeNet, dataset_id: str, total: int) -> Path: """Download a version, rendering a live progress bar on stderr when the terminal supports it. Args: client: The client to download through. - dataset_id: The dataset id. - version: The version string, or ``None`` for the latest. + dataset_id: The dataset id, optionally followed by ``@version``. total: The total byte size of the version's files, for the bar's scale. Returns: The local version directory. """ if ui.quiet or not sys.stderr.isatty(): - return client.download(dataset_id, version) + return client.download(dataset_id) progress = Progress( TextColumn("[bold blue]{task.description}"), BarColumn(bar_width=None), @@ -207,7 +222,7 @@ def _download(client: TimeNet, dataset_id: str, version: str | None, total: int) ) with progress: task = progress.add_task("downloading", total=total) - return client.download(dataset_id, version, progress_cb=lambda n: progress.advance(task, n)) + return client.download(dataset_id, progress_cb=lambda n: progress.advance(task, n)) cache_app = typer.Typer(help="Inspect and clear the local cache.", no_args_is_help=True) diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index bef66e354..afff4d29a 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -1,9 +1,20 @@ -"""Browse, download, and read datasets with :class:`TimeNet`.""" +"""The :class:`TimeNet` class is the SDK's single entry point for using TimeNet from code. + +A client reads releases through an ordered list of sources: registries and +``InstalledConnectors()`` (see :mod:`timenet.sources`). Its methods ``open``, ``load``, and +``open_torch`` read a release where it is, through the read coordinator in +:mod:`timenet.coordinator.read`. If only ``InstalledConnectors()`` provides a release, they build it +into the first :class:`~timenet.registry.LocalRegistry` of the list that is not +``read_only=True``. The methods ``list``, ``get``, ``search``, and ``download`` ask one registry of the +list. :func:`default_sources` makes the list of ``TimeNet()``. +""" # The public API has a method named ``list``. Deferred annotations keep the type hint # ``list[str]`` resolved to the builtin type, not to the method. from __future__ import annotations +from collections.abc import Sequence +import os from pathlib import Path from typing import TYPE_CHECKING, TypeAlias, TypeVar @@ -37,7 +48,37 @@ T = TypeVar("T") _Metadatas: TypeAlias = list[DatasetMetadata] _OrList: TypeAlias = T | list[T] | None -_Sources: TypeAlias = list[ReleaseSource] + + +def default_sources(*, remote_registry: str | Path | None = None) -> list[ReleaseSource]: + """Return the list of sources of ``TimeNet()``: the store, a registry, and the installed connectors. + + The store is ``$TIMENET_REGISTRY`` if it names a local path, else ``/registry``. + The registry after the store is ``remote_registry``, else ``$TIMENET_REGISTRY`` if it names a + registry that is not local, else the hosted registry, ``timenet://``. A local + ``$TIMENET_REGISTRY`` without ``remote_registry`` gives a list without that registry. + + Args: + remote_registry: The registry between the store and the installed connectors: a registry + URL, a ``file://`` URI, or a local path. + + Returns: + A new list, made from the environment at the time of the call. + """ + # Glue: this body reads TIMENET_REGISTRY and TIMENET_HOME. The variables of the defaults + # (TIMENET_LOCAL_REGISTRY, TIMENET_REMOTE_REGISTRY, TIMENET_AUTO_BUILD) replace them later. + cfg = settings() + configured = None if cfg.registry is None else open_registry(cfg.registry) + local = configured if isinstance(configured, LocalRegistry) else None + store = LocalRegistry(cfg.registry_path if local is None else local.root) + if remote_registry is not None: + remote = open_registry(remote_registry) + elif local is not None: + # A client of a local $TIMENET_REGISTRY read no other registry, and neither does this list. + return [store, InstalledConnectors()] + else: + remote = configured if configured is not None else open_registry("timenet://") + return [store, remote, InstalledConnectors()] def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None]: @@ -59,45 +100,91 @@ def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None] return ref_id, ref_version if ref_version is not None else (version or None) +def _open_sources(sources: Sequence[ReleaseSource | str | Path] | None) -> tuple[ReleaseSource, ...]: + """Validate sources and open registry URLs and paths. + + Args: + sources: The value of ``sources``, other than ``None``. + + Returns: + The sources, in order. + + Raises: + TimeFValidationError: If sources are not a nonempty sequence of valid sources, registry URLs, or paths. + TimeNetRegistryError: If a registry URL or path is blank or cannot be opened. + """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry + if isinstance(sources, (str, os.PathLike)): + shown = repr(str(sources)) + raise TimeFValidationError( + f"sources takes a list of sources. Pass sources=[{shown}], or " + f"sources=[{shown}, InstalledConnectors()] to build what the path lacks" + ) + if not isinstance(sources, Sequence): + raise TimeFValidationError(f"sources takes a list of sources. Pass sources=[{type(sources).__name__}(...)]") + if not sources: + raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") + opened: list[ReleaseSource] = [] + for index, item in enumerate(sources): + if isinstance(item, (str, os.PathLike)): + opened.append(open_registry(item if isinstance(item, str) else Path(item), label=f"sources[{index}]")) + elif isinstance(item, (BaseRegistry, InstalledConnectors)): + opened.append(item) + else: + raise TimeFValidationError(f"sources[{index}] is not a source, a registry URL or a path: {item!r}") + return tuple(opened) + + class TimeNet: - """Browse, download, and read datasets from a registry.""" + """Browse, download, and read TimeNet datasets.""" - def __init__( - self, registry: str | Path | BaseRegistry | None = None, *, storage_path: str | Path | None = None - ) -> None: - """Open a client for a registry. + def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) -> None: + """Create a client that reads releases through an ordered list of sources. - The client selects the registry in this order: the ``registry`` argument, then - ``$TIMENET_REGISTRY``, then the hosted TimeNet registry (``timenet://``). + Without a version, a call takes the newest version of the first source that serves the + dataset. An exact version comes from the first source that serves it, and ``"latest"`` is + the highest version that any source provides. Only "not found" and "not served" fall + through to the next source. Any other error stops the call. See :mod:`timenet.sources`. + + A source is a registry or ``InstalledConnectors()``. A string or a + :class:`~pathlib.Path` is a registry: a URL (``"timenet://"``, ``"https://..."``, + ``"s3://..."``), a ``file://`` URI, or a local path. A local path opens a writable + :class:`~timenet.registry.LocalRegistry`. Builds land in the first local registry of the + list that is not ``read_only=True``. The client only reads any later local registry. + ``InstalledConnectors()`` in the list is the consent to builds, so a list with it needs a + writable local registry. Args: - registry: A registry instance, URL, ``file://`` URI, or local path. - storage_path: The directory for cached downloads. The default is - ``$TIMENET_STORAGE``, or ``/storage`` if that variable is not set. + sources: The sources, in order. ``None`` means :func:`default_sources`. + + Raises: + TimeFValidationError: If ``sources`` is one string or path instead of a list, or is + empty. Also if it holds an item that is not a source, a string, or a path, a string + that is not a registry URL, or ``InstalledConnectors()`` without a writable local + registry. """ - given_registry = registry if isinstance(registry, BaseRegistry) else None - cfg = settings( - registry=None if given_registry is not None or registry is None else str(registry), - storage=storage_path, + self._sources = tuple(default_sources()) if sources is None else _open_sources(sources) + self._store = next( + (source for source in self._sources if isinstance(source, LocalRegistry) and not source.read_only), None ) - if given_registry is not None: - self._registry = given_registry - elif cfg.registry is not None: - self._registry = open_registry(cfg.registry, cache_dir=cfg.storage_dir) - else: - self._registry = open_registry("timenet://", cache_dir=cfg.storage_dir) - self._storage = cfg.storage_dir + if self._store is None and any(isinstance(source, InstalledConnectors) for source in self._sources): + raise TimeFValidationError( + "InstalledConnectors() needs a writable local registry to build into. " + "Add a LocalRegistry without read_only=True to the sources" + ) def list(self) -> _Metadatas: - """Return the metadata of every dataset in the registry. + """Return the metadata of every dataset in one registry of the sources. + + The registry is the first registry of the sources that is not the local registry of the + builds, else that local registry. Returns: One :class:`~timenet.types.DatasetMetadata` per dataset. """ - return self._registry.list_datasets() + return self._catalog().list_datasets() def get(self, dataset_id: str, version: str | None = None) -> Manifest: - """Return a dataset's manifest. + """Return a dataset's manifest from the registry that :meth:`list` asks. Args: dataset_id: The dataset id. @@ -107,7 +194,7 @@ def get(self, dataset_id: str, version: str | None = None) -> Manifest: The dataset's manifest. """ dataset_id, version = _resolve_ref(dataset_id, version) - return self._registry.get_manifest(dataset_id, version) + return self._catalog().get_manifest(dataset_id, version) def search( # ruff: ignore[too-many-arguments] self, @@ -121,7 +208,9 @@ def search( # ruff: ignore[too-many-arguments] tag: _OrList[str] = None, limit: int = 100, ) -> _Metadatas: - """Search the registry. This method mirrors :meth:`~timenet.registry.BaseRegistry.search`. + """Search the registry that :meth:`list` asks. + + This method mirrors :meth:`~timenet.registry.BaseRegistry.search`. Args: query: Free text terms for the name, description, and tags. @@ -136,7 +225,7 @@ def search( # ruff: ignore[too-many-arguments] Returns: The matching dataset metadata. """ - return self._registry.search( + return self._catalog().search( query=query, domain=domain, task=task, @@ -155,9 +244,11 @@ def download( force: bool = False, progress_cb: ProgressCallback | None = None, ) -> Path: - """Fetch a version and each of its payload sources into local storage. + """Fetch a version and each of its payload sources from the registry that :meth:`list` asks. - Build-only parents remain recorded in the manifest and are not downloaded. + The copy lands in the storage directory: ``$TIMENET_STORAGE``, or ``/storage`` + if that variable is not set. Build-only parents remain recorded in the manifest and are not + downloaded. Args: dataset_id: The dataset id. @@ -168,26 +259,20 @@ def download( Returns: The local ``///`` directory. """ + # Glue: the interim download() copies from one registry into the storage directory, as before + # the list of sources. A download into the store replaces it. dataset_id, version = _resolve_ref(dataset_id, version) + registry = self._catalog() self._reject_unhosted_access(dataset_id, version) - manifest = self._registry.get_manifest(dataset_id, version) + manifest = registry.get_manifest(dataset_id, version) resolved = str(manifest.metadata.dataset_version) - target = release_dir(self._storage, DatasetRef.parse(f"{dataset_id}@{resolved}")) + target = release_dir(settings().storage_dir, DatasetRef.parse(f"{dataset_id}@{resolved}")) # The registry owns the fetch: the base implementation stages each file and swaps atomically, # and a remote registry overrides it to resolve and stream every file in parallel. - self._registry.download_version( - dataset_id, resolved, target, force=force, manifest=manifest, progress_cb=progress_cb - ) + registry.download_version(dataset_id, resolved, target, force=force, manifest=manifest, progress_cb=progress_cb) return target - def open( - self, - dataset_id: str, - version: str | None = None, - *, - auto_build: bool = True, - fetch_from: str | BaseRegistry | None = None, - ) -> TimeFReader: + def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: """Open a dataset version for lazy reads, where it is. Opening reads the version's manifest, but neither its catalog nor any value. The reader's @@ -196,15 +281,14 @@ def open( from their manifests upfront and read payload ranges on demand, reusing complete cached releases. - The call reads each release where it is, and copies nothing. A local client asks its own - registry first, then the registry that ``fetch_from`` names, then the installed connectors - if ``auto_build`` is set. Without a version, the call reads the newest version of the first - of them that serves the dataset. An exact version comes from the first of them that serves - it, and ``"latest"`` is the highest version that any of them provides. Each payload source - of the version resolves on its own, at its exact version. If the call builds the version, - the build also needs the declared parents of the version and their payload sources. + The call reads each release where it is, and copies nothing. Without a version, it reads the + newest version of the first source that serves the dataset. An exact version comes from the + first source that serves it, and ``"latest"`` is the highest version of any source. Each + payload source of the version resolves on its own, at its exact version. If the call builds + the version, the build also needs the declared parents of the version and their payload + sources. - If only a connector provides a release, the call builds it into the local registry first. + If only ``InstalledConnectors()`` provides a release, the call builds it into the local registry first. The call builds a payload source with its own connector, at the exact version of the source. The build gets the pins of the version. The build runs in staging. It publishes the source only if it reproduces the pinned files. @@ -212,67 +296,49 @@ def open( A local copy is damaged if its manifest is present but a file is missing or has another size. The call checks each release that it opens in a local registry before it reads or builds anything. It never reads past a damaged copy to a later source, and it never builds - a release over a damaged copy. Against a remote registry, ``open`` never runs - connector code. If a remote client gets ``fetch_from``, ``open`` raises an error. + a release over a damaged copy. Args: dataset_id: The dataset id. version: An exact version, ``"latest"``, or ``None`` for no version. - auto_build: Build a release that only an installed connector provides. Set false to - raise instead. - fetch_from: A registry, or its URI, that a local client reads a release from if its own - registry does not provide it. Returns: A reader for the version. Raises: - TimeNetAccessError: If a registry that is not local lists the version or a payload - source but does not serve its files, and no later source provides it. - TimeNetDatasetNotFoundError: If no source provides the version. + TimeNetAccessError: If a registry lists the version or a payload source but does not + serve its files, and no later source provides it. + TimeNetDatasetNotFoundError: If no source provides the version. The message says which + source to add. TimeNetVersionUnavailableError: If no source provides the version, and the installed connector declares another version. The build does not start then. TimeNetInvalidCardError: If the card of a connector cannot be read. TimeNetMissingParentError: If no source provides a payload source, or a release that a - build needs. The message names each keyword that the call did not use. + build needs. The message says which source to add. TimeNetFileError: If a release that the call opens in a local registry has a missing file or a file of another size. TimeFFormatError: If a payload source differs from the files that the version pins, for example because its build does not reproduce them. TimeNetBuildError: If a build runs but fails. - TimeFValidationError: If a remote client gets ``fetch_from``. """ # ruff: ignore[docstring-extraneous-exception] (raised by read_release and the builder) dataset_id, version = _resolve_ref(dataset_id, version) - sources, store = self._read_sources(auto_build=auto_build, fetch_from=fetch_from) - # TODO(adrian): Remove temporary glue which maps fetch_from and auto_build onto hints for - # a local client until the client takes a list of sources. try: - return TimeFReader(read_release(dataset_id, version, sources=sources, store=store)) + return TimeFReader(read_release(dataset_id, version, sources=self._sources, store=self._store)) except TimeNetAccessError as error: - if store is None: - error.hint = _unhosted_hint(dataset_id) - elif not auto_build: - error.hint = "Build it from its connector with auto_build=True." + error.hint = _access_hint(dataset_id, self._sources) raise except TimeNetMissingParentError as error: - if store is not None: - error.hint = _missing_hint(auto_build=auto_build, fetch_from=fetch_from) + error.hint = _not_provided_hint(self._sources) raise except TimeNetDatasetNotFoundError as error: - hint = None if store is None else _missing_hint(auto_build=auto_build, fetch_from=fetch_from) # A subclass, such as TimeNetVersionUnavailableError, names its own fix. - if hint is None or type(error) is not TimeNetDatasetNotFoundError: + if type(error) is not TimeNetDatasetNotFoundError: raise - raise TimeNetDatasetNotFoundError(f"{str(error).rstrip('.')}. {hint}") from error + raise TimeNetDatasetNotFoundError( + f"{str(error).rstrip('.')}. {_not_provided_hint(self._sources)}" + ) from error - def load( - self, - dataset_id: str, - version: str | None = None, - *, - auto_build: bool = True, - fetch_from: str | BaseRegistry | None = None, - ) -> TimeFDataset: + def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: """Read a whole dataset version into memory, with every stored value loaded. The dataset needs no storage afterwards: its reader is closed before it is returned. For a @@ -281,17 +347,13 @@ def load( Args: dataset_id: The dataset id. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. - auto_build: Build a release that only an installed connector provides. Set false to - raise instead. - fetch_from: A registry, or its URI, that a local client reads a release from if its own - registry does not provide it. Returns: The dataset with every signal array, time offset, and text value loaded. Raises: - TimeNetAccessError: If a registry that is not local lists the version or a payload - source but does not serve its files, and no later source provides it. + TimeNetAccessError: If a registry lists the version or a payload source but does not + serve its files, and no later source provides it. TimeNetDatasetNotFoundError: If no source provides the version. TimeNetVersionUnavailableError: If no source provides the version, and the installed connector declares another version. The build does not start then. @@ -302,40 +364,29 @@ def load( file or a file of another size. TimeFFormatError: If a payload source differs from the files that the version pins. TimeNetBuildError: If a build runs but fails. - TimeFValidationError: If a remote client gets ``fetch_from``. """ # ruff: ignore[docstring-extraneous-exception] (all come from open()) - with self.open(dataset_id, version, auto_build=auto_build, fetch_from=fetch_from) as reader: + with self.open(dataset_id, version) as reader: return reader.read(load=LoadPlan.ALL_EAGER) - def _read_sources( - self, *, auto_build: bool, fetch_from: str | BaseRegistry | None - ) -> tuple[_Sources, LocalRegistry | None]: - """Return the ordered sources for a read and the local registry for builds. - - Remote reads use only the configured registry. + def _catalog(self) -> BaseRegistry: + """Return the registry that :meth:`list`, :meth:`get`, :meth:`search`, and :meth:`download` ask. Returns: - The source list, and the local registry for builds or ``None`` for a remote client. + The first registry of the sources that is not the local registry of the builds, else + that local registry. Raises: - TimeFValidationError: If ``fetch_from`` is set for a remote registry. + TimeFValidationError: If the sources hold no registry. """ - registry = self._registry - if not isinstance(registry, LocalRegistry): - if fetch_from is not None: - raise TimeFValidationError( - "fetch_from adds a registry to the reads of a local registry; a remote client reads from its " - "own registry" - ) - return [registry], None - source = open_registry(fetch_from) if isinstance(fetch_from, str) else fetch_from - sources: _Sources = [registry] - if source is not None: - sources.append(source) - if auto_build: - sources.append(InstalledConnectors()) - return sources, registry - + # Glue: until list() and search() ask every source, these methods ask one registry. This + # registry gives the results of the client of one registry, from before the list of sources. + registries = [source for source in self._sources if isinstance(source, BaseRegistry)] + registry = next((registry for registry in registries if registry is not self._store), self._store) + if registry is None: + raise TimeFValidationError("list(), get(), search() and download() need a registry in the sources") + return registry + + # Glue: only the interim download() uses this check. A later change replaces download(). def _reject_unhosted_access(self, dataset_id: str, version: str | None) -> None: """Enforce the registry's open access requirement. @@ -346,23 +397,18 @@ def _reject_unhosted_access(self, dataset_id: str, version: str | None) -> None: Raises: TimeNetAccessError: If the dataset lacks open access and the registry requires it. """ - if not self._registry.requires_open_access: + registry = self._catalog() + if not registry.requires_open_access: return try: - manifest = self._registry.get_manifest(dataset_id, version) + manifest = registry.get_manifest(dataset_id, version) require_open_access(manifest.metadata, manifest.lineage) except TimeNetAccessError as error: error.hint = _unhosted_hint(dataset_id) raise def open_torch( - self, - dataset_id: str, - version: str | None = None, - *, - auto_build: bool = True, - fetch_from: str | BaseRegistry | None = None, - load: LoadPlan = LoadPlan.ALL_EAGER, + self, dataset_id: str, version: str | None = None, *, load: LoadPlan = LoadPlan.ALL_EAGER ) -> TimeFTorchDataset: """Open a dataset version as a read-only PyTorch ``Dataset`` that reads it a batch at a time. @@ -374,10 +420,6 @@ def open_torch( Args: dataset_id: The dataset id. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. - auto_build: Build a release that only an installed connector provides. Set false to - raise instead. - fetch_from: A registry, or its URI, that a local client reads a release from if its own - registry does not provide it. load: The plan that each batch loads with. A text-only plan, for example, skips the arrays of candidate and target Records. @@ -387,32 +429,47 @@ def open_torch( """ from timenet.torch import TimeFTorchDataset # ruff: ignore[import-outside-top-level] - return TimeFTorchDataset.from_reader( - self.open(dataset_id, version, auto_build=auto_build, fetch_from=fetch_from), load=load - ) + return TimeFTorchDataset.from_reader(self.open(dataset_id, version), load=load) -def _unhosted_hint(dataset_id: str) -> str: - """Return instructions for building a dataset locally. +def _not_provided_hint(sources: Sequence[ReleaseSource]) -> str: + """Tell the caller which source can provide a release that no source provides. + + Args: + sources: The sources of the client. Returns: - A sentence with the dataset's build command. + The hint as a full sentence. It names ``InstalledConnectors()`` only if the sources lack it. """ - return f"TimeNet does not host its data, so build it locally with `timenet-build build {dataset_id}`." + if any(isinstance(source, InstalledConnectors) for source in sources): + return "Add a registry that holds it to the sources." + return "Add a registry that holds it to the sources, or add InstalledConnectors() to build it." + + +def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: + """Tell the caller how to get a release that a registry lists but does not serve. + + Args: + dataset_id: The bare dataset id. + sources: The sources of the client. + Returns: + The hint as full sentences. It asks for ``InstalledConnectors()`` only if the sources lack it. + """ + build = f"`timenet-build build {dataset_id}`" + if any(isinstance(source, InstalledConnectors) for source in sources): + return ( + "No source serves its files, and no installed connector builds this release. " + f"Install its connector package and build it locally with {build}." + ) + return f"No source serves its files. Add InstalledConnectors() to the sources, or build it locally with {build}." -def _missing_hint(*, auto_build: bool, fetch_from: str | BaseRegistry | None) -> str | None: - """Suggest options for reading or building a missing release. + +# Glue: only the interim download() uses this hint. A later change replaces download(). +def _unhosted_hint(dataset_id: str) -> str: + """Tell the caller how to get a dataset whose data the registry of :meth:`TimeNet.download` does not serve. Returns: - A sentence with unused options, or ``None`` if both options are already enabled. + A sentence with the dataset's build command. """ - ways = [] - if fetch_from is None: - ways.append("read it from a registry that holds it with fetch_from=") - if not auto_build: - ways.append("build it from its connector with auto_build=True") - if not ways: - return None - hint = " or ".join(ways) - return f"{hint[0].upper()}{hint[1:]}." + return f"TimeNet does not host its data, so build it locally with `timenet-build build {dataset_id}`." diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index 65ed5e381..6ac98f8e4 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -64,10 +64,13 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = options: Build settings, including the progress callback. Raises: + TimeNetRegistryError: If the plan copies or builds a release and ``store`` is read-only. TimeNetVersionConflictError: If a copied release conflicts with the files in ``store``. TimeFFormatError: If a build changes files pinned by other releases. TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the copy and the backends + if any(not isinstance(step, ReadStep) for step in plan.steps): + store.check_writable("the copies and builds of a plan") options = options or BuildOptions() for step in plan.steps: match step: diff --git a/packages/timenet/src/timenet/registry/__init__.py b/packages/timenet/src/timenet/registry/__init__.py index bc13fea69..4ebfb5825 100644 --- a/packages/timenet/src/timenet/registry/__init__.py +++ b/packages/timenet/src/timenet/registry/__init__.py @@ -8,7 +8,6 @@ default_registry_path, local_registry_path, open_registry, - open_writable_registry, ) from timenet.registry.local import LocalRegistry from timenet.registry.remote import RemoteRegistry @@ -30,5 +29,4 @@ "local_registry_path", "open_local_version", "open_registry", - "open_writable_registry", ] diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index db7ad0b6f..f29131ebb 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -40,12 +40,14 @@ def copy_release( progress: Callback for ``StepStarted`` and ``StepFinished`` events for each copied release. Raises: + TimeNetRegistryError: If ``dest`` is read-only. TimeNetVersionConflictError: If ``dest`` holds other files or another card for a release being copied. TimeNetAccessError: If a release fails an open access requirement of ``dest`` or ``source``. TimeNetDatasetNotFoundError: If ``source`` does not hold the release. TimeNetMissingParentError: If ``source`` lacks a payload source and ``dest`` holds no complete copy of it. TimeFFormatError: If a manifest names a different release or a pin fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the helpers + dest.check_writable(f"a copy of {reference}") manifests = _manifests(reference, source, dest, with_payload_sources=with_payload_sources) # Check every release before anything is copied, so that a refusal leaves dest unchanged. A payload # source that only dest holds is complete there, so it is never pending. diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index aa4c0364f..81e7be8d4 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -13,7 +13,7 @@ from typing import BinaryIO, TypeVar from timenet._fs import staged_directory -from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError +from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError, TimeNetRegistryError from timenet.format.checksums import file_checksum from timenet.format.constants import MANIFEST_FILE from timenet.manifest import FilePart, Manifest @@ -41,9 +41,30 @@ class BaseRegistry(ABC): sources. :func:`~timenet.registry.copy_release` copies a release with its payload sources. """ + _read_only: bool = False + def __str__(self) -> str: return type(self).__name__ + @property + def read_only(self) -> bool: + """Whether this registry refuses every write, set with ``read_only=True``.""" + return self._read_only + + def check_writable(self, write: str) -> None: + """Refuse a write if this registry is read-only. + + Args: + write: The write to refuse, for the message, such as ``"put_dataset()"``. + + Raises: + TimeNetRegistryError: If this registry is read-only. + """ + if self._read_only: + raise TimeNetRegistryError( + f"{self} has read_only=True, so it refuses {write}. Open it without read_only=True to write to it." + ) + @property @abstractmethod def local_root(self) -> Path: diff --git a/packages/timenet/src/timenet/registry/factory.py b/packages/timenet/src/timenet/registry/factory.py index 506fef9f9..328565b25 100644 --- a/packages/timenet/src/timenet/registry/factory.py +++ b/packages/timenet/src/timenet/registry/factory.py @@ -6,7 +6,6 @@ from timenet.config import settings from timenet.errors import TimeNetRegistryError -from timenet.registry.base import BaseRegistry from timenet.registry.local import LocalRegistry from timenet.registry.remote import RemoteRegistry from timenet.registry.s3 import S3Registry @@ -19,7 +18,10 @@ _REMOTE_SCHEMES = ("timenet://", "http://", "https://", "s3://") -def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None) -> BaseRegistry: +_SCHEMES = "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + + +def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None, label: str = "registry") -> WritableRegistry: """Open a registry from a URI or path. The scheme selects the backend. ``http(s)://`` and ``timenet://`` open a :class:`RemoteRegistry`, @@ -30,14 +32,15 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None) -> Ba uri: A URL, ``timenet://`` / ``s3://`` / ``file://`` URI, or local path. cache_dir: Where a remote backend caches downloads. The local backend ignores it. Defaults to the configured storage directory when ``None``. + label: The name of the value in an error message, such as ``sources[0]``. Returns: The matching registry backend. Raises: - ValueError: If ``uri`` carries a scheme no backend handles, if a ``timenet://`` URI carries a - path, or if ``uri`` is a ``file://`` URI with a host component (the host would be dropped - without notice). + TimeNetRegistryError: If ``uri`` is blank or carries a scheme no backend handles, if a + ``timenet://`` URI carries a path, or if ``uri`` is a ``file://`` URI with a host component + (the host would be dropped without notice) or without a path. """ text = str(uri) if text.startswith("timenet://"): @@ -45,68 +48,44 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None) -> Ba # passed to the client methods, not folded into the registry URI). Reject a stray remainder # rather than drop it silently, so a mistyped URI fails loudly instead of hitting the default host. if text[len("timenet://") :].strip("/"): - raise ValueError(f"timenet:// takes no path; got {text!r}") + raise TimeNetRegistryError(f"{label} {text!r}: timenet:// takes no path") return RemoteRegistry(TIMENET_REGISTRY_URL, cache_dir=cache_dir) if text.startswith(("http://", "https://")): return RemoteRegistry(text, cache_dir=cache_dir) if text.startswith("s3://"): return S3Registry(text, cache_dir=cache_dir) - if text.startswith("file://"): - parsed = urlparse(text) - if parsed.netloc: - raise ValueError(f"file:// registry URI must be absolute (three slashes), got {text!r}") - return LocalRegistry(Path(url2pathname(parsed.path)).expanduser()) - if "://" in text: - raise ValueError(f"unsupported registry scheme in {text!r}") - return LocalRegistry(Path(text).expanduser()) - + return LocalRegistry(local_registry_path(text, label=label)) -def open_writable_registry(uri: str | Path) -> WritableRegistry: - """Open a writable registry from a URI or local path. - Args: - uri: A URL, ``timenet://`` / ``s3://`` / ``file://`` URI, or local path. - - Returns: - The writable registry for the URI or path. - - Raises: - ValueError: If the registry does not support writing. - """ - registry = open_registry(uri) - if not isinstance(registry, WritableRegistry): - raise ValueError(f"registry {uri!r} is not writable") - return registry - - -def local_registry_path(uri: str | Path) -> Path: +def local_registry_path(uri: str | Path, *, label: str = "registry") -> Path: """Resolve a registry URI to the local directory it names, for build to write into. - Every backend is a :class:`WritableRegistry`, so :func:`open_writable_registry` cannot separate a - directory the engine can write to from a remote stub. This function can. - Args: uri: A ``file://`` URI or local path. + label: The name of the value in an error message, such as ``sources[0]``. Returns: The local directory the URI names, with ``~`` expanded. Raises: - TimeNetRegistryError: If the URI names a remote backend or carries an unsupported scheme, which - build cannot write to. + TimeNetRegistryError: If the value is blank, names a remote backend, or is not a valid local + registry URI. """ text = str(uri) + if not text.strip(): + # Path("") is the current directory, which a blank value never means. + raise TimeNetRegistryError(f"{label} is blank; give a registry URL or a local path") if text.startswith(_REMOTE_SCHEMES): - raise TimeNetRegistryError(f"registry {text!r} is remote; build writes to a local directory") + raise TimeNetRegistryError(f"{label} {text!r} is remote; build writes to a local directory") if text.startswith("file://"): parsed = urlparse(text) if parsed.netloc: - raise TimeNetRegistryError(f"file:// registry URI must be absolute (three slashes), got {text!r}") + raise TimeNetRegistryError(f"{label} {text!r} must be an absolute file:// URI (three slashes). {_SCHEMES}") if not parsed.path: - raise TimeNetRegistryError("file:// registry URI must name an absolute path") + raise TimeNetRegistryError(f"{label} {text!r} must name an absolute path") return Path(url2pathname(parsed.path)).expanduser() if "://" in text: - raise TimeNetRegistryError(f"unsupported registry scheme in {text!r}") + raise TimeNetRegistryError(f"{label} {text!r} has an unsupported registry scheme. {_SCHEMES}") return Path(text).expanduser() diff --git a/packages/timenet/src/timenet/registry/lazy_read.py b/packages/timenet/src/timenet/registry/lazy_read.py index 9368a596e..32654c6f6 100644 --- a/packages/timenet/src/timenet/registry/lazy_read.py +++ b/packages/timenet/src/timenet/registry/lazy_read.py @@ -52,8 +52,10 @@ def put_dataset( A reference to the stored release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeFValidationError: If ``force`` is set. - """ + """ # ruff: ignore[docstring-extraneous-exception] - raised by check_writable + self.check_writable("put_dataset()") if force: raise TimeFValidationError( f"force=True needs a local registry: {self} never writes a published version again" diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index ab4e8fb43..2e6d7273e 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -26,13 +26,15 @@ class LocalRegistry(WritableRegistry): """A registry that reads and writes datasets in a local directory.""" - def __init__(self, root: Path) -> None: + def __init__(self, root: str | Path, *, read_only: bool = False) -> None: """Open a local registry rooted at a directory. Args: root: The directory containing ``//`` layouts. This expands ``~``. + read_only: Refuse every write to this registry. """ self._root = Path(root).expanduser() + self._read_only = read_only # (dataset_id, version) -> (manifest mtime, parsed manifest). This avoids re-parsing on repeat # reads, for example list_datasets then search's schema filter. An mtime change invalidates it. self._manifest_cache: dict[tuple[str, str], tuple[float, Manifest]] = {} @@ -200,9 +202,11 @@ def put_dataset( A reference to the stored release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeNetDatasetNotFoundError: If a release that stores reused data is missing from this registry. TimeFFormatError: If a source release or the new output fails a file pin check. """ # ruff: ignore[docstring-extraneous-exception] - the writer checks payload sources before it publishes + self.check_writable("put_dataset()") if dataset.schema is None: dataset.derive_schema() version = str(dataset.metadata.dataset_version) @@ -236,10 +240,12 @@ def put_release(self, from_directory: Path) -> DatasetRef: A reference to the published release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. OSError: If copying or renaming fails, with the backup path if restoring the old directory also fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release and publish_directory + self.check_writable("put_release()") manifest = self._read_release(from_directory) reference = DatasetRef(dataset_id=manifest.dataset_id, version=manifest.metadata.dataset_version) with staged_directory(release_dir(self._root, reference)) as staging: diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index ad093591c..692e3d8a7 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -41,6 +41,7 @@ def __init__( token: str | None = None, cache_dir: str | Path | None = None, transport: httpx.BaseTransport | None = None, + read_only: bool = False, ) -> None: """Open a remote registry. @@ -49,8 +50,10 @@ def __init__( token: Bearer token. Defaults to ``$TIMENET_TOKEN`` (``None`` is anonymous). cache_dir: Where downloads are cached; defaults to the configured storage directory. transport: An httpx transport for testing; ``None`` uses the network. + read_only: Refuse every write to this registry. """ cfg = settings() + self._read_only = read_only self._base_url = base_url self._http = RegistryHttpClient(base_url, token=token if token is not None else cfg.token, transport=transport) self._cache_dir = Path(cache_dir) if cache_dir is not None else cfg.storage_dir @@ -175,8 +178,10 @@ def put_release(self, from_directory: Path) -> DatasetRef: Raises: TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. - TimeNetRegistryError: If the service skips a listed file or requests a missing file. + TimeNetRegistryError: If this registry is read-only, or the service skips a listed file or + requests a missing file. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release + self.check_writable("put_release()") manifest = self._read_release(from_directory) dataset_id = manifest.dataset_id version = str(manifest.metadata.dataset_version) diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 4bdf97d5f..1ea8a46fe 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -42,12 +42,13 @@ class S3Registry(LazyReadRegistry): _staging_prefix = "timenet-s3-" - def __init__(self, uri: str, *, cache_dir: str | Path | None = None) -> None: + def __init__(self, uri: str, *, cache_dir: str | Path | None = None, read_only: bool = False) -> None: """Open an S3-backed registry. Args: uri: The bucket or prefix root, for example ``s3://my-bucket/registry``. cache_dir: Where downloads are cached; defaults to the configured storage directory. + read_only: Refuse every write to this registry. Raises: TimeNetRegistryError: If ``uri`` is not an ``s3://bucket[/prefix]`` URL. @@ -58,6 +59,7 @@ def __init__(self, uri: str, *, cache_dir: str | Path | None = None) -> None: self._bucket = parsed.netloc self._prefix = parsed.path.strip("/") self._cache_dir = Path(cache_dir) if cache_dir is not None else settings().storage_dir + self._read_only = read_only def __str__(self) -> str: return f"S3 registry {self._display_root()}" @@ -168,9 +170,11 @@ def put_release(self, from_directory: Path) -> DatasetRef: A reference to the published release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release + self.check_writable("put_release()") manifest = self._read_release(from_directory) client = self._client() final_base = self._key(manifest.dataset_id, str(manifest.metadata.dataset_version)) diff --git a/packages/timenet/tests/coordinator/test_build.py b/packages/timenet/tests/coordinator/test_build.py index 7f8e95ace..c957ffd8c 100644 --- a/packages/timenet/tests/coordinator/test_build.py +++ b/packages/timenet/tests/coordinator/test_build.py @@ -5,9 +5,9 @@ from timenet import builders from timenet.builders import BuildOptions from timenet.coordinator.build import plan_release_build, run_plan -from timenet.coordinator.plan import BuildStep, CopyStep +from timenet.coordinator.plan import BuildStep, CopyStep, Plan from timenet.dataset import TimeFDataset -from timenet.errors import TimeNetAccessError, TimeNetVersionUnavailableError +from timenet.errors import TimeNetAccessError, TimeNetRegistryError, TimeNetVersionUnavailableError from timenet.progress import StepFinished, StepStarted from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version @@ -125,6 +125,13 @@ def test_plan_release_build_rejects_a_version_the_connector_does_not_declare(sto assert (caught.value.release, caught.value.declared) == (DatasetRef.parse("a/b@2.0.0"), REF_AB.version) +def test_a_plan_that_builds_refuses_a_read_only_local_registry_before_any_step(tmp_path, fake): + store = LocalRegistry(tmp_path / "store", read_only=True) + with pytest.raises(TimeNetRegistryError, match=r"read_only=True, so it refuses the copies and builds of a plan\."): + run_plan(Plan((_build(REF_AB),)), store) + assert fake.calls == [] + + def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, store, fake_parent, fake_child): origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) diff --git a/packages/timenet/tests/reader/test_reader.py b/packages/timenet/tests/reader/test_reader.py index 473138274..e7a0559e7 100644 --- a/packages/timenet/tests/reader/test_reader.py +++ b/packages/timenet/tests/reader/test_reader.py @@ -374,7 +374,7 @@ def counting_open(*args, **kwargs): if entry_point == "reader": reader = TimeFReader(open_local_version(version_dir)) elif entry_point == "client": - reader = TimeNet(tmp_path, storage_path=tmp_path / "store").open("timenet/hello-world") + reader = TimeNet(sources=[str(tmp_path)]).open("timenet/hello-world") else: reader = TimeFReader.open(version_dir) with reader: diff --git a/packages/timenet/tests/registry/test_registry.py b/packages/timenet/tests/registry/test_registry.py index 7459a30ba..d7eeda740 100644 --- a/packages/timenet/tests/registry/test_registry.py +++ b/packages/timenet/tests/registry/test_registry.py @@ -13,11 +13,10 @@ LocalRegistry, RemoteRegistry, S3Registry, - WritableRegistry, + copy_release, local_registry_path, open_local_version, open_registry, - open_writable_registry, ) from timenet.testing import make_dataset from timenet.types import Annotation, AnswerTask, Domain, License @@ -56,7 +55,7 @@ def test_open_registry_timenet_scheme_aliases_hosted_remote(): def test_open_registry_timenet_scheme_rejects_a_path(): # A path after timenet:// used to be dropped silently; now it is rejected, so a mistyped registry # URI fails loudly instead of hitting the default host. - with pytest.raises(ValueError, match="timenet:// takes no path"): + with pytest.raises(TimeNetRegistryError, match="timenet:// takes no path"): open_registry("timenet://chengsenwang/tsqa") @@ -66,21 +65,22 @@ def test_open_registry_bare_timenet_scheme(): assert registry._base_url == TIMENET_REGISTRY_URL -def test_open_writable_registry_returns_writable(registry_root): - assert isinstance(open_writable_registry(registry_root), WritableRegistry) - - def test_open_registry_unknown_scheme_rejected(): # must not fall through to a LocalRegistry rooted at the literal "gs://bucket" string - with pytest.raises(ValueError, match="unsupported registry scheme"): + with pytest.raises(TimeNetRegistryError, match="unsupported registry scheme"): open_registry("gs://bucket/registry") def test_open_registry_file_uri_with_host_rejected(): - with pytest.raises(ValueError, match="absolute"): + with pytest.raises(TimeNetRegistryError, match="absolute"): open_registry("file://home/timo/registry") +def test_open_registry_rejects_a_blank_value_instead_of_opening_the_current_directory(): + with pytest.raises(TimeNetRegistryError, match=r"^--registry is blank; give a registry URL or a local path$"): + open_registry(" ", label="--registry") + + def test_open_registry_expands_user(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) registry = open_registry("~/registry") @@ -338,3 +338,16 @@ def test_s3_registry_has_no_catalog(): s3.list_datasets() with pytest.raises(NotImplementedError): s3.search() + + +def test_a_read_only_registry_refuses_put_dataset_and_a_copy_into_it(tmp_path): + origin = LocalRegistry(tmp_path / "origin") + release = origin.put_dataset(make_dataset()) + registry = LocalRegistry(tmp_path / "read-only", read_only=True) + refused = rf"^local registry {re.escape(str(registry.root))} has read_only=True, so it refuses " + + with pytest.raises(TimeNetRegistryError, match=refused + r"put_dataset\(\)\."): + registry.put_dataset(make_dataset()) + with pytest.raises(TimeNetRegistryError, match=refused + r"a copy of timenet/hello-world@1\.0\.0\."): + copy_release(release, origin, registry) + assert not registry.root.exists() diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 1190d58ff..b3bf47166 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -296,7 +296,7 @@ def test_load_full_round_trips(version_dir, tmp_path, entry_point): directory, manifest = version_dir with registry_server(directory) as server: registry = RemoteRegistry(server.url, cache_dir=tmp_path / "cache") - client = TimeNet(registry=registry, storage_path=tmp_path / "storage") + client = TimeNet(sources=[registry]) if entry_point == "open": with client.open(manifest.metadata.dataset_id) as reader: assert_datasets_equal(make_dataset(), reader.read()) @@ -321,7 +321,7 @@ def single_use_download(request): with registry_server(directory, serve_file=single_use_download if expire_urls else None) as server: registry = RemoteRegistry(server.url, cache_dir=tmp_path / "cache") - client = TimeNet(registry=registry, storage_path=tmp_path / "storage") + client = TimeNet(sources=[registry]) with client.open(manifest.metadata.dataset_id) as reader: # A worker must reopen its own clients, including after the main process has read values. reader.get(Signal, make_dataset().records[0].signals[0].id).to_arrow() @@ -355,7 +355,7 @@ def serve_file(request): with registry_server(directory, serve_file=serve_file) as server: remote = RemoteRegistry(server.url, token="private-token", cache_dir=tmp_path / "cache") - with TimeNet(remote).open(dataset.metadata.dataset_id, "1.0.0") as reader: + with TimeNet(sources=[remote]).open(dataset.metadata.dataset_id, "1.0.0") as reader: assert all(response.request.url.path.startswith("/api/v1/") for response in server.responses) if not support_ranges: # The get reads the catalog, which already needs range requests. @@ -484,12 +484,13 @@ def handler(request): registry.get_manifest("demo/example") -def test_remote_registry_uses_client_storage_path(tmp_path): - # A URL-configured remote registry must cache under the client's storage_path, so download() and - # load() share one cache-first directory. - client = TimeNet(registry="https://registry.example.test", storage_path=tmp_path / "store") - assert isinstance(client._registry, RemoteRegistry) - assert client._registry._cache_dir == client._storage == tmp_path / "store" +def test_a_registry_url_in_the_sources_caches_under_the_storage_directory(tmp_path, monkeypatch): + # A remote registry that the sources name by URL caches under the storage directory, where + # download() copies to, so download() and load() share one cache-first directory. + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "store")) + (registry,) = TimeNet(sources=["https://registry.example.test"])._sources + assert isinstance(registry, RemoteRegistry) + assert registry._cache_dir == tmp_path / "store" def test_put_dataset_without_write_token_is_rejected(tmp_path): @@ -538,14 +539,14 @@ def handler(request): cache = tmp_path / "cache" remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=cache) with range_server(origin) as blob_url: - with TimeNet(remote).open("test/child", "1.0.0") as reader: + with TimeNet(sources=[remote]).open("test/child", "1.0.0") as reader: assert all("/download/" not in path for path in requests) assert reader.get(Signal, signal_id).to_arrow().equals(expected) assert any(parent_id + "/1.0.0/download/" in path for path in requests) assert not any(parent_id + "/1.0.0/download/control.duckdb" in path for path in requests) assert not (cache / parent_id / "1.0.0").exists() remote.download_version("test/child", "1.0.0", cache / "test/child/1.0.0") - with TimeNet(remote).open("test/child", "1.0.0") as reader: + with TimeNet(sources=[remote]).open("test/child", "1.0.0") as reader: assert reader.get(Signal, signal_id).to_arrow().equals(expected) assert not any("test/build-only" in path for path in requests) assert (cache / parent_id / "1.0.0/control.duckdb").exists() @@ -578,16 +579,19 @@ def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: @pytest.mark.parametrize("page", ["own", "ancestor"]) -def test_a_local_client_checks_access_before_it_reads_from_a_remote_registry(tmp_path, page): +def test_a_read_checks_access_before_it_reads_from_a_remote_registry(tmp_path, page): dataset_id, release = _gated_release(tmp_path / "origin", page) transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") with pytest.raises(TimeNetAccessError) as caught: - TimeNet(tmp_path / "local").open(dataset_id, auto_build=False, fetch_from=remote) + TimeNet(sources=[LocalRegistry(tmp_path / "local"), remote]).open(dataset_id) assert caught.value.requirements == (AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua"),) - assert caught.value.hint == "Build it from its connector with auto_build=True." + assert caught.value.hint == ( + "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " + f"`timenet-build build {dataset_id}`." + ) assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) assert not (tmp_path / "local").exists() diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index 96f2f30a2..4797ab5bc 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -51,7 +51,7 @@ def test_put_dataset_and_remote_load_round_trip(s3_root, tmp_path, backend, entr registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") reference = registry.put_dataset(make_dataset(), values_backend=backend) assert reference == DatasetRef.parse(f"{make_dataset().metadata.dataset_id}@1.0.0") - client = TimeNet(registry=registry, storage_path=tmp_path / "cache") + client = TimeNet(sources=[registry]) if entry_point == "open": with client.open(make_dataset().metadata.dataset_id) as reader: # The reader reads the release from S3, not from a downloaded copy. @@ -99,11 +99,12 @@ def test_objects_live_under_a_datasets_prefix(s3_root, tmp_path): assert all(key.startswith("registry/datasets/") for key in keys) -def test_download_then_load_serves_from_cache(s3_root, tmp_path): +def test_download_then_load_serves_from_cache(s3_root, tmp_path, monkeypatch): registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") dataset_id = make_dataset().metadata.dataset_id registry.put_dataset(make_dataset()) - client = TimeNet(registry=registry, storage_path=tmp_path / "cache") + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "cache")) + client = TimeNet(sources=[registry]) path = client.download(dataset_id) # boto3 GetObject per file, into the cache assert (path / "manifest.json").exists() handle = registry.open_release(DatasetRef.parse(f"{dataset_id}@1.0.0")) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index d972b8c3a..5453f18f5 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -81,23 +81,23 @@ def test_cache_clear_all_removes_everything(home): def test_list(registry_root): assert runner.invoke(app, ["list", "--registry", str(registry_root)]).exit_code == 0 - assert any(d.dataset_id == "timenet/hello-world" for d in TimeNet(str(registry_root)).list()) + assert any(d.dataset_id == "timenet/hello-world" for d in TimeNet(sources=[str(registry_root)]).list()) def test_search_by_domain(registry_root): assert runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "general"]).exit_code == 0 - results = TimeNet(str(registry_root)).search(domain=[Domain("general")]) + results = TimeNet(sources=[str(registry_root)]).search(domain=[Domain("general")]) assert any(d.dataset_id == "timenet/hello-world" for d in results) def test_search_no_match(registry_root): assert runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "cardiology"]).exit_code == 0 - assert TimeNet(str(registry_root)).search(domain=[Domain("cardiology")]) == [] + assert TimeNet(sources=[str(registry_root)]).search(domain=[Domain("cardiology")]) == [] def test_info(registry_root): assert runner.invoke(app, ["info", "timenet/hello-world", "--registry", str(registry_root)]).exit_code == 0 - assert TimeNet(str(registry_root)).get("timenet/hello-world").dataset_id == "timenet/hello-world" + assert TimeNet(sources=[str(registry_root)]).get("timenet/hello-world").dataset_id == "timenet/hello-world" def _write(root, dataset): @@ -148,12 +148,11 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): assert cells[start + 3][0] == "domains" -def test_download(registry_root, tmp_path): - result = runner.invoke( - app, - ["download", "timenet/hello-world", "--registry", str(registry_root), "--storage", str(tmp_path / "store")], - ) - assert result.exit_code == 0 +def test_download(registry_root, tmp_path, monkeypatch): + monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "store")) + result = runner.invoke(app, ["download", "timenet/hello-world", "--registry", str(registry_root)]) + assert result.exit_code == 0, result.output assert (tmp_path / "store" / "timenet/hello-world" / "1.0.0" / "manifest.json").exists() diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 955e02556..a88146473 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -8,7 +8,7 @@ import pytest from timenet import builders as builders_module -from timenet.client import TimeNet +from timenet.client import TimeNet, default_sources from timenet.connectors import BaseConnector from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset from timenet.engine import open_parents, run_pipeline @@ -17,10 +17,10 @@ TimeNetAccessError, TimeNetDatasetNotFoundError, TimeNetMissingParentError, + TimeNetRegistryError, TimeNetVersionUnavailableError, ) from timenet.format.stored_values import StoredText -from timenet.manifest import Manifest from timenet.manifest.files import FilePart from timenet.reader import Loading, LoadPlan, TimeFReader from timenet.registry import TIMENET_REGISTRY_URL, LocalRegistry, RemoteRegistry, open_local_version @@ -61,42 +61,36 @@ def build(self, release, store, *, force, pinned_by, options): store.put_dataset(dataset, force=force, values_backend=self.BACKEND, pinned_by=pinned_by or None) -def test_a_local_child_reads_its_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): - monkeypatch.setattr(builders_module, "entry_points", lambda group: []) +def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path): origin, reference = _child_of_the_demo(tmp_path / "origin") cache = tmp_path / "cache" shutil.copytree(origin.root / "test/child/1.0.0", cache / "test/child/1.0.0") - client = TimeNet(cache) + store = LocalRegistry(cache) with pytest.raises(TimeNetMissingParentError) as caught: - client.open("test/child", auto_build=False) + TimeNet(sources=[store]).open("test/child") assert (caught.value.missing, caught.value.needed_by) == (reference, DatasetRef.parse("test/child@1.0.0")) assert str(caught.value).endswith( - ". Read it from a registry that holds it with fetch_from= or build it from its connector with " - "auto_build=True." + ". Add a registry that holds it to the sources, or add InstalledConnectors() to build it." ) - # No installed connector builds the payload source, so with auto_build the hint names only fetch_from. - with pytest.raises( - TimeNetMissingParentError, match=r"\. Read it from a registry that holds it with fetch_from=\.$" - ): - client.open("test/child") - with client.open("test/child", fetch_from=str(origin.root)) as reader: + with TimeNet(sources=[store, str(origin.root), InstalledConnectors()]).open("test/child") as reader: assert reader.read().records[0].signals[0].to_arrow().equals(make_dataset().records[0].signals[0].to_arrow()) # The read opens the payload source in origin, where it is, and copies nothing. assert not (cache / reference.dataset_id).exists() -@pytest.mark.parametrize("in_fetch_from", [True, False]) -def test_a_missing_version_is_read_from_fetch_from_before_it_is_built(tmp_path, monkeypatch, in_fetch_from): +@pytest.mark.parametrize("in_origin", [True, False]) +def test_a_missing_version_is_read_from_a_registry_before_it_is_built(tmp_path, monkeypatch, in_origin): origin = LocalRegistry(tmp_path / "origin") - if in_fetch_from: + if in_origin: _child_of_the_demo(origin.root) calls = [] install_backends(monkeypatch, _Builder(calls)) + store = LocalRegistry(tmp_path / "local") - with TimeNet(tmp_path / "local").open("test/child", fetch_from=origin) as reader: + with TimeNet(sources=[store, origin, InstalledConnectors()]).open("test/child") as reader: assert reader.read().records - if in_fetch_from: + if in_origin: assert calls == [] assert not (tmp_path / "local").exists() return @@ -127,7 +121,7 @@ def compose(self, raw_refs, context): return child -def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_path): +def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_path, monkeypatch): origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) metadata = _Resampled().metadata() @@ -139,15 +133,16 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p expected = make_dataset().records[0].signals[0].to_arrow()[::2] shutil.rmtree(origin.root / reference.dataset_id) - # Reading the compiled result needs no original samples, even when fetching is enabled. - loaded = TimeNet(origin).load(metadata.dataset_id, fetch_from=str(origin.root)) + # Reading the compiled result needs no original samples. + loaded = TimeNet(sources=[origin]).load(metadata.dataset_id) assert loaded.records[0].signals[0].to_arrow().equals(expected) # Running the connector again still requires its declared parent. with pytest.raises(TimeNetDatasetNotFoundError, match=reference.dataset_id): open_parents(metadata, origin) cache = tmp_path / "download" - directory = TimeNet(origin, storage_path=cache).download(metadata.dataset_id) + monkeypatch.setenv("TIMENET_STORAGE", str(cache)) + directory = TimeNet(sources=[origin]).download(metadata.dataset_id) assert not (cache / reference.dataset_id).exists() shutil.rmtree(origin.root) version = open_local_version(directory) @@ -157,7 +152,7 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p assert reader.read().records[0].signals[0].to_arrow().equals(expected) -def test_a_remote_client_takes_no_fetch_from_and_never_builds(tmp_path, monkeypatch): +def test_a_list_of_one_remote_registry_never_builds(tmp_path, monkeypatch): calls = [] install_backends(monkeypatch, _Builder(calls)) # The remote registry holds the child, but not its payload source, which the installed backend builds. @@ -165,17 +160,16 @@ def test_a_remote_client_takes_no_fetch_from_and_never_builds(tmp_path, monkeypa shutil.rmtree(origin.root / reference.dataset_id) transport, _ = build_service_fake(origin.root) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + hint = "Add a registry that holds it to the sources, or add InstalledConnectors() to build it." - message = "fetch_from adds a registry to the reads of a local registry; a remote client reads from its own registry" - with pytest.raises(TimeFValidationError, match=message): - TimeNet(remote).open("test/child", fetch_from=str(tmp_path)) - # Neither fetch_from nor auto_build helps a remote client, so the messages name no keyword. with pytest.raises(TimeNetMissingParentError) as missing: - TimeNet(remote).open("test/child") - assert str(missing.value) == f"payload source {reference} of test/child@1.0.0 is not in registry http://api.local." + TimeNet(sources=[remote]).open("test/child") + assert str(missing.value) == ( + f"payload source {reference} of test/child@1.0.0 is not in registry http://api.local. {hint}" + ) with pytest.raises(TimeNetDatasetNotFoundError) as caught: - TimeNet(remote).open(reference.dataset_id) - assert str(caught.value) == f"{reference.dataset_id} is in none of the sources: registry http://api.local." + TimeNet(sources=[remote]).open(reference.dataset_id) + assert str(caught.value) == f"{reference.dataset_id} is in none of the sources: registry http://api.local. {hint}" assert calls == [] @@ -194,11 +188,41 @@ def registry_root(tmp_path): return tmp_path / "reg" -def test_no_args_defaults_to_the_hosted_registry(monkeypatch): - monkeypatch.delenv("TIMENET_REGISTRY", raising=False) - client = TimeNet() # no registry, no storage - assert isinstance(client._registry, RemoteRegistry) - assert client._registry._base_url == TIMENET_REGISTRY_URL +@pytest.fixture +def storage(tmp_path, monkeypatch): + """The directory that ``download()`` copies into.""" + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "storage")) + return tmp_path / "storage" + + +def _entry(source): + """Describe one source of a list: a local registry by its root, another source by its text.""" + if isinstance(source, LocalRegistry): + return ("local", source.root) + if isinstance(source, InstalledConnectors): + return "installed connectors" + return str(source) + + +@pytest.mark.parametrize("registry", [None, "https://registry.example.test", "local"]) +def test_the_default_list_follows_timenet_registry(tmp_path, monkeypatch, registry): + monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) + store = ("local", tmp_path / "home" / "registry") + if registry == "local": + monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "elsewhere")) + expected = [("local", tmp_path / "elsewhere"), "installed connectors"] + elif registry is not None: + monkeypatch.setenv("TIMENET_REGISTRY", registry) + expected = [store, f"registry {registry}", "installed connectors"] + else: + expected = [store, f"registry {TIMENET_REGISTRY_URL}", "installed connectors"] + client = TimeNet() + + assert [_entry(source) for source in client._sources] == expected + assert client._store is client._sources[0] + # The argument comes before $TIMENET_REGISTRY, and adds a remote to a local $TIMENET_REGISTRY. + with_remote = [_entry(source) for source in default_sources(remote_registry="https://other.example.test")] + assert with_remote == [expected[0], "registry https://other.example.test", "installed connectors"] def test_timenet_registry_env_selects_a_local_registry(tmp_path, monkeypatch): @@ -208,38 +232,142 @@ def test_timenet_registry_env_selects_a_local_registry(tmp_path, monkeypatch): dataset.derive_schema() with TimeFWriter(tmp_path / "home" / "registry", dataset) as writer: writer.write() - client = TimeNet() # registry comes from $TIMENET_REGISTRY + client = TimeNet() # the store comes from $TIMENET_REGISTRY + assert [_entry(source) for source in client._sources] == [ + ("local", tmp_path / "home" / "registry"), + "installed connectors", + ] assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} assert client.download("timenet/hello-world") == tmp_path / "home" / "storage" / "timenet/hello-world" / "1.0.0" -def test_list(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") - assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} +def test_a_client_takes_an_ordered_list_of_sources(tmp_path): + store = LocalRegistry(tmp_path / "store") + client = TimeNet(sources=[store, str(tmp_path / "a"), tmp_path / "b", InstalledConnectors()]) + + first, second, third, fourth = client._sources + assert first is store + for source, root in ((second, tmp_path / "a"), (third, tmp_path / "b")): + assert isinstance(source, LocalRegistry) + assert source.root == root + assert fourth == InstalledConnectors() + + +@pytest.mark.parametrize("writable", ["class", "path"]) +def test_builds_land_in_the_first_local_registry_that_is_not_read_only(tmp_path, monkeypatch, writable): + install_backends(monkeypatch, _Builder([])) + roots = [tmp_path / name for name in ("read-only", "first", "later")] + first = LocalRegistry(roots[1]) if writable == "class" else str(roots[1]) + sources = [LocalRegistry(roots[0], read_only=True), first, LocalRegistry(roots[2]), InstalledConnectors()] + TimeNet(sources=sources).load("timenet/hello-world") -def test_get_returns_manifest(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") - assert isinstance(client.get("timenet/hello-world"), Manifest) + built = [root.name for root in roots if LocalRegistry(root).exists("timenet/hello-world", "1.0.0")] + assert built == ["first"] -def test_search(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_a_client_rejects_one_source_given_as_a_string(tmp_path): + for value in (str(tmp_path), tmp_path): + with pytest.raises(TimeFValidationError) as caught: + TimeNet(value) + assert str(caught.value) == ( + f"sources takes a list of sources. Pass sources=[{str(tmp_path)!r}], or " + f"sources=[{str(tmp_path)!r}, InstalledConnectors()] " + "to build what the path lacks" + ) + + +def test_a_client_rejects_one_registry_given_without_a_list(tmp_path): + remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404))) + for registry in (LocalRegistry(tmp_path), remote): + with pytest.raises(TimeFValidationError, match=r"^sources takes a list of sources\. Pass sources=\["): + TimeNet(registry) # ty: ignore[invalid-argument-type] + + +@pytest.mark.parametrize( + ("value", "message"), + [ + ("", "sources[0] is blank; give a registry URL or a local path"), + ( + "ftp://x", + "sources[0] 'ftp://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path.", + ), + ], + ids=["blank", "unsupported-scheme"], +) +def test_a_client_rejects_a_string_that_is_not_a_registry(value, message): + with pytest.raises(TimeNetRegistryError) as caught: + TimeNet(sources=[value]) + assert str(caught.value) == message + + +def test_a_client_rejects_an_empty_list(): + with pytest.raises(TimeFValidationError, match=r"^sources is empty\."): + TimeNet(sources=[]) + + +def test_a_client_rejects_an_item_that_is_not_a_source(tmp_path): + with pytest.raises(TimeFValidationError, match=r"^sources\[1\] is not a source, a registry URL or a path: 7$"): + TimeNet(sources=[str(tmp_path), 7]) # ty: ignore[invalid-argument-type] + + +def test_installed_connectors_need_a_local_registry_that_is_not_read_only(tmp_path): + with pytest.raises(TimeFValidationError) as caught: + TimeNet(sources=[LocalRegistry(tmp_path, read_only=True), InstalledConnectors()]) + assert str(caught.value) == ( + "InstalledConnectors() needs a writable local registry to build into. " + "Add a LocalRegistry without read_only=True to the sources" + ) + + +def test_a_list_without_installed_connectors_never_builds(registry_root, monkeypatch): + calls = [] + monkeypatch.setattr(_Builder, "VERSION", "2.0.0") + install_backends(monkeypatch, _Builder(calls)) + client = TimeNet(sources=[LocalRegistry(registry_root)]) + assert str(client.load("timenet/hello-world").metadata.dataset_version) == "1.0.0" + assert calls == [] + + +def test_get_list_and_search_ask_the_first_registry_that_is_not_the_store(registry_root, tmp_path): + store = LocalRegistry(tmp_path / "store") + other = make_dataset() + other._metadata = other.metadata.model_copy(update={"dataset_id": "test/other"}) + store.put_dataset(other) + client = TimeNet(sources=[store, str(registry_root), InstalledConnectors()]) + + assert [metadata.dataset_id for metadata in client.list()] == ["timenet/hello-world"] + assert [metadata.dataset_id for metadata in client.search(query="hello")] == ["timenet/hello-world"] + assert client.get("timenet/hello-world").dataset_id == "timenet/hello-world" + with pytest.raises(TimeNetDatasetNotFoundError): + client.get("test/other") + # Without such a registry, they ask the store. + alone = TimeNet(sources=[store, InstalledConnectors()]) + assert [metadata.dataset_id for metadata in alone.list()] == ["test/other"] + + +def test_list(registry_root): + client = TimeNet(sources=[str(registry_root)]) + assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} + + +def test_search(registry_root): + client = TimeNet(sources=[str(registry_root)]) assert {m.dataset_id for m in client.search(domain=Domain.GENERAL)} == {"timenet/hello-world"} assert client.search(domain=Domain.CARDIOLOGY) == [] -def test_download_copies_into_storage(registry_root, tmp_path): - storage = tmp_path / "store" - client = TimeNet(registry_root, storage_path=storage) +def test_download_copies_into_storage(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) version_dir = client.download("timenet/hello-world") assert version_dir == storage / "timenet/hello-world" / "1.0.0" assert (version_dir / "manifest.json").exists() assert list(version_dir.glob("time_series/part-*.parquet")) -def test_download_reports_progress(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_download_reports_progress(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) reported: list[int] = [] client.download("timenet/hello-world", progress_cb=reported.append) expected = sum(part.size for part in client.get("timenet/hello-world").files.all_files()) @@ -252,15 +380,15 @@ def test_manifest_file_rejects_path_traversal(): FilePart(path="../../escape.txt", checksum="sha256:" + "0" * 64, size=0) -def test_download_is_idempotent(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_download_is_idempotent(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) first = client.download("timenet/hello-world") second = client.download("timenet/hello-world") assert first == second -def test_force_redownload_replaces_atomically(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_force_redownload_replaces_atomically(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) first = client.download("timenet/hello-world") (first / "stale.txt").write_text("left over") # a marker that a clean replace should remove @@ -271,9 +399,8 @@ def test_force_redownload_replaces_atomically(registry_root, tmp_path): assert not list(again.parent.glob("*.tmp-*")) # staging dir cleaned up -def test_download_leaves_sibling_staging_dirs_untouched(registry_root, tmp_path): - storage = tmp_path / "store" - client = TimeNet(registry_root, storage_path=storage) +def test_download_leaves_sibling_staging_dirs_untouched(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) # A concurrent download of the same version has a live .tmp-* dir. download() must not # delete a sibling staging dir: sweeping siblings would corrupt that other download mid-write. sibling = storage / "timenet/hello-world" / "1.0.0.tmp-deadbeef" @@ -285,8 +412,8 @@ def test_download_leaves_sibling_staging_dirs_untouched(registry_root, tmp_path) assert (version_dir / "manifest.json").exists() # the download still completed -def test_open_returns_a_lazy_reader(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_open_returns_a_lazy_reader(registry_root): + client = TimeNet(sources=[str(registry_root)]) with client.open("timenet/hello-world") as reader: assert isinstance(reader, TimeFReader) assert reader._control is None # the catalog opens on the first read @@ -315,7 +442,7 @@ def counting_close(reader): close(reader) monkeypatch.setattr(TimeFReader, "close", counting_close) - restored = TimeNet(tmp_path / "reg", storage_path=tmp_path / "store").load("timenet/hello-world") + restored = TimeNet(sources=[str(tmp_path / "reg")]).load("timenet/hello-world") assert isinstance(restored, TimeFDataset) assert len(closed) == 1 signals = [signal for record in restored.records for signal in record.signals] @@ -332,12 +459,13 @@ def counting_close(reader): assert_datasets_equal(expected, restored) -def test_load_reads_in_place_without_downloading(registry_root, tmp_path): - # load() reads the release where it is, not download-then-read, so it must not - # write a copy into local storage the way download() does. - storage = tmp_path / "store" - client = TimeNet(registry_root, storage_path=storage) +def test_load_reads_in_place_without_downloading(registry_root, tmp_path, storage): + # load() reads the release where it is, not download-then-read, so it must not write a copy into + # the store or into local storage the way download() does. + store = tmp_path / "store" + client = TimeNet(sources=[LocalRegistry(store), str(registry_root), InstalledConnectors()]) client.load("timenet/hello-world") + assert not store.exists() # nothing copied into the store assert not (storage / "timenet/hello-world").exists() # nothing fetched into the download cache @@ -357,7 +485,7 @@ def counting_close(reader): monkeypatch.setattr(TimeFReader, "read", read_everything) monkeypatch.setattr(TimeFReader, "close", counting_close) - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[str(registry_root)]) with client.open_torch("timenet/hello-world") as ds: assert isinstance(ds, Dataset) assert len(ds) == len(make_dataset().records) @@ -368,9 +496,9 @@ def counting_close(reader): @pytest.mark.parametrize("arrays_loaded", [True, False], ids=["default-plan", "text-only-plan"]) -def test_open_torch_passes_its_load_plan_to_the_view(registry_root, tmp_path, arrays_loaded): +def test_open_torch_passes_its_load_plan_to_the_view(registry_root, arrays_loaded): pytest.importorskip("torch") - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[str(registry_root)]) view = ( client.open_torch("timenet/hello-world") if arrays_loaded @@ -383,9 +511,9 @@ def test_open_torch_passes_its_load_plan_to_the_view(registry_root, tmp_path, ar assert {signal._values.is_loaded for signal in record.signals} == {arrays_loaded} -def test_registry_env_var(registry_root, tmp_path, monkeypatch): +def test_registry_env_var(registry_root, monkeypatch): monkeypatch.setenv("TIMENET_REGISTRY", str(registry_root)) - client = TimeNet(storage_path=tmp_path / "store") + client = TimeNet() assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} @@ -402,22 +530,22 @@ def versioned_registry(tmp_path): return root -def test_version_ref_pins_and_defaults_to_latest(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_version_ref_pins_and_defaults_to_latest(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) assert str(client.get("timenet/hello-world").metadata.dataset_version) == "1.1.0" # no ref -> latest assert str(client.get("timenet/hello-world@latest").metadata.dataset_version) == "1.1.0" assert str(client.get("timenet/hello-world@1.0.0").metadata.dataset_version) == "1.0.0" # pinned assert str(client.get("timenet/hello-world", "1.0.0").metadata.dataset_version) == "1.0.0" # explicit arg -def test_open_and_load_read_the_version_asked_for(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_open_and_load_read_the_version_asked_for(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) with client.open("timenet/hello-world", version="1.0.0") as reader: assert str(reader.metadata.dataset_version) == "1.0.0" assert str(client.load("timenet/hello-world", version="1.0.0").metadata.dataset_version) == "1.0.0" -def test_open_torch_reads_the_version_asked_for(versioned_registry, tmp_path, monkeypatch): +def test_open_torch_reads_the_version_asked_for(versioned_registry, monkeypatch): pytest.importorskip("torch") opened: list[TimeFReader] = [] original_open = TimeNet.open @@ -427,55 +555,56 @@ def recording_open(client, *args, **kwargs): return opened[-1] monkeypatch.setattr(TimeNet, "open", recording_open) - TimeNet(versioned_registry, storage_path=tmp_path / "store").open_torch("timenet/hello-world", version="1.0.0") + TimeNet(sources=[str(versioned_registry)]).open_torch("timenet/hello-world", version="1.0.0") (reader,) = opened assert str(reader.metadata.dataset_version) == "1.0.0" -def test_version_ref_download_pins(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") - assert client.download("timenet/hello-world@1.0.0") == tmp_path / "store" / "timenet/hello-world" / "1.0.0" +def test_version_ref_download_pins(versioned_registry, storage): + client = TimeNet(sources=[str(versioned_registry)]) + assert client.download("timenet/hello-world@1.0.0") == storage / "timenet/hello-world" / "1.0.0" -def test_version_ref_missing_pin_raises(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_version_ref_missing_pin_raises(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) with pytest.raises(TimeNetDatasetNotFoundError): client.get("timenet/hello-world@9.9.9") -def test_version_given_twice_raises(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_version_given_twice_raises(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) with pytest.raises(ValueError, match="twice"): client.get("timenet/hello-world@1.0.0", "1.1.0") -def test_load_names_the_buildable_version_when_a_pin_cannot_be_built(registry_root, tmp_path, monkeypatch): +def test_load_names_the_buildable_version_when_a_pin_cannot_be_built(registry_root, monkeypatch): # The builder declares only one version, so a pin it cannot satisfy must say which version it # builds, and must fail before the expensive build rather than after it. calls = [] install_backends(monkeypatch, _Builder(calls)) - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[LocalRegistry(registry_root), InstalledConnectors()]) with pytest.raises(TimeNetVersionUnavailableError, match=r"builds version 1\.0\.0, not the requested 9\.9\.9"): client.load("timenet/hello-world@9.9.9") assert calls == [] # the pin was rejected before the build ran -@pytest.mark.parametrize("fetch_from", [None, "empty"]) -def test_a_missing_version_that_no_connector_builds_names_the_keyword_that_can_get_it( - tmp_path, monkeypatch, fetch_from -): +@pytest.mark.parametrize("connectors", [True, False]) +def test_a_release_that_no_source_provides_names_how_to_get_it(tmp_path, monkeypatch, connectors): monkeypatch.setattr(builders_module, "entry_points", lambda group: []) - client = TimeNet(tmp_path / "reg", storage_path=tmp_path / "store") - source = None if fetch_from is None else LocalRegistry(tmp_path / fetch_from) + sources = [ + LocalRegistry(tmp_path / "reg"), + LocalRegistry(tmp_path / "empty"), + *([InstalledConnectors()] if connectors else []), + ] with pytest.raises(TimeNetDatasetNotFoundError) as caught: - client.open("x/y", fetch_from=source) + TimeNet(sources=sources).open("x/y") - sources = [LocalRegistry(tmp_path / "reg"), *([] if source is None else [source]), InstalledConnectors()] message = f"x/y is in none of the sources: {', '.join(str(entry) for entry in sources)}." - # The hint names only the keyword that the call did not use. - hint = " Read it from a registry that holds it with fetch_from=." if source is None else "" + # The hint names InstalledConnectors() only if the list does not have it. + hint = " Add a registry that holds it to the sources" + hint += "." if connectors else ", or add InstalledConnectors() to build it." assert str(caught.value) == message + hint @@ -484,31 +613,28 @@ def test_load_accepts_the_latest_sentinel_after_a_build(tmp_path, monkeypatch): empty_root = tmp_path / "reg" empty_root.mkdir() install_backends(monkeypatch, _Builder([])) - client = TimeNet(empty_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[LocalRegistry(empty_root), InstalledConnectors()]) assert str(client.load("timenet/hello-world", "latest").metadata.dataset_version) == "1.0.0" @pytest.mark.parametrize("entry_point", ["open", "load", "open_torch"]) -def test_load_does_not_build_when_auto_build_is_false(registry_root, tmp_path, monkeypatch, entry_point): - # A caller can opt out of the build-on-miss, so a notebook load fails fast instead of starting a - # multi-GB download and a build. The builder declares the requested version, so only - # auto_build=False keeps it from running. +def test_a_list_without_installed_connectors_does_not_build_a_missing_version(registry_root, monkeypatch, entry_point): + # A list without InstalledConnectors() does not build on a miss, so a notebook load fails fast + # instead of starting a multi-GB download and a build. The builder declares the requested + # version, so only the list keeps it from running. if entry_point == "open_torch": pytest.importorskip("torch") calls = [] monkeypatch.setattr(_Builder, "VERSION", "9.9.9") install_backends(monkeypatch, _Builder(calls)) - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[LocalRegistry(registry_root)]) - # The hint names the keywords that could obtain the release, in a sentence of its own. - hint = ( - r"\. Read it from a registry that holds it with fetch_from= or build it from its connector with " - r"auto_build=True\.$" - ) + # The hint names the sources that could provide the release, in a sentence of its own. + hint = r"\. Add a registry that holds it to the sources, or add InstalledConnectors\(\) to build it\.$" with pytest.raises(TimeNetDatasetNotFoundError, match=hint): - getattr(client, entry_point)("timenet/hello-world@9.9.9", auto_build=False) - assert calls == [] # auto_build=False skips the connector entirely + getattr(client, entry_point)("timenet/hello-world@9.9.9") + assert calls == [] # the list skips the connector entirely def _card(dataset_id, access, access_url=None): @@ -523,41 +649,57 @@ def _card(dataset_id, access, access_url=None): ) -def _hosted(client, metadata, lineage=()): - """Serve one manifest from a registry that is not local, so the build-your-own gate applies.""" +def _hosted(tmp_path, metadata, lineage=()): + """Return a registry that is not local and serves one manifest, so the build-your-own gate applies.""" class _Hosted(RemoteRegistry): def get_manifest(self, dataset_id, version=None): return SimpleNamespace(metadata=metadata, lineage=lineage) - client._registry = _Hosted( + return _Hosted( "http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404)), - cache_dir=client._storage, + cache_dir=tmp_path / "cache", ) def test_load_rejects_a_credentialed_dataset_from_a_hosted_registry(tmp_path): - client = TimeNet(tmp_path / "registry", storage_path=tmp_path / "store") - _hosted(client, _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua")) + client = TimeNet( + sources=[_hosted(tmp_path, _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua"))] + ) message = ( "org/gated@1.0.0 is credentialed. Get access at https://physionet.example/dua for org/gated@1.0.0. " - "TimeNet does not host its data, so build it locally with `timenet-build build org/gated`." + "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " + "`timenet-build build org/gated`." ) - for entry_point in (client.open, client.load, client.download): + for entry_point in (client.open, client.load): with pytest.raises(TimeNetAccessError) as caught: entry_point("org/gated") assert str(caught.value) == message +def test_the_access_hint_says_that_no_installed_connector_builds_the_release(tmp_path): + card = _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua") + client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), _hosted(tmp_path, card), InstalledConnectors()]) + + with pytest.raises(TimeNetAccessError) as caught: + client.open("org/gated") + + assert str(caught.value) == ( + "org/gated@1.0.0 is credentialed. Get access at https://physionet.example/dua for org/gated@1.0.0. " + "No source serves its files, and no installed connector builds this release. " + "Install its connector package and build it locally with `timenet-build build org/gated`." + ) + + def test_the_hint_for_a_hosted_child_names_the_access_page_of_each_ancestor(tmp_path): - client = TimeNet(tmp_path / "registry", storage_path=tmp_path / "store") # The card of the child stays open and has no access page. Its parents set the access. parents = ( _card("org/a", Access.CREDENTIALED, "https://a.example"), _card("org/b", Access.RESTRICTED, "https://b.example"), ) - _hosted(client, _card("org/child", Access.OPEN), tuple(ParentDatasetCard(metadata=card) for card in parents)) + lineage = tuple(ParentDatasetCard(metadata=card) for card in parents) + client = TimeNet(sources=[_hosted(tmp_path, _card("org/child", Access.OPEN), lineage)]) with pytest.raises(TimeNetAccessError) as caught: client.open("org/child") @@ -569,5 +711,6 @@ def test_the_hint_for_a_hosted_child_names_the_access_page_of_each_ancestor(tmp_ assert str(caught.value) == ( "org/child@1.0.0 is restricted. " "Get access at https://a.example for org/a@1.0.0 and at https://b.example for org/b@1.0.0. " - "TimeNet does not host its data, so build it locally with `timenet-build build org/child`." + "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " + "`timenet-build build org/child`." ) From b4c7494cb19e2bba7c29da2d2406b486e14ee004 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 11:42:40 +0100 Subject: [PATCH 2/5] refactor: mark the glue of the list of sources as a TODO The glue comments of this change now start with TODO(adrian), so a search finds each one that is left. Part of TNET-157. --- packages/timenet/src/timenet/cli/app.py | 3 ++- packages/timenet/src/timenet/client.py | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index ca411fd93..d36604d6c 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -69,7 +69,8 @@ def _client(registry: str | None) -> TimeNet: Returns: The client. """ - # Glue: --registry maps onto default_sources() until the CLI takes the flags of the defaults. + # TODO(adrian): Remove temporary glue which maps --registry onto default_sources() until the + # CLI takes the flags of the defaults. return TimeNet(sources=default_sources(remote_registry=registry)) diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index afff4d29a..3dd9898d9 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -65,8 +65,9 @@ def default_sources(*, remote_registry: str | Path | None = None) -> list[Releas Returns: A new list, made from the environment at the time of the call. """ - # Glue: this body reads TIMENET_REGISTRY and TIMENET_HOME. The variables of the defaults - # (TIMENET_LOCAL_REGISTRY, TIMENET_REMOTE_REGISTRY, TIMENET_AUTO_BUILD) replace them later. + # TODO(adrian): Remove temporary glue which maps TIMENET_REGISTRY and TIMENET_HOME onto the + # list until the variables of the defaults (TIMENET_LOCAL_REGISTRY, TIMENET_REMOTE_REGISTRY, + # TIMENET_AUTO_BUILD) replace them. cfg = settings() configured = None if cfg.registry is None else open_registry(cfg.registry) local = configured if isinstance(configured, LocalRegistry) else None @@ -259,8 +260,9 @@ def download( Returns: The local ``///`` directory. """ - # Glue: the interim download() copies from one registry into the storage directory, as before - # the list of sources. A download into the store replaces it. + # TODO(adrian): Remove temporary glue which maps download() onto a copy from one registry + # into the storage directory, as before the list of sources, until a download into the + # local registry replaces it. dataset_id, version = _resolve_ref(dataset_id, version) registry = self._catalog() self._reject_unhosted_access(dataset_id, version) @@ -378,15 +380,17 @@ def _catalog(self) -> BaseRegistry: Raises: TimeFValidationError: If the sources hold no registry. """ - # Glue: until list() and search() ask every source, these methods ask one registry. This - # registry gives the results of the client of one registry, from before the list of sources. + # TODO(adrian): Remove temporary glue which maps these methods onto one registry until + # list() and search() ask every source. This registry gives the results of the client of + # one registry, from before the list of sources. registries = [source for source in self._sources if isinstance(source, BaseRegistry)] registry = next((registry for registry in registries if registry is not self._store), self._store) if registry is None: raise TimeFValidationError("list(), get(), search() and download() need a registry in the sources") return registry - # Glue: only the interim download() uses this check. A later change replaces download(). + # TODO(adrian): Remove temporary glue which checks access only for the interim download() + # until a later change replaces download(). def _reject_unhosted_access(self, dataset_id: str, version: str | None) -> None: """Enforce the registry's open access requirement. @@ -465,7 +469,8 @@ def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: return f"No source serves its files. Add InstalledConnectors() to the sources, or build it locally with {build}." -# Glue: only the interim download() uses this hint. A later change replaces download(). +# TODO(adrian): Remove temporary glue which gives a hint only for the interim download() until a +# later change replaces download(). def _unhosted_hint(dataset_id: str) -> str: """Tell the caller how to get a dataset whose data the registry of :meth:`TimeNet.download` does not serve. From 6e08968d08d777926d2aaa7681fc6b244f3dad17 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Wed, 4 Nov 2026 19:10:00 +0100 Subject: [PATCH 3/5] docs: shorten the docstrings and docs of the list of sources Say only what a caller of TimeNet(sources=...) and its store needs to know. Part of TNET-157. --- examples/load_tsqa.py | 7 ++- .../src/timenet_connectors/api.py | 8 ++- packages/timenet/src/timenet/client.py | 53 +++++++------------ 3 files changed, 24 insertions(+), 44 deletions(-) diff --git a/examples/load_tsqa.py b/examples/load_tsqa.py index f361dbc51..795c44f8a 100644 --- a/examples/load_tsqa.py +++ b/examples/load_tsqa.py @@ -5,10 +5,9 @@ uv run timenet-build build "chengsenwang/tsqa" --out .timenet-registry uv run python examples/load_tsqa.py -``TimeNet(sources=[registry]).open(...)`` returns a reader, and its ``read()`` returns a -``TimeFDataset``. ``describe()`` prints its identity, counts, per-spec columns, and a record -preview. Signal values stay in storage until you ask for them. Pass a dataset ID as the first -argument and a registry path as the second argument. +``TimeNet(sources=[registry]).open(...)`` returns a reader, whose ``read()`` returns a +``TimeFDataset`` with its Signal values still in storage. Pass a dataset ID as the first argument +and a registry path as the second argument. """ from pathlib import Path diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py index 88c5e03bc..f736c3209 100644 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ b/packages/timenet-connectors/src/timenet_connectors/api.py @@ -109,15 +109,13 @@ def load( ) -> "TimeFDataset": """Build a dataset if needed and load it into memory. - The build lands in the store of the default list of sources - (:func:`timenet.client.default_sources`), with each parent and payload source that it needs. The - read then reads it through that list. + This runs :func:`build` into the store of :func:`timenet.client.default_sources`, then + :meth:`timenet.client.TimeNet.load` through that list. Args: dataset_id: The dataset ID (``org/name``). version: The version to load, which the card must declare, or ``None`` for the latest. - fetch_from: The registry, or its URI, that the build copies missing parent releases and - payload sources from. + fetch_from: The registry, or its URI, to copy missing parents and payload sources from. build_parents: Build missing parent releases from their installed connectors. Returns: diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 3dd9898d9..1df6bd3c5 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -1,12 +1,8 @@ """The :class:`TimeNet` class is the SDK's single entry point for using TimeNet from code. -A client reads releases through an ordered list of sources: registries and -``InstalledConnectors()`` (see :mod:`timenet.sources`). Its methods ``open``, ``load``, and -``open_torch`` read a release where it is, through the read coordinator in -:mod:`timenet.coordinator.read`. If only ``InstalledConnectors()`` provides a release, they build it -into the first :class:`~timenet.registry.LocalRegistry` of the list that is not -``read_only=True``. The methods ``list``, ``get``, ``search``, and ``download`` ask one registry of the -list. :func:`default_sources` makes the list of ``TimeNet()``. +A client reads releases through an ordered list of sources, and builds a release that only +``InstalledConnectors()`` provides into its first local registry that is not read-only. +:func:`default_sources` makes the list of ``TimeNet()``. """ # The public API has a method named ``list``. Deferred annotations keep the type hint @@ -51,16 +47,14 @@ def default_sources(*, remote_registry: str | Path | None = None) -> list[ReleaseSource]: - """Return the list of sources of ``TimeNet()``: the store, a registry, and the installed connectors. + """Return the default sources: the store, a registry, and the installed connectors. The store is ``$TIMENET_REGISTRY`` if it names a local path, else ``/registry``. - The registry after the store is ``remote_registry``, else ``$TIMENET_REGISTRY`` if it names a - registry that is not local, else the hosted registry, ``timenet://``. A local - ``$TIMENET_REGISTRY`` without ``remote_registry`` gives a list without that registry. + The registry is ``remote_registry``, else ``$TIMENET_REGISTRY`` if it is not local, else + ``timenet://``, and a local ``$TIMENET_REGISTRY`` without ``remote_registry`` leaves it out. Args: - remote_registry: The registry between the store and the installed connectors: a registry - URL, a ``file://`` URI, or a local path. + remote_registry: The registry after the store, as a URL, a ``file://`` URI or a local path. Returns: A new list, made from the environment at the time of the call. @@ -141,27 +135,17 @@ class TimeNet: def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) -> None: """Create a client that reads releases through an ordered list of sources. - Without a version, a call takes the newest version of the first source that serves the - dataset. An exact version comes from the first source that serves it, and ``"latest"`` is - the highest version that any source provides. Only "not found" and "not served" fall - through to the next source. Any other error stops the call. See :mod:`timenet.sources`. - - A source is a registry or ``InstalledConnectors()``. A string or a - :class:`~pathlib.Path` is a registry: a URL (``"timenet://"``, ``"https://..."``, - ``"s3://..."``), a ``file://`` URI, or a local path. A local path opens a writable - :class:`~timenet.registry.LocalRegistry`. Builds land in the first local registry of the - list that is not ``read_only=True``. The client only reads any later local registry. - ``InstalledConnectors()`` in the list is the consent to builds, so a list with it needs a - writable local registry. + A source is a registry, a registry URL or path, or ``InstalledConnectors()`` (see + :mod:`timenet.sources`). A path opens a writable :class:`~timenet.registry.LocalRegistry`. + Builds land in the first local registry that is not ``read_only=True``, and a list with + ``InstalledConnectors()`` needs one. Args: sources: The sources, in order. ``None`` means :func:`default_sources`. Raises: - TimeFValidationError: If ``sources`` is one string or path instead of a list, or is - empty. Also if it holds an item that is not a source, a string, or a path, a string - that is not a registry URL, or ``InstalledConnectors()`` without a writable local - registry. + TimeFValidationError: If ``sources`` is not a valid list, or has ``InstalledConnectors()`` + without a writable local registry. """ self._sources = tuple(default_sources()) if sources is None else _open_sources(sources) self._store = next( @@ -188,7 +172,7 @@ def get(self, dataset_id: str, version: str | None = None) -> Manifest: """Return a dataset's manifest from the registry that :meth:`list` asks. Args: - dataset_id: The dataset id. + dataset_id: The dataset ID. version: The version string, or ``None`` for the latest. Returns: @@ -247,12 +231,11 @@ def download( ) -> Path: """Fetch a version and each of its payload sources from the registry that :meth:`list` asks. - The copy lands in the storage directory: ``$TIMENET_STORAGE``, or ``/storage`` - if that variable is not set. Build-only parents remain recorded in the manifest and are not - downloaded. + The copy lands in ``$TIMENET_STORAGE``, else in ``/storage``. Parents that + only the build needs are not downloaded. Args: - dataset_id: The dataset id. + dataset_id: The dataset ID. version: The version string, or ``None`` for the latest. force: Download again, even if an up-to-date copy already exists. progress_cb: Called with each file's byte count as it lands, for a progress display. @@ -472,7 +455,7 @@ def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: # TODO(adrian): Remove temporary glue which gives a hint only for the interim download() until a # later change replaces download(). def _unhosted_hint(dataset_id: str) -> str: - """Tell the caller how to get a dataset whose data the registry of :meth:`TimeNet.download` does not serve. + """Tell the caller how to get a dataset whose files the download registry does not serve. Returns: A sentence with the dataset's build command. From 22a5055cfc27ab511d6ca3aa9ef6848e1737f713 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 00:23:52 +0100 Subject: [PATCH 4/5] docs: shorten the docstrings of the list of sources Shorter docstrings for the code that this change adds or changes. Part of TNET-157. --- examples/load_tsqa.py | 10 +- .../src/timenet_connectors/api.py | 11 +-- .../src/timenet_connectors/builder/cli.py | 11 +-- packages/timenet/src/timenet/cli/app.py | 15 +-- packages/timenet/src/timenet/client.py | 97 +++++++++---------- .../timenet/src/timenet/registry/factory.py | 7 +- .../timenet/src/timenet/registry/local.py | 6 +- 7 files changed, 73 insertions(+), 84 deletions(-) diff --git a/examples/load_tsqa.py b/examples/load_tsqa.py index 795c44f8a..9256ee49c 100644 --- a/examples/load_tsqa.py +++ b/examples/load_tsqa.py @@ -1,13 +1,11 @@ -"""Load a built TimeF dataset and print a summary (no pandas or extras needed). +"""Print a local dataset's summary and sample signal values. -Build the dataset into an explicit local registry, then run this script:: +Build the dataset into a local registry, then run this script:: uv run timenet-build build "chengsenwang/tsqa" --out .timenet-registry uv run python examples/load_tsqa.py -``TimeNet(sources=[registry]).open(...)`` returns a reader, whose ``read()`` returns a -``TimeFDataset`` with its Signal values still in storage. Pass a dataset ID as the first argument -and a registry path as the second argument. +Pass a dataset ID and a registry path as arguments to load another dataset. """ from pathlib import Path @@ -17,7 +15,7 @@ def main() -> None: - """Open the dataset, print its describe() summary, and show one raw series slice.""" + """Print a local dataset's summary and the first five values of its first signal.""" arguments = sys.argv[1:] dataset_id = arguments[0] if arguments else "chengsenwang/tsqa" registry = Path(arguments[1]) if len(arguments) > 1 else Path(".timenet-registry") diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py index f736c3209..b45c92bc6 100644 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ b/packages/timenet-connectors/src/timenet_connectors/api.py @@ -107,19 +107,18 @@ def load( fetch_from: "str | BaseRegistry | None" = None, build_parents: bool = False, ) -> "TimeFDataset": - """Build a dataset if needed and load it into memory. + """Build a dataset if needed and load its values into memory. - This runs :func:`build` into the store of :func:`timenet.client.default_sources`, then - :meth:`timenet.client.TimeNet.load` through that list. + The build writes to the default local registry. Args: dataset_id: The dataset ID (``org/name``). - version: The version to load, which the card must declare, or ``None`` for the latest. - fetch_from: The registry, or its URI, to copy missing parents and payload sources from. + version: The version declared in the connector's card, or ``None`` for the latest. + fetch_from: A registry or URI to copy missing dependencies from. build_parents: Build missing parent releases from their installed connectors. Returns: - The loaded dataset. + The dataset with all stored values in memory. """ from timenet.client import TimeNet # ruff: ignore[import-outside-top-level] diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index c2ab2775c..3df7a9670 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -48,18 +48,15 @@ def _root(quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress stat def _resolve_target(out: str | None) -> WritableRegistry | Path: - """Resolve a build's output target: a writable registry for a remote URL, else a local directory. + """Resolve the registry or local directory for build output. - The selection order matches the consumer side, so the CLI that writes a dataset and the SDK that - reads it agree on where it lands: the ``--out`` argument, then ``$TIMENET_REGISTRY``, then the - default local registry. + It tries ``out``, then ``$TIMENET_REGISTRY``, then the default local registry. Args: - out: The ``--out`` value, or ``None`` to fall back to ``$TIMENET_REGISTRY`` then the default. + out: The output URL or path, or ``None`` to use the configured default. Returns: - A :class:`~timenet.registry.WritableRegistry` for a ``timenet://`` / ``http(s)://`` / ``s3://`` - target, or the local directory ``Path`` for a plain path or ``file://`` URI. + A registry for a remote URL, or a ``Path`` for a local directory. """ target = out if out is not None else settings().registry if target is None: diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index d36604d6c..4b27f4610 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -61,13 +61,13 @@ def _enum_list(values: list[str], factory: Callable[[str], T], flag: str, choice def _client(registry: str | None) -> TimeNet: - """Create the client of a command, with the default list of sources. + """Create a client with the default sources and an optional registry. Args: - registry: The value of ``--registry``: the registry after the store, or ``None``. + registry: The registry URL or path to read after the local registry, or ``None`` for the default. Returns: - The client. + A client with the configured sources. """ # TODO(adrian): Remove temporary glue which maps --registry onto default_sources() until the # CLI takes the flags of the defaults. @@ -101,7 +101,7 @@ def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table: @app.command("list") def list_datasets(registry: str | None = _registry_option) -> None: - """List every dataset in the registry.""" + """Display metadata for every dataset in the selected registry.""" rows = _client(registry).list() if not rows: typer.echo("No datasets in the registry.") @@ -121,7 +121,10 @@ def search( # ruff: ignore[too-many-arguments, too-many-positional-arguments] tag: list[str] = typer.Option([], "--tag", help="Tag(s)."), limit: int = typer.Option(100, "--limit", min=0, help="Maximum results."), ) -> None: - """Search datasets by filter. Repeat a flag to pass several values.""" + """Find datasets that match the filters. + + Repeat a filter flag to pass several values. + """ results = _client(registry).search( query=query or None, domain=_enum_list(domain, Domain, "--domain", [d.value for d in Domain]), @@ -202,7 +205,7 @@ def _download(client: TimeNet, dataset_id: str, total: int) -> Path: Args: client: The client to download through. dataset_id: The dataset id, optionally followed by ``@version``. - total: The total byte size of the version's files, for the bar's scale. + total: The total file size in bytes for the progress bar. Returns: The local version directory. diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 1df6bd3c5..0a2a49918 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -1,9 +1,4 @@ -"""The :class:`TimeNet` class is the SDK's single entry point for using TimeNet from code. - -A client reads releases through an ordered list of sources, and builds a release that only -``InstalledConnectors()`` provides into its first local registry that is not read-only. -:func:`default_sources` makes the list of ``TimeNet()``. -""" +"""Browse and load datasets with the :class:`TimeNet` client.""" # The public API has a method named ``list``. Deferred annotations keep the type hint # ``list[str]`` resolved to the builtin type, not to the method. @@ -47,17 +42,17 @@ def default_sources(*, remote_registry: str | Path | None = None) -> list[ReleaseSource]: - """Return the default sources: the store, a registry, and the installed connectors. + """Return the local registry, an optional remote registry, and the installed connectors in that order. - The store is ``$TIMENET_REGISTRY`` if it names a local path, else ``/registry``. - The registry is ``remote_registry``, else ``$TIMENET_REGISTRY`` if it is not local, else - ``timenet://``, and a local ``$TIMENET_REGISTRY`` without ``remote_registry`` leaves it out. + The local registry uses a local ``$TIMENET_REGISTRY`` or ``/registry``. + The optional registry uses ``remote_registry``, a remote ``$TIMENET_REGISTRY``, or ``timenet://`` + if the variable is unset. Args: - remote_registry: The registry after the store, as a URL, a ``file://`` URI or a local path. + remote_registry: A registry URL or path, or ``None`` to use the environment settings. Returns: - A new list, made from the environment at the time of the call. + A new source list based on the current environment. """ # TODO(adrian): Remove temporary glue which maps TIMENET_REGISTRY and TIMENET_HOME onto the # list until the variables of the defaults (TIMENET_LOCAL_REGISTRY, TIMENET_REMOTE_REGISTRY, @@ -99,10 +94,10 @@ def _open_sources(sources: Sequence[ReleaseSource | str | Path] | None) -> tuple """Validate sources and open registry URLs and paths. Args: - sources: The value of ``sources``, other than ``None``. + sources: A nonempty sequence of sources, registry URLs, or paths. Returns: - The sources, in order. + A tuple of sources in the given order. Raises: TimeFValidationError: If sources are not a nonempty sequence of valid sources, registry URLs, or paths. @@ -133,19 +128,17 @@ class TimeNet: """Browse, download, and read TimeNet datasets.""" def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) -> None: - """Create a client that reads releases through an ordered list of sources. + """Create a client with an ordered list of dataset sources. - A source is a registry, a registry URL or path, or ``InstalledConnectors()`` (see - :mod:`timenet.sources`). A path opens a writable :class:`~timenet.registry.LocalRegistry`. - Builds land in the first local registry that is not ``read_only=True``, and a list with - ``InstalledConnectors()`` needs one. + Sources accept registries, registry URLs or paths, and ``InstalledConnectors()``. + A path opens a writable ``LocalRegistry``. Builds land in the first ``LocalRegistry`` without + ``read_only=True``, which ``InstalledConnectors()`` requires. Args: - sources: The sources, in order. ``None`` means :func:`default_sources`. + sources: Sources in read order, or ``None`` to use :func:`default_sources`. Raises: - TimeFValidationError: If ``sources`` is not a valid list, or has ``InstalledConnectors()`` - without a writable local registry. + TimeFValidationError: If sources are invalid, or have connectors but no writable local registry. """ self._sources = tuple(default_sources()) if sources is None else _open_sources(sources) self._store = next( @@ -158,10 +151,9 @@ def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) ) def list(self) -> _Metadatas: - """Return the metadata of every dataset in one registry of the sources. + """Return metadata for every dataset in the selected registry. - The registry is the first registry of the sources that is not the local registry of the - builds, else that local registry. + The client uses the first registry other than the local registry, or the local registry if it is the only one. Returns: One :class:`~timenet.types.DatasetMetadata` per dataset. @@ -169,7 +161,9 @@ def list(self) -> _Metadatas: return self._catalog().list_datasets() def get(self, dataset_id: str, version: str | None = None) -> Manifest: - """Return a dataset's manifest from the registry that :meth:`list` asks. + """Return a dataset version's manifest from the selected registry. + + The client uses the first registry other than the local registry, or the local registry if it is the only one. Args: dataset_id: The dataset ID. @@ -193,18 +187,18 @@ def search( # ruff: ignore[too-many-arguments] tag: _OrList[str] = None, limit: int = 100, ) -> _Metadatas: - """Search the registry that :meth:`list` asks. + """Search the selected registry for datasets that match the filters. - This method mirrors :meth:`~timenet.registry.BaseRegistry.search`. + The client uses the first registry other than the local registry, or the local registry if it is the only one. Args: query: Free text terms for the name, description, and tags. - domain: Keep datasets that share any of these domains. - task: Keep datasets whose schema includes any of these task classes. - license: Keep datasets with any of these licenses. - time_series_spec: Keep datasets that declare all of these ``spec_type`` values. - dataset_id: Keep only these ids. - tag: Keep datasets that declare all of these tags. + domain: Match any of these domains. + task: Match any of these task classes in the schema. + license: Match any of these licenses. + time_series_spec: Match all of these ``spec_type`` values. + dataset_id: Match only these IDs. + tag: Match all of these tags. limit: The maximum number of results. Returns: @@ -229,16 +223,16 @@ def download( force: bool = False, progress_cb: ProgressCallback | None = None, ) -> Path: - """Fetch a version and each of its payload sources from the registry that :meth:`list` asks. + """Download a dataset version and the data needed to read it. - The copy lands in ``$TIMENET_STORAGE``, else in ``/storage``. Parents that - only the build needs are not downloaded. + The client uses the first registry other than the local registry, or the local registry if it is the only one. + Files go to ``$TIMENET_STORAGE`` or ``/storage``. Args: dataset_id: The dataset ID. version: The version string, or ``None`` for the latest. force: Download again, even if an up-to-date copy already exists. - progress_cb: Called with each file's byte count as it lands, for a progress display. + progress_cb: A callback that receives each file's byte count after download. Returns: The local ``///`` directory. @@ -354,14 +348,13 @@ def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: return reader.read(load=LoadPlan.ALL_EAGER) def _catalog(self) -> BaseRegistry: - """Return the registry that :meth:`list`, :meth:`get`, :meth:`search`, and :meth:`download` ask. + """Select a registry from the configured sources. Returns: - The first registry of the sources that is not the local registry of the builds, else - that local registry. + The first registry other than the local registry, or the local registry if it is the only one. Raises: - TimeFValidationError: If the sources hold no registry. + TimeFValidationError: If the sources contain no registry. """ # TODO(adrian): Remove temporary glue which maps these methods onto one registry until # list() and search() ask every source. This registry gives the results of the client of @@ -375,7 +368,7 @@ def _catalog(self) -> BaseRegistry: # TODO(adrian): Remove temporary glue which checks access only for the interim download() # until a later change replaces download(). def _reject_unhosted_access(self, dataset_id: str, version: str | None) -> None: - """Enforce the registry's open access requirement. + """Reject datasets without open access when the registry requires it. Args: dataset_id: The dataset ID without a version suffix. @@ -420,13 +413,13 @@ def open_torch( def _not_provided_hint(sources: Sequence[ReleaseSource]) -> str: - """Tell the caller which source can provide a release that no source provides. + """Suggest sources for a missing dataset release. Args: - sources: The sources of the client. + sources: The configured sources. Returns: - The hint as a full sentence. It names ``InstalledConnectors()`` only if the sources lack it. + Instructions to add a registry or installed connectors. """ if any(isinstance(source, InstalledConnectors) for source in sources): return "Add a registry that holds it to the sources." @@ -434,14 +427,14 @@ def _not_provided_hint(sources: Sequence[ReleaseSource]) -> str: def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: - """Tell the caller how to get a release that a registry lists but does not serve. + """Suggest how to obtain a release whose files are unavailable. Args: - dataset_id: The bare dataset id. - sources: The sources of the client. + dataset_id: The dataset ID without a version suffix. + sources: The configured sources. Returns: - The hint as full sentences. It asks for ``InstalledConnectors()`` only if the sources lack it. + A hint with options to add a source or build the release locally. """ build = f"`timenet-build build {dataset_id}`" if any(isinstance(source, InstalledConnectors) for source in sources): @@ -455,9 +448,9 @@ def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: # TODO(adrian): Remove temporary glue which gives a hint only for the interim download() until a # later change replaces download(). def _unhosted_hint(dataset_id: str) -> str: - """Tell the caller how to get a dataset whose files the download registry does not serve. + """Suggest a local build for a dataset whose registry does not serve its files. Returns: - A sentence with the dataset's build command. + A hint with the build command. """ return f"TimeNet does not host its data, so build it locally with `timenet-build build {dataset_id}`." diff --git a/packages/timenet/src/timenet/registry/factory.py b/packages/timenet/src/timenet/registry/factory.py index 328565b25..117f5483d 100644 --- a/packages/timenet/src/timenet/registry/factory.py +++ b/packages/timenet/src/timenet/registry/factory.py @@ -58,18 +58,17 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None, label def local_registry_path(uri: str | Path, *, label: str = "registry") -> Path: - """Resolve a registry URI to the local directory it names, for build to write into. + """Convert a local registry URL or path to a directory path. Args: uri: A ``file://`` URI or local path. label: The name of the value in an error message, such as ``sources[0]``. Returns: - The local directory the URI names, with ``~`` expanded. + The local directory path with ``~`` expanded. Raises: - TimeNetRegistryError: If the value is blank, names a remote backend, or is not a valid local - registry URI. + TimeNetRegistryError: If ``uri`` is blank, names a remote registry, or is not a valid local URL. """ text = str(uri) if not text.strip(): diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index 2e6d7273e..22dc22664 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -24,13 +24,13 @@ class LocalRegistry(WritableRegistry): - """A registry that reads and writes datasets in a local directory.""" + """Read and write dataset versions in a local directory.""" def __init__(self, root: str | Path, *, read_only: bool = False) -> None: - """Open a local registry rooted at a directory. + """Open a registry in a local directory. Args: - root: The directory containing ``//`` layouts. This expands ``~``. + root: The directory containing ``//`` folders, with ``~`` expanded. read_only: Refuse every write to this registry. """ self._root = Path(root).expanduser() From 5e0655ab368d8f46c9d3a70e59688a05ff71623e Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 10:32:46 +0100 Subject: [PATCH 5/5] test: trim the tests of the list of sources Drop tests that cannot fail or repeat other tests, and sharpen the rest. Part of TNET-157. --- .../timenet/tests/registry/test_remote.py | 5 +- packages/timenet/tests/test_cli.py | 23 ++-- packages/timenet/tests/test_client.py | 128 ++++-------------- 3 files changed, 42 insertions(+), 114 deletions(-) diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index b3bf47166..59c182c16 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -578,9 +578,8 @@ def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: return dataset.metadata.dataset_id, release -@pytest.mark.parametrize("page", ["own", "ancestor"]) -def test_a_read_checks_access_before_it_reads_from_a_remote_registry(tmp_path, page): - dataset_id, release = _gated_release(tmp_path / "origin", page) +def test_a_read_checks_access_before_it_reads_from_a_remote_registry(tmp_path): + dataset_id, release = _gated_release(tmp_path / "origin", "ancestor") transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index 5453f18f5..8e82e141a 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -2,13 +2,13 @@ import subprocess import sys +import click import pytest import typer from typer.testing import CliRunner from timenet.cache import cached_datasets, raw_cache_size from timenet.cli.app import _enum_list, app -from timenet.client import TimeNet from timenet.dataset import TimeFDataset from timenet.errors import TimeNetDatasetNotFoundError from timenet.reader import TimeFReader @@ -80,24 +80,27 @@ def test_cache_clear_all_removes_everything(home): def test_list(registry_root): - assert runner.invoke(app, ["list", "--registry", str(registry_root)]).exit_code == 0 - assert any(d.dataset_id == "timenet/hello-world" for d in TimeNet(sources=[str(registry_root)]).list()) + result = runner.invoke(app, ["list", "--registry", str(registry_root)]) + assert result.exit_code == 0, result.output + assert "timenet/hello-world" in click.unstyle(result.output) def test_search_by_domain(registry_root): - assert runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "general"]).exit_code == 0 - results = TimeNet(sources=[str(registry_root)]).search(domain=[Domain("general")]) - assert any(d.dataset_id == "timenet/hello-world" for d in results) + result = runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "general"]) + assert result.exit_code == 0, result.output + assert "timenet/hello-world" in click.unstyle(result.output) def test_search_no_match(registry_root): - assert runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "cardiology"]).exit_code == 0 - assert TimeNet(sources=[str(registry_root)]).search(domain=[Domain("cardiology")]) == [] + result = runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "cardiology"]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.output) == "No datasets match.\n" def test_info(registry_root): - assert runner.invoke(app, ["info", "timenet/hello-world", "--registry", str(registry_root)]).exit_code == 0 - assert TimeNet(sources=[str(registry_root)]).get("timenet/hello-world").dataset_id == "timenet/hello-world" + result = runner.invoke(app, ["info", "timenet/hello-world", "--registry", str(registry_root)]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.output).startswith("timenet/hello-world") def _write(root, dataset): diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index a88146473..36ba34bd7 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -2,7 +2,6 @@ from types import SimpleNamespace from _demo_child import _child_of_the_demo -from _fake_registry import build_service_fake import httpx from pydantic import ValidationError import pytest @@ -26,7 +25,7 @@ from timenet.registry import TIMENET_REGISTRY_URL, LocalRegistry, RemoteRegistry, open_local_version from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal, install_backends, make_dataset -from timenet.types import Access, DatasetMetadata, Domain, License, Version +from timenet.types import Access, DatasetMetadata, License, Version from timenet.types.access import AccessRequirement from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.reference_keys import ParentAlias @@ -61,7 +60,8 @@ def build(self, release, store, *, force, pinned_by, options): store.put_dataset(dataset, force=force, values_backend=self.BACKEND, pinned_by=pinned_by or None) -def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path): +def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path, monkeypatch): + monkeypatch.setattr(builders_module, "entry_points", lambda group: []) origin, reference = _child_of_the_demo(tmp_path / "origin") cache = tmp_path / "cache" shutil.copytree(origin.root / "test/child/1.0.0", cache / "test/child/1.0.0") @@ -72,31 +72,15 @@ def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path assert str(caught.value).endswith( ". Add a registry that holds it to the sources, or add InstalledConnectors() to build it." ) + # No installed connector builds the payload source, so with InstalledConnectors() the hint names only a registry. + with pytest.raises(TimeNetMissingParentError, match=r"\. Add a registry that holds it to the sources\.$"): + TimeNet(sources=[store, InstalledConnectors()]).open("test/child") with TimeNet(sources=[store, str(origin.root), InstalledConnectors()]).open("test/child") as reader: assert reader.read().records[0].signals[0].to_arrow().equals(make_dataset().records[0].signals[0].to_arrow()) # The read opens the payload source in origin, where it is, and copies nothing. assert not (cache / reference.dataset_id).exists() -@pytest.mark.parametrize("in_origin", [True, False]) -def test_a_missing_version_is_read_from_a_registry_before_it_is_built(tmp_path, monkeypatch, in_origin): - origin = LocalRegistry(tmp_path / "origin") - if in_origin: - _child_of_the_demo(origin.root) - calls = [] - install_backends(monkeypatch, _Builder(calls)) - store = LocalRegistry(tmp_path / "local") - - with TimeNet(sources=[store, origin, InstalledConnectors()]).open("test/child") as reader: - assert reader.read().records - - if in_origin: - assert calls == [] - assert not (tmp_path / "local").exists() - return - assert calls == [(DatasetRef.parse("test/child@1.0.0"), False, {})] - - class _Resampled(BaseConnector[str]): """Writes the first demo Signal at half its rate, from its declared parent, into a Record of its own.""" @@ -152,27 +136,6 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p assert reader.read().records[0].signals[0].to_arrow().equals(expected) -def test_a_list_of_one_remote_registry_never_builds(tmp_path, monkeypatch): - calls = [] - install_backends(monkeypatch, _Builder(calls)) - # The remote registry holds the child, but not its payload source, which the installed backend builds. - origin, reference = _child_of_the_demo(tmp_path / "origin") - shutil.rmtree(origin.root / reference.dataset_id) - transport, _ = build_service_fake(origin.root) - remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") - hint = "Add a registry that holds it to the sources, or add InstalledConnectors() to build it." - - with pytest.raises(TimeNetMissingParentError) as missing: - TimeNet(sources=[remote]).open("test/child") - assert str(missing.value) == ( - f"payload source {reference} of test/child@1.0.0 is not in registry http://api.local. {hint}" - ) - with pytest.raises(TimeNetDatasetNotFoundError) as caught: - TimeNet(sources=[remote]).open(reference.dataset_id) - assert str(caught.value) == f"{reference.dataset_id} is in none of the sources: registry http://api.local. {hint}" - assert calls == [] - - @pytest.fixture(autouse=True) def _clean_env(monkeypatch): for var in ("TIMENET_HOME", "TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): @@ -278,28 +241,17 @@ def test_a_client_rejects_one_source_given_as_a_string(tmp_path): def test_a_client_rejects_one_registry_given_without_a_list(tmp_path): - remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404))) - for registry in (LocalRegistry(tmp_path), remote): - with pytest.raises(TimeFValidationError, match=r"^sources takes a list of sources\. Pass sources=\["): - TimeNet(registry) # ty: ignore[invalid-argument-type] - - -@pytest.mark.parametrize( - ("value", "message"), - [ - ("", "sources[0] is blank; give a registry URL or a local path"), - ( - "ftp://x", - "sources[0] 'ftp://x' has an unsupported registry scheme. " - "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path.", - ), - ], - ids=["blank", "unsupported-scheme"], -) -def test_a_client_rejects_a_string_that_is_not_a_registry(value, message): + with pytest.raises(TimeFValidationError, match=r"^sources takes a list of sources\. Pass sources=\["): + TimeNet(LocalRegistry(tmp_path)) # ty: ignore[invalid-argument-type] + + +def test_a_client_rejects_a_string_that_is_not_a_registry(): with pytest.raises(TimeNetRegistryError) as caught: - TimeNet(sources=[value]) - assert str(caught.value) == message + TimeNet(sources=["ftp://x"]) + assert str(caught.value) == ( + "sources[0] 'ftp://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + ) def test_a_client_rejects_an_empty_list(): @@ -321,15 +273,6 @@ def test_installed_connectors_need_a_local_registry_that_is_not_read_only(tmp_pa ) -def test_a_list_without_installed_connectors_never_builds(registry_root, monkeypatch): - calls = [] - monkeypatch.setattr(_Builder, "VERSION", "2.0.0") - install_backends(monkeypatch, _Builder(calls)) - client = TimeNet(sources=[LocalRegistry(registry_root)]) - assert str(client.load("timenet/hello-world").metadata.dataset_version) == "1.0.0" - assert calls == [] - - def test_get_list_and_search_ask_the_first_registry_that_is_not_the_store(registry_root, tmp_path): store = LocalRegistry(tmp_path / "store") other = make_dataset() @@ -347,17 +290,6 @@ def test_get_list_and_search_ask_the_first_registry_that_is_not_the_store(regist assert [metadata.dataset_id for metadata in alone.list()] == ["test/other"] -def test_list(registry_root): - client = TimeNet(sources=[str(registry_root)]) - assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} - - -def test_search(registry_root): - client = TimeNet(sources=[str(registry_root)]) - assert {m.dataset_id for m in client.search(domain=Domain.GENERAL)} == {"timenet/hello-world"} - assert client.search(domain=Domain.CARDIOLOGY) == [] - - def test_download_copies_into_storage(registry_root, storage): client = TimeNet(sources=[str(registry_root)]) version_dir = client.download("timenet/hello-world") @@ -459,12 +391,15 @@ def counting_close(reader): assert_datasets_equal(expected, restored) -def test_load_reads_in_place_without_downloading(registry_root, tmp_path, storage): +def test_load_reads_in_place_without_downloading(registry_root, tmp_path, storage, monkeypatch): # load() reads the release where it is, not download-then-read, so it must not write a copy into # the store or into local storage the way download() does. + calls = [] + install_backends(monkeypatch, _Builder(calls)) store = tmp_path / "store" client = TimeNet(sources=[LocalRegistry(store), str(registry_root), InstalledConnectors()]) client.load("timenet/hello-world") + assert calls == [] assert not store.exists() # nothing copied into the store assert not (storage / "timenet/hello-world").exists() # nothing fetched into the download cache @@ -511,12 +446,6 @@ def test_open_torch_passes_its_load_plan_to_the_view(registry_root, arrays_loade assert {signal._values.is_loaded for signal in record.signals} == {arrays_loaded} -def test_registry_env_var(registry_root, monkeypatch): - monkeypatch.setenv("TIMENET_REGISTRY", str(registry_root)) - client = TimeNet() - assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} - - @pytest.fixture def versioned_registry(tmp_path): """A registry holding hello_world at both 1.0.0 and 1.1.0.""" @@ -618,13 +547,10 @@ def test_load_accepts_the_latest_sentinel_after_a_build(tmp_path, monkeypatch): assert str(client.load("timenet/hello-world", "latest").metadata.dataset_version) == "1.0.0" -@pytest.mark.parametrize("entry_point", ["open", "load", "open_torch"]) -def test_a_list_without_installed_connectors_does_not_build_a_missing_version(registry_root, monkeypatch, entry_point): +def test_a_list_without_installed_connectors_does_not_build_a_missing_version(registry_root, monkeypatch): # A list without InstalledConnectors() does not build on a miss, so a notebook load fails fast # instead of starting a multi-GB download and a build. The builder declares the requested # version, so only the list keeps it from running. - if entry_point == "open_torch": - pytest.importorskip("torch") calls = [] monkeypatch.setattr(_Builder, "VERSION", "9.9.9") install_backends(monkeypatch, _Builder(calls)) @@ -633,7 +559,7 @@ def test_a_list_without_installed_connectors_does_not_build_a_missing_version(re # The hint names the sources that could provide the release, in a sentence of its own. hint = r"\. Add a registry that holds it to the sources, or add InstalledConnectors\(\) to build it\.$" with pytest.raises(TimeNetDatasetNotFoundError, match=hint): - getattr(client, entry_point)("timenet/hello-world@9.9.9") + client.load("timenet/hello-world@9.9.9") assert calls == [] # the list skips the connector entirely @@ -672,13 +598,13 @@ def test_load_rejects_a_credentialed_dataset_from_a_hosted_registry(tmp_path): "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " "`timenet-build build org/gated`." ) - for entry_point in (client.open, client.load): - with pytest.raises(TimeNetAccessError) as caught: - entry_point("org/gated") - assert str(caught.value) == message + with pytest.raises(TimeNetAccessError) as caught: + client.load("org/gated") + assert str(caught.value) == message -def test_the_access_hint_says_that_no_installed_connector_builds_the_release(tmp_path): +def test_the_access_hint_says_that_no_installed_connector_builds_the_release(tmp_path, monkeypatch): + monkeypatch.setattr(builders_module, "entry_points", lambda group: []) card = _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua") client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), _hosted(tmp_path, card), InstalledConnectors()])