diff --git a/examples/load_tsqa.py b/examples/load_tsqa.py index 77a98e418..9256ee49c 100644 --- a/examples/load_tsqa.py +++ b/examples/load_tsqa.py @@ -1,14 +1,11 @@ -"""Load a built TimeF dataset and print a summary (no pandas or extras needed). +"""Print a local dataset's summary and sample signal values. -Build the dataset into an explicit local registry, then run this script:: +Build the dataset into a local registry, then run this script:: uv run timenet-build build "chengsenwang/tsqa" --out .timenet-registry uv run python examples/load_tsqa.py -``TimeNet(registry).open(...)`` returns a reader, and its ``read()`` returns a ``TimeFDataset``. -``describe()`` prints its identity, counts, per-spec columns, and a record preview. Signal values -stay in storage until you ask for them. Pass a dataset ID as the first argument and a registry path -as the second argument. +Pass a dataset ID and a registry path as arguments to load another dataset. """ from pathlib import Path @@ -18,11 +15,11 @@ def main() -> None: - """Open the dataset, print its describe() summary, and show one raw series slice.""" + """Print a local dataset's summary and the first five values of its first signal.""" arguments = sys.argv[1:] dataset_id = arguments[0] if arguments else "chengsenwang/tsqa" registry = Path(arguments[1]) if len(arguments) > 1 else Path(".timenet-registry") - with TimeNet(registry).open(dataset_id) as reader: + with TimeNet(sources=[registry]).open(dataset_id) as reader: dataset = reader.read() dataset.describe() diff --git a/examples/test_mean_classifier.py b/examples/test_mean_classifier.py index 0090cac99..e99f382d8 100644 --- a/examples/test_mean_classifier.py +++ b/examples/test_mean_classifier.py @@ -24,7 +24,7 @@ # Build and load through the same explicit local registry. registry = Path(".timenet-registry") timenet_connectors.build("timenet/test-mean", out=registry) -dataset = TimeNet(registry).load("timenet/test-mean") +dataset = TimeNet(sources=[registry]).load("timenet/test-mean") # A quick tour of what we loaded: identity, counts, specs, and a record preview. dataset.describe() diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py index f57b12dcf..b45c92bc6 100644 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ b/packages/timenet-connectors/src/timenet_connectors/api.py @@ -107,18 +107,20 @@ def load( fetch_from: "str | BaseRegistry | None" = None, build_parents: bool = False, ) -> "TimeFDataset": - """Build a dataset if needed and load it into memory. + """Build a dataset if needed and load its values into memory. + + The build writes to the default local registry. Args: dataset_id: The dataset ID (``org/name``). - version: The version to load, which the card must declare, or ``None`` for the latest. - fetch_from: The registry or URI that supplies missing parents and payload sources. - build_parents: Build missing parent releases with installed connectors. + version: The version declared in the connector's card, or ``None`` for the latest. + fetch_from: A registry or URI to copy missing dependencies from. + build_parents: Build missing parent releases from their installed connectors. Returns: - The loaded dataset. + The dataset with all stored values in memory. """ from timenet.client import TimeNet # ruff: ignore[import-outside-top-level] build(dataset_id, version=version, fetch_from=fetch_from, build_parents=build_parents) - return TimeNet().load(dataset_id, version, fetch_from=fetch_from) + return TimeNet().load(dataset_id, version) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index a49ac0511..3df7a9670 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -30,7 +30,6 @@ copy_release, local_registry_path, open_registry, - open_writable_registry, ) from timenet.registry._paths import release_dir # ruff: ignore[import-private-name] from timenet.sources import InstalledConnectors @@ -49,18 +48,15 @@ def _root(quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress stat def _resolve_target(out: str | None) -> WritableRegistry | Path: - """Resolve a build's output target: a writable registry for a remote URL, else a local directory. + """Resolve the registry or local directory for build output. - The selection order matches the consumer side, so the CLI that writes a dataset and the SDK that - reads it agree on where it lands: the ``--out`` argument, then ``$TIMENET_REGISTRY``, then the - default local registry. + It tries ``out``, then ``$TIMENET_REGISTRY``, then the default local registry. Args: - out: The ``--out`` value, or ``None`` to fall back to ``$TIMENET_REGISTRY`` then the default. + out: The output URL or path, or ``None`` to use the configured default. Returns: - A :class:`~timenet.registry.WritableRegistry` for a ``timenet://`` / ``http(s)://`` / ``s3://`` - target, or the local directory ``Path`` for a plain path or ``file://`` URI. + A registry for a remote URL, or a ``Path`` for a local directory. """ target = out if out is not None else settings().registry if target is None: @@ -68,7 +64,7 @@ def _resolve_target(out: str | None) -> WritableRegistry | Path: try: return local_registry_path(target) except TimeNetRegistryError: - return open_writable_registry(target) + return open_registry(target) @app.command() diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index 48acf8bf2..89111b6a4 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -31,7 +31,6 @@ from timenet.types.references import DatasetRef from timenet.values_backends import ValuesBackend import timenet_connectors -from timenet_connectors import api as api_module from timenet_connectors.builder import backend as backend_module, cli as cli_module, env as env_module from timenet_connectors.builder.backend import ConnectorBuilder from timenet_connectors.builder.cli import app @@ -567,23 +566,20 @@ def test_load_obtains_a_missing_parent_before_it_builds_and_reads(tmp_path, monk assert calls == (builds if obtain == "fetch_from" else [(PARENT.dataset_id, False), *builds]) -def test_load_reads_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): +def test_load_copies_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): + _install_cards(monkeypatch, parent={"parent": GRAND}) out = LocalRegistry(tmp_path / "out") _parent_without_its_source(out) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) monkeypatch.setenv("TIMENET_REGISTRY", str(out.root)) - # The build has nothing to do, and no builder backend can build the payload source, so only the - # read can get it. - monkeypatch.setattr(api_module, "build", lambda dataset_id, **options: None) - monkeypatch.setattr(builders, "entry_points", lambda group: []) dataset = timenet_connectors.load(PARENT.dataset_id, fetch_from=origin) assert len(dataset.records) == 2 - # The read opens the payload source in origin, where it is, and copies nothing. - assert not out.exists(GRAND.dataset_id, str(GRAND.version)) + # The build copies the payload source into the store, where the read finds it. + assert out.exists(GRAND.dataset_id, str(GRAND.version)) def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch): diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 64dc2a5d6..44fef0388 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -9,7 +9,8 @@ from timenet.engine import run_pipeline from timenet.errors import TimeFValidationError from timenet.provenance import build_env -from timenet.registry import RemoteRegistry +from timenet.registry import LocalRegistry, RemoteRegistry +from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal from timenet_connectors import build, load from timenet_connectors.builder.cli import _resolve_target, app as build_app @@ -53,7 +54,7 @@ def test_build_then_load_round_trips(converted, tmp_path): assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists() # CONSUME: load it back through the SDK and compare to the dataset the build wrote. - restored = TimeNet(registry, storage_path=tmp_path / "store").load("timenet/hello-world") + restored = TimeNet(sources=[str(registry)]).load("timenet/hello-world") [original] = converted assert_datasets_equal(original, restored) @@ -97,7 +98,7 @@ def test_build_honors_timenet_registry(clean_env, monkeypatch, tmp_path): assert runner.invoke(build_app, ["build", "timenet/hello-world", "--no-isolation"]).exit_code == 0 assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists() # The SDK resolves $TIMENET_REGISTRY the same way, so it reads back what the build just wrote. - assert TimeNet(storage_path=tmp_path / "store").list()[0].dataset_id == "timenet/hello-world" + assert TimeNet().list()[0].dataset_id == "timenet/hello-world" def test_build_out_overrides_timenet_registry(clean_env, monkeypatch, tmp_path): @@ -167,7 +168,7 @@ def test_build_cleans_cache_but_keep_flag_retains(tmp_path, monkeypatch): @pytest.mark.slow def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monkeypatch): monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - client = TimeNet(tmp_path / "registry") + client = TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]) dataset = client.load("timenet/hello-world") @@ -178,7 +179,7 @@ def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monk @pytest.mark.slow def test_an_isolated_build_records_its_environment(tmp_path, monkeypatch): monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - TimeNet(tmp_path / "registry").load("timenet/hello-world") + TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]).load("timenet/hello-world") manifest = json.loads((tmp_path / "registry/timenet/hello-world/1.0.0/manifest.json").read_text()) packages = manifest["build_env"]["packages"] diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 0dab74b7c..4b27f4610 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -13,7 +13,7 @@ from timenet.cache import CachedDataset, cached_datasets, clear_cache, human_bytes, raw_cache_size from timenet.cli.runner import run_cli from timenet.cli.ui import console as ui -from timenet.client import TimeNet +from timenet.client import TimeNet, default_sources from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access from timenet.types.tasks import TASKS @@ -22,7 +22,12 @@ app = typer.Typer(help="Browse a TimeNet registry and fetch datasets.", no_args_is_help=True) console = Console() -_registry_option = typer.Option(None, "--registry", "-r", help="Registry URL or path (else $TIMENET_REGISTRY).") +_registry_option = typer.Option( + None, + "--registry", + "-r", + help="Registry URL or path that this command reads (else $TIMENET_REGISTRY, then timenet://).", +) T = TypeVar("T") @@ -55,6 +60,20 @@ def _enum_list(values: list[str], factory: Callable[[str], T], flag: str, choice return parsed or None +def _client(registry: str | None) -> TimeNet: + """Create a client with the default sources and an optional registry. + + Args: + registry: The registry URL or path to read after the local registry, or ``None`` for the default. + + Returns: + A client with the configured sources. + """ + # TODO(adrian): Remove temporary glue which maps --registry onto default_sources() until the + # CLI takes the flags of the defaults. + return TimeNet(sources=default_sources(remote_registry=registry)) + + def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table: """Build a table of dataset metadata (id, name, license, domains). @@ -82,8 +101,8 @@ def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table: @app.command("list") def list_datasets(registry: str | None = _registry_option) -> None: - """List every dataset in the registry.""" - rows = TimeNet(registry).list() + """Display metadata for every dataset in the selected registry.""" + rows = _client(registry).list() if not rows: typer.echo("No datasets in the registry.") return @@ -102,8 +121,11 @@ def search( # ruff: ignore[too-many-arguments, too-many-positional-arguments] tag: list[str] = typer.Option([], "--tag", help="Tag(s)."), limit: int = typer.Option(100, "--limit", min=0, help="Maximum results."), ) -> None: - """Search datasets by filter. Repeat a flag to pass several values.""" - results = TimeNet(registry).search( + """Find datasets that match the filters. + + Repeat a filter flag to pass several values. + """ + results = _client(registry).search( query=query or None, domain=_enum_list(domain, Domain, "--domain", [d.value for d in Domain]), task=_enum_list(task, lambda value: TASKS[TaskType(value)], "--task", [t.value for t in TaskType]), @@ -120,9 +142,9 @@ def search( # ruff: ignore[too-many-arguments, too-many-positional-arguments] @app.command() -def info(dataset_id: str, version: str | None = None, registry: str | None = _registry_option) -> None: - """Print a dataset's manifest summary: metadata, schema, and counts.""" - manifest = TimeNet(registry).get(dataset_id, version) +def info(dataset_id: str, registry: str | None = _registry_option) -> None: + """Display a dataset's metadata, schema, and counts.""" + manifest = _client(registry).get(dataset_id) counts = manifest.counts table = Table(title=manifest.dataset_id, title_justify="left", title_style="bold", show_header=False) table.add_column("Field", style="cyan") @@ -162,37 +184,34 @@ def info(dataset_id: str, version: str | None = None, registry: str | None = _re @app.command() -def download( - dataset_id: str, - version: str | None = None, - registry: str | None = _registry_option, - storage: str | None = typer.Option(None, "--storage", help="Local storage dir (else $TIMENET_STORAGE)."), -) -> None: - """Fetch a complete TimeF version to local storage. Prints its directory to stdout.""" - client = TimeNet(registry, storage_path=storage) - manifest = client.get(dataset_id, version) +def download(dataset_id: str, registry: str | None = _registry_option) -> None: + """Download a dataset version and print its local directory to stdout. + + Files go to ``$TIMENET_STORAGE`` or ``/storage``. + """ + client = _client(registry) + manifest = client.get(dataset_id) files = list(manifest.files.all_files()) total = sum(part.size for part in files) ui.status("⬇️", f"Downloading '{dataset_id}' ({len(files)} files, {human_bytes(total)})…") - target = _download(client, dataset_id, version, total) + target = _download(client, dataset_id, total) ui.success(f"Downloaded '{dataset_id}' ({target.name})") typer.echo(str(target)) -def _download(client: TimeNet, dataset_id: str, version: str | None, total: int) -> Path: +def _download(client: TimeNet, dataset_id: str, total: int) -> Path: """Download a version, rendering a live progress bar on stderr when the terminal supports it. Args: client: The client to download through. - dataset_id: The dataset id. - version: The version string, or ``None`` for the latest. - total: The total byte size of the version's files, for the bar's scale. + dataset_id: The dataset id, optionally followed by ``@version``. + total: The total file size in bytes for the progress bar. Returns: The local version directory. """ if ui.quiet or not sys.stderr.isatty(): - return client.download(dataset_id, version) + return client.download(dataset_id) progress = Progress( TextColumn("[bold blue]{task.description}"), BarColumn(bar_width=None), @@ -207,7 +226,7 @@ def _download(client: TimeNet, dataset_id: str, version: str | None, total: int) ) with progress: task = progress.add_task("downloading", total=total) - return client.download(dataset_id, version, progress_cb=lambda n: progress.advance(task, n)) + return client.download(dataset_id, progress_cb=lambda n: progress.advance(task, n)) cache_app = typer.Typer(help="Inspect and clear the local cache.", no_args_is_help=True) diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index bef66e354..0a2a49918 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -1,9 +1,11 @@ -"""Browse, download, and read datasets with :class:`TimeNet`.""" +"""Browse and load datasets with the :class:`TimeNet` client.""" # The public API has a method named ``list``. Deferred annotations keep the type hint # ``list[str]`` resolved to the builtin type, not to the method. from __future__ import annotations +from collections.abc import Sequence +import os from pathlib import Path from typing import TYPE_CHECKING, TypeAlias, TypeVar @@ -37,7 +39,36 @@ T = TypeVar("T") _Metadatas: TypeAlias = list[DatasetMetadata] _OrList: TypeAlias = T | list[T] | None -_Sources: TypeAlias = list[ReleaseSource] + + +def default_sources(*, remote_registry: str | Path | None = None) -> list[ReleaseSource]: + """Return the local registry, an optional remote registry, and the installed connectors in that order. + + The local registry uses a local ``$TIMENET_REGISTRY`` or ``/registry``. + The optional registry uses ``remote_registry``, a remote ``$TIMENET_REGISTRY``, or ``timenet://`` + if the variable is unset. + + Args: + remote_registry: A registry URL or path, or ``None`` to use the environment settings. + + Returns: + A new source list based on the current environment. + """ + # TODO(adrian): Remove temporary glue which maps TIMENET_REGISTRY and TIMENET_HOME onto the + # list until the variables of the defaults (TIMENET_LOCAL_REGISTRY, TIMENET_REMOTE_REGISTRY, + # TIMENET_AUTO_BUILD) replace them. + cfg = settings() + configured = None if cfg.registry is None else open_registry(cfg.registry) + local = configured if isinstance(configured, LocalRegistry) else None + store = LocalRegistry(cfg.registry_path if local is None else local.root) + if remote_registry is not None: + remote = open_registry(remote_registry) + elif local is not None: + # A client of a local $TIMENET_REGISTRY read no other registry, and neither does this list. + return [store, InstalledConnectors()] + else: + remote = configured if configured is not None else open_registry("timenet://") + return [store, remote, InstalledConnectors()] def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None]: @@ -59,55 +90,90 @@ def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None] return ref_id, ref_version if ref_version is not None else (version or None) +def _open_sources(sources: Sequence[ReleaseSource | str | Path] | None) -> tuple[ReleaseSource, ...]: + """Validate sources and open registry URLs and paths. + + Args: + sources: A nonempty sequence of sources, registry URLs, or paths. + + Returns: + A tuple of sources in the given order. + + Raises: + TimeFValidationError: If sources are not a nonempty sequence of valid sources, registry URLs, or paths. + TimeNetRegistryError: If a registry URL or path is blank or cannot be opened. + """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry + if isinstance(sources, (str, os.PathLike)): + shown = repr(str(sources)) + raise TimeFValidationError( + f"sources takes a list of sources. Pass sources=[{shown}], or " + f"sources=[{shown}, InstalledConnectors()] to build what the path lacks" + ) + if not isinstance(sources, Sequence): + raise TimeFValidationError(f"sources takes a list of sources. Pass sources=[{type(sources).__name__}(...)]") + if not sources: + raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") + opened: list[ReleaseSource] = [] + for index, item in enumerate(sources): + if isinstance(item, (str, os.PathLike)): + opened.append(open_registry(item if isinstance(item, str) else Path(item), label=f"sources[{index}]")) + elif isinstance(item, (BaseRegistry, InstalledConnectors)): + opened.append(item) + else: + raise TimeFValidationError(f"sources[{index}] is not a source, a registry URL or a path: {item!r}") + return tuple(opened) + + class TimeNet: - """Browse, download, and read datasets from a registry.""" + """Browse, download, and read TimeNet datasets.""" - def __init__( - self, registry: str | Path | BaseRegistry | None = None, *, storage_path: str | Path | None = None - ) -> None: - """Open a client for a registry. + def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) -> None: + """Create a client with an ordered list of dataset sources. - The client selects the registry in this order: the ``registry`` argument, then - ``$TIMENET_REGISTRY``, then the hosted TimeNet registry (``timenet://``). + Sources accept registries, registry URLs or paths, and ``InstalledConnectors()``. + A path opens a writable ``LocalRegistry``. Builds land in the first ``LocalRegistry`` without + ``read_only=True``, which ``InstalledConnectors()`` requires. Args: - registry: A registry instance, URL, ``file://`` URI, or local path. - storage_path: The directory for cached downloads. The default is - ``$TIMENET_STORAGE``, or ``/storage`` if that variable is not set. + sources: Sources in read order, or ``None`` to use :func:`default_sources`. + + Raises: + TimeFValidationError: If sources are invalid, or have connectors but no writable local registry. """ - given_registry = registry if isinstance(registry, BaseRegistry) else None - cfg = settings( - registry=None if given_registry is not None or registry is None else str(registry), - storage=storage_path, + self._sources = tuple(default_sources()) if sources is None else _open_sources(sources) + self._store = next( + (source for source in self._sources if isinstance(source, LocalRegistry) and not source.read_only), None ) - if given_registry is not None: - self._registry = given_registry - elif cfg.registry is not None: - self._registry = open_registry(cfg.registry, cache_dir=cfg.storage_dir) - else: - self._registry = open_registry("timenet://", cache_dir=cfg.storage_dir) - self._storage = cfg.storage_dir + if self._store is None and any(isinstance(source, InstalledConnectors) for source in self._sources): + raise TimeFValidationError( + "InstalledConnectors() needs a writable local registry to build into. " + "Add a LocalRegistry without read_only=True to the sources" + ) def list(self) -> _Metadatas: - """Return the metadata of every dataset in the registry. + """Return metadata for every dataset in the selected registry. + + The client uses the first registry other than the local registry, or the local registry if it is the only one. Returns: One :class:`~timenet.types.DatasetMetadata` per dataset. """ - return self._registry.list_datasets() + return self._catalog().list_datasets() def get(self, dataset_id: str, version: str | None = None) -> Manifest: - """Return a dataset's manifest. + """Return a dataset version's manifest from the selected registry. + + The client uses the first registry other than the local registry, or the local registry if it is the only one. Args: - dataset_id: The dataset id. + dataset_id: The dataset ID. version: The version string, or ``None`` for the latest. Returns: The dataset's manifest. """ dataset_id, version = _resolve_ref(dataset_id, version) - return self._registry.get_manifest(dataset_id, version) + return self._catalog().get_manifest(dataset_id, version) def search( # ruff: ignore[too-many-arguments] self, @@ -121,22 +187,24 @@ def search( # ruff: ignore[too-many-arguments] tag: _OrList[str] = None, limit: int = 100, ) -> _Metadatas: - """Search the registry. This method mirrors :meth:`~timenet.registry.BaseRegistry.search`. + """Search the selected registry for datasets that match the filters. + + The client uses the first registry other than the local registry, or the local registry if it is the only one. Args: query: Free text terms for the name, description, and tags. - domain: Keep datasets that share any of these domains. - task: Keep datasets whose schema includes any of these task classes. - license: Keep datasets with any of these licenses. - time_series_spec: Keep datasets that declare all of these ``spec_type`` values. - dataset_id: Keep only these ids. - tag: Keep datasets that declare all of these tags. + domain: Match any of these domains. + task: Match any of these task classes in the schema. + license: Match any of these licenses. + time_series_spec: Match all of these ``spec_type`` values. + dataset_id: Match only these IDs. + tag: Match all of these tags. limit: The maximum number of results. Returns: The matching dataset metadata. """ - return self._registry.search( + return self._catalog().search( query=query, domain=domain, task=task, @@ -155,39 +223,35 @@ def download( force: bool = False, progress_cb: ProgressCallback | None = None, ) -> Path: - """Fetch a version and each of its payload sources into local storage. + """Download a dataset version and the data needed to read it. - Build-only parents remain recorded in the manifest and are not downloaded. + The client uses the first registry other than the local registry, or the local registry if it is the only one. + Files go to ``$TIMENET_STORAGE`` or ``/storage``. Args: - dataset_id: The dataset id. + dataset_id: The dataset ID. version: The version string, or ``None`` for the latest. force: Download again, even if an up-to-date copy already exists. - progress_cb: Called with each file's byte count as it lands, for a progress display. + progress_cb: A callback that receives each file's byte count after download. Returns: The local ``///`` directory. """ + # TODO(adrian): Remove temporary glue which maps download() onto a copy from one registry + # into the storage directory, as before the list of sources, until a download into the + # local registry replaces it. dataset_id, version = _resolve_ref(dataset_id, version) + registry = self._catalog() self._reject_unhosted_access(dataset_id, version) - manifest = self._registry.get_manifest(dataset_id, version) + manifest = registry.get_manifest(dataset_id, version) resolved = str(manifest.metadata.dataset_version) - target = release_dir(self._storage, DatasetRef.parse(f"{dataset_id}@{resolved}")) + target = release_dir(settings().storage_dir, DatasetRef.parse(f"{dataset_id}@{resolved}")) # The registry owns the fetch: the base implementation stages each file and swaps atomically, # and a remote registry overrides it to resolve and stream every file in parallel. - self._registry.download_version( - dataset_id, resolved, target, force=force, manifest=manifest, progress_cb=progress_cb - ) + registry.download_version(dataset_id, resolved, target, force=force, manifest=manifest, progress_cb=progress_cb) return target - def open( - self, - dataset_id: str, - version: str | None = None, - *, - auto_build: bool = True, - fetch_from: str | BaseRegistry | None = None, - ) -> TimeFReader: + def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: """Open a dataset version for lazy reads, where it is. Opening reads the version's manifest, but neither its catalog nor any value. The reader's @@ -196,15 +260,14 @@ def open( from their manifests upfront and read payload ranges on demand, reusing complete cached releases. - The call reads each release where it is, and copies nothing. A local client asks its own - registry first, then the registry that ``fetch_from`` names, then the installed connectors - if ``auto_build`` is set. Without a version, the call reads the newest version of the first - of them that serves the dataset. An exact version comes from the first of them that serves - it, and ``"latest"`` is the highest version that any of them provides. Each payload source - of the version resolves on its own, at its exact version. If the call builds the version, - the build also needs the declared parents of the version and their payload sources. + The call reads each release where it is, and copies nothing. Without a version, it reads the + newest version of the first source that serves the dataset. An exact version comes from the + first source that serves it, and ``"latest"`` is the highest version of any source. Each + payload source of the version resolves on its own, at its exact version. If the call builds + the version, the build also needs the declared parents of the version and their payload + sources. - If only a connector provides a release, the call builds it into the local registry first. + If only ``InstalledConnectors()`` provides a release, the call builds it into the local registry first. The call builds a payload source with its own connector, at the exact version of the source. The build gets the pins of the version. The build runs in staging. It publishes the source only if it reproduces the pinned files. @@ -212,67 +275,49 @@ def open( A local copy is damaged if its manifest is present but a file is missing or has another size. The call checks each release that it opens in a local registry before it reads or builds anything. It never reads past a damaged copy to a later source, and it never builds - a release over a damaged copy. Against a remote registry, ``open`` never runs - connector code. If a remote client gets ``fetch_from``, ``open`` raises an error. + a release over a damaged copy. Args: dataset_id: The dataset id. version: An exact version, ``"latest"``, or ``None`` for no version. - auto_build: Build a release that only an installed connector provides. Set false to - raise instead. - fetch_from: A registry, or its URI, that a local client reads a release from if its own - registry does not provide it. Returns: A reader for the version. Raises: - TimeNetAccessError: If a registry that is not local lists the version or a payload - source but does not serve its files, and no later source provides it. - TimeNetDatasetNotFoundError: If no source provides the version. + TimeNetAccessError: If a registry lists the version or a payload source but does not + serve its files, and no later source provides it. + TimeNetDatasetNotFoundError: If no source provides the version. The message says which + source to add. TimeNetVersionUnavailableError: If no source provides the version, and the installed connector declares another version. The build does not start then. TimeNetInvalidCardError: If the card of a connector cannot be read. TimeNetMissingParentError: If no source provides a payload source, or a release that a - build needs. The message names each keyword that the call did not use. + build needs. The message says which source to add. TimeNetFileError: If a release that the call opens in a local registry has a missing file or a file of another size. TimeFFormatError: If a payload source differs from the files that the version pins, for example because its build does not reproduce them. TimeNetBuildError: If a build runs but fails. - TimeFValidationError: If a remote client gets ``fetch_from``. """ # ruff: ignore[docstring-extraneous-exception] (raised by read_release and the builder) dataset_id, version = _resolve_ref(dataset_id, version) - sources, store = self._read_sources(auto_build=auto_build, fetch_from=fetch_from) - # TODO(adrian): Remove temporary glue which maps fetch_from and auto_build onto hints for - # a local client until the client takes a list of sources. try: - return TimeFReader(read_release(dataset_id, version, sources=sources, store=store)) + return TimeFReader(read_release(dataset_id, version, sources=self._sources, store=self._store)) except TimeNetAccessError as error: - if store is None: - error.hint = _unhosted_hint(dataset_id) - elif not auto_build: - error.hint = "Build it from its connector with auto_build=True." + error.hint = _access_hint(dataset_id, self._sources) raise except TimeNetMissingParentError as error: - if store is not None: - error.hint = _missing_hint(auto_build=auto_build, fetch_from=fetch_from) + error.hint = _not_provided_hint(self._sources) raise except TimeNetDatasetNotFoundError as error: - hint = None if store is None else _missing_hint(auto_build=auto_build, fetch_from=fetch_from) # A subclass, such as TimeNetVersionUnavailableError, names its own fix. - if hint is None or type(error) is not TimeNetDatasetNotFoundError: + if type(error) is not TimeNetDatasetNotFoundError: raise - raise TimeNetDatasetNotFoundError(f"{str(error).rstrip('.')}. {hint}") from error + raise TimeNetDatasetNotFoundError( + f"{str(error).rstrip('.')}. {_not_provided_hint(self._sources)}" + ) from error - def load( - self, - dataset_id: str, - version: str | None = None, - *, - auto_build: bool = True, - fetch_from: str | BaseRegistry | None = None, - ) -> TimeFDataset: + def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: """Read a whole dataset version into memory, with every stored value loaded. The dataset needs no storage afterwards: its reader is closed before it is returned. For a @@ -281,17 +326,13 @@ def load( Args: dataset_id: The dataset id. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. - auto_build: Build a release that only an installed connector provides. Set false to - raise instead. - fetch_from: A registry, or its URI, that a local client reads a release from if its own - registry does not provide it. Returns: The dataset with every signal array, time offset, and text value loaded. Raises: - TimeNetAccessError: If a registry that is not local lists the version or a payload - source but does not serve its files, and no later source provides it. + TimeNetAccessError: If a registry lists the version or a payload source but does not + serve its files, and no later source provides it. TimeNetDatasetNotFoundError: If no source provides the version. TimeNetVersionUnavailableError: If no source provides the version, and the installed connector declares another version. The build does not start then. @@ -302,42 +343,32 @@ def load( file or a file of another size. TimeFFormatError: If a payload source differs from the files that the version pins. TimeNetBuildError: If a build runs but fails. - TimeFValidationError: If a remote client gets ``fetch_from``. """ # ruff: ignore[docstring-extraneous-exception] (all come from open()) - with self.open(dataset_id, version, auto_build=auto_build, fetch_from=fetch_from) as reader: + with self.open(dataset_id, version) as reader: return reader.read(load=LoadPlan.ALL_EAGER) - def _read_sources( - self, *, auto_build: bool, fetch_from: str | BaseRegistry | None - ) -> tuple[_Sources, LocalRegistry | None]: - """Return the ordered sources for a read and the local registry for builds. - - Remote reads use only the configured registry. + def _catalog(self) -> BaseRegistry: + """Select a registry from the configured sources. Returns: - The source list, and the local registry for builds or ``None`` for a remote client. + The first registry other than the local registry, or the local registry if it is the only one. Raises: - TimeFValidationError: If ``fetch_from`` is set for a remote registry. + TimeFValidationError: If the sources contain no registry. """ - registry = self._registry - if not isinstance(registry, LocalRegistry): - if fetch_from is not None: - raise TimeFValidationError( - "fetch_from adds a registry to the reads of a local registry; a remote client reads from its " - "own registry" - ) - return [registry], None - source = open_registry(fetch_from) if isinstance(fetch_from, str) else fetch_from - sources: _Sources = [registry] - if source is not None: - sources.append(source) - if auto_build: - sources.append(InstalledConnectors()) - return sources, registry - + # TODO(adrian): Remove temporary glue which maps these methods onto one registry until + # list() and search() ask every source. This registry gives the results of the client of + # one registry, from before the list of sources. + registries = [source for source in self._sources if isinstance(source, BaseRegistry)] + registry = next((registry for registry in registries if registry is not self._store), self._store) + if registry is None: + raise TimeFValidationError("list(), get(), search() and download() need a registry in the sources") + return registry + + # TODO(adrian): Remove temporary glue which checks access only for the interim download() + # until a later change replaces download(). def _reject_unhosted_access(self, dataset_id: str, version: str | None) -> None: - """Enforce the registry's open access requirement. + """Reject datasets without open access when the registry requires it. Args: dataset_id: The dataset ID without a version suffix. @@ -346,23 +377,18 @@ def _reject_unhosted_access(self, dataset_id: str, version: str | None) -> None: Raises: TimeNetAccessError: If the dataset lacks open access and the registry requires it. """ - if not self._registry.requires_open_access: + registry = self._catalog() + if not registry.requires_open_access: return try: - manifest = self._registry.get_manifest(dataset_id, version) + manifest = registry.get_manifest(dataset_id, version) require_open_access(manifest.metadata, manifest.lineage) except TimeNetAccessError as error: error.hint = _unhosted_hint(dataset_id) raise def open_torch( - self, - dataset_id: str, - version: str | None = None, - *, - auto_build: bool = True, - fetch_from: str | BaseRegistry | None = None, - load: LoadPlan = LoadPlan.ALL_EAGER, + self, dataset_id: str, version: str | None = None, *, load: LoadPlan = LoadPlan.ALL_EAGER ) -> TimeFTorchDataset: """Open a dataset version as a read-only PyTorch ``Dataset`` that reads it a batch at a time. @@ -374,10 +400,6 @@ def open_torch( Args: dataset_id: The dataset id. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. - auto_build: Build a release that only an installed connector provides. Set false to - raise instead. - fetch_from: A registry, or its URI, that a local client reads a release from if its own - registry does not provide it. load: The plan that each batch loads with. A text-only plan, for example, skips the arrays of candidate and target Records. @@ -387,32 +409,48 @@ def open_torch( """ from timenet.torch import TimeFTorchDataset # ruff: ignore[import-outside-top-level] - return TimeFTorchDataset.from_reader( - self.open(dataset_id, version, auto_build=auto_build, fetch_from=fetch_from), load=load - ) + return TimeFTorchDataset.from_reader(self.open(dataset_id, version), load=load) -def _unhosted_hint(dataset_id: str) -> str: - """Return instructions for building a dataset locally. +def _not_provided_hint(sources: Sequence[ReleaseSource]) -> str: + """Suggest sources for a missing dataset release. + + Args: + sources: The configured sources. Returns: - A sentence with the dataset's build command. + Instructions to add a registry or installed connectors. """ - return f"TimeNet does not host its data, so build it locally with `timenet-build build {dataset_id}`." + if any(isinstance(source, InstalledConnectors) for source in sources): + return "Add a registry that holds it to the sources." + return "Add a registry that holds it to the sources, or add InstalledConnectors() to build it." + + +def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: + """Suggest how to obtain a release whose files are unavailable. + + Args: + dataset_id: The dataset ID without a version suffix. + sources: The configured sources. + Returns: + A hint with options to add a source or build the release locally. + """ + build = f"`timenet-build build {dataset_id}`" + if any(isinstance(source, InstalledConnectors) for source in sources): + return ( + "No source serves its files, and no installed connector builds this release. " + f"Install its connector package and build it locally with {build}." + ) + return f"No source serves its files. Add InstalledConnectors() to the sources, or build it locally with {build}." -def _missing_hint(*, auto_build: bool, fetch_from: str | BaseRegistry | None) -> str | None: - """Suggest options for reading or building a missing release. + +# TODO(adrian): Remove temporary glue which gives a hint only for the interim download() until a +# later change replaces download(). +def _unhosted_hint(dataset_id: str) -> str: + """Suggest a local build for a dataset whose registry does not serve its files. Returns: - A sentence with unused options, or ``None`` if both options are already enabled. + A hint with the build command. """ - ways = [] - if fetch_from is None: - ways.append("read it from a registry that holds it with fetch_from=") - if not auto_build: - ways.append("build it from its connector with auto_build=True") - if not ways: - return None - hint = " or ".join(ways) - return f"{hint[0].upper()}{hint[1:]}." + return f"TimeNet does not host its data, so build it locally with `timenet-build build {dataset_id}`." diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index 65ed5e381..6ac98f8e4 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -64,10 +64,13 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = options: Build settings, including the progress callback. Raises: + TimeNetRegistryError: If the plan copies or builds a release and ``store`` is read-only. TimeNetVersionConflictError: If a copied release conflicts with the files in ``store``. TimeFFormatError: If a build changes files pinned by other releases. TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the copy and the backends + if any(not isinstance(step, ReadStep) for step in plan.steps): + store.check_writable("the copies and builds of a plan") options = options or BuildOptions() for step in plan.steps: match step: diff --git a/packages/timenet/src/timenet/registry/__init__.py b/packages/timenet/src/timenet/registry/__init__.py index bc13fea69..4ebfb5825 100644 --- a/packages/timenet/src/timenet/registry/__init__.py +++ b/packages/timenet/src/timenet/registry/__init__.py @@ -8,7 +8,6 @@ default_registry_path, local_registry_path, open_registry, - open_writable_registry, ) from timenet.registry.local import LocalRegistry from timenet.registry.remote import RemoteRegistry @@ -30,5 +29,4 @@ "local_registry_path", "open_local_version", "open_registry", - "open_writable_registry", ] diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index db7ad0b6f..f29131ebb 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -40,12 +40,14 @@ def copy_release( progress: Callback for ``StepStarted`` and ``StepFinished`` events for each copied release. Raises: + TimeNetRegistryError: If ``dest`` is read-only. TimeNetVersionConflictError: If ``dest`` holds other files or another card for a release being copied. TimeNetAccessError: If a release fails an open access requirement of ``dest`` or ``source``. TimeNetDatasetNotFoundError: If ``source`` does not hold the release. TimeNetMissingParentError: If ``source`` lacks a payload source and ``dest`` holds no complete copy of it. TimeFFormatError: If a manifest names a different release or a pin fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the helpers + dest.check_writable(f"a copy of {reference}") manifests = _manifests(reference, source, dest, with_payload_sources=with_payload_sources) # Check every release before anything is copied, so that a refusal leaves dest unchanged. A payload # source that only dest holds is complete there, so it is never pending. diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index aa4c0364f..81e7be8d4 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -13,7 +13,7 @@ from typing import BinaryIO, TypeVar from timenet._fs import staged_directory -from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError +from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError, TimeNetRegistryError from timenet.format.checksums import file_checksum from timenet.format.constants import MANIFEST_FILE from timenet.manifest import FilePart, Manifest @@ -41,9 +41,30 @@ class BaseRegistry(ABC): sources. :func:`~timenet.registry.copy_release` copies a release with its payload sources. """ + _read_only: bool = False + def __str__(self) -> str: return type(self).__name__ + @property + def read_only(self) -> bool: + """Whether this registry refuses every write, set with ``read_only=True``.""" + return self._read_only + + def check_writable(self, write: str) -> None: + """Refuse a write if this registry is read-only. + + Args: + write: The write to refuse, for the message, such as ``"put_dataset()"``. + + Raises: + TimeNetRegistryError: If this registry is read-only. + """ + if self._read_only: + raise TimeNetRegistryError( + f"{self} has read_only=True, so it refuses {write}. Open it without read_only=True to write to it." + ) + @property @abstractmethod def local_root(self) -> Path: diff --git a/packages/timenet/src/timenet/registry/factory.py b/packages/timenet/src/timenet/registry/factory.py index 506fef9f9..117f5483d 100644 --- a/packages/timenet/src/timenet/registry/factory.py +++ b/packages/timenet/src/timenet/registry/factory.py @@ -6,7 +6,6 @@ from timenet.config import settings from timenet.errors import TimeNetRegistryError -from timenet.registry.base import BaseRegistry from timenet.registry.local import LocalRegistry from timenet.registry.remote import RemoteRegistry from timenet.registry.s3 import S3Registry @@ -19,7 +18,10 @@ _REMOTE_SCHEMES = ("timenet://", "http://", "https://", "s3://") -def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None) -> BaseRegistry: +_SCHEMES = "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + + +def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None, label: str = "registry") -> WritableRegistry: """Open a registry from a URI or path. The scheme selects the backend. ``http(s)://`` and ``timenet://`` open a :class:`RemoteRegistry`, @@ -30,14 +32,15 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None) -> Ba uri: A URL, ``timenet://`` / ``s3://`` / ``file://`` URI, or local path. cache_dir: Where a remote backend caches downloads. The local backend ignores it. Defaults to the configured storage directory when ``None``. + label: The name of the value in an error message, such as ``sources[0]``. Returns: The matching registry backend. Raises: - ValueError: If ``uri`` carries a scheme no backend handles, if a ``timenet://`` URI carries a - path, or if ``uri`` is a ``file://`` URI with a host component (the host would be dropped - without notice). + TimeNetRegistryError: If ``uri`` is blank or carries a scheme no backend handles, if a + ``timenet://`` URI carries a path, or if ``uri`` is a ``file://`` URI with a host component + (the host would be dropped without notice) or without a path. """ text = str(uri) if text.startswith("timenet://"): @@ -45,68 +48,43 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None) -> Ba # passed to the client methods, not folded into the registry URI). Reject a stray remainder # rather than drop it silently, so a mistyped URI fails loudly instead of hitting the default host. if text[len("timenet://") :].strip("/"): - raise ValueError(f"timenet:// takes no path; got {text!r}") + raise TimeNetRegistryError(f"{label} {text!r}: timenet:// takes no path") return RemoteRegistry(TIMENET_REGISTRY_URL, cache_dir=cache_dir) if text.startswith(("http://", "https://")): return RemoteRegistry(text, cache_dir=cache_dir) if text.startswith("s3://"): return S3Registry(text, cache_dir=cache_dir) - if text.startswith("file://"): - parsed = urlparse(text) - if parsed.netloc: - raise ValueError(f"file:// registry URI must be absolute (three slashes), got {text!r}") - return LocalRegistry(Path(url2pathname(parsed.path)).expanduser()) - if "://" in text: - raise ValueError(f"unsupported registry scheme in {text!r}") - return LocalRegistry(Path(text).expanduser()) - - -def open_writable_registry(uri: str | Path) -> WritableRegistry: - """Open a writable registry from a URI or local path. - - Args: - uri: A URL, ``timenet://`` / ``s3://`` / ``file://`` URI, or local path. - - Returns: - The writable registry for the URI or path. - - Raises: - ValueError: If the registry does not support writing. - """ - registry = open_registry(uri) - if not isinstance(registry, WritableRegistry): - raise ValueError(f"registry {uri!r} is not writable") - return registry - + return LocalRegistry(local_registry_path(text, label=label)) -def local_registry_path(uri: str | Path) -> Path: - """Resolve a registry URI to the local directory it names, for build to write into. - Every backend is a :class:`WritableRegistry`, so :func:`open_writable_registry` cannot separate a - directory the engine can write to from a remote stub. This function can. +def local_registry_path(uri: str | Path, *, label: str = "registry") -> Path: + """Convert a local registry URL or path to a directory path. Args: uri: A ``file://`` URI or local path. + label: The name of the value in an error message, such as ``sources[0]``. Returns: - The local directory the URI names, with ``~`` expanded. + The local directory path with ``~`` expanded. Raises: - TimeNetRegistryError: If the URI names a remote backend or carries an unsupported scheme, which - build cannot write to. + TimeNetRegistryError: If ``uri`` is blank, names a remote registry, or is not a valid local URL. """ text = str(uri) + if not text.strip(): + # Path("") is the current directory, which a blank value never means. + raise TimeNetRegistryError(f"{label} is blank; give a registry URL or a local path") if text.startswith(_REMOTE_SCHEMES): - raise TimeNetRegistryError(f"registry {text!r} is remote; build writes to a local directory") + raise TimeNetRegistryError(f"{label} {text!r} is remote; build writes to a local directory") if text.startswith("file://"): parsed = urlparse(text) if parsed.netloc: - raise TimeNetRegistryError(f"file:// registry URI must be absolute (three slashes), got {text!r}") + raise TimeNetRegistryError(f"{label} {text!r} must be an absolute file:// URI (three slashes). {_SCHEMES}") if not parsed.path: - raise TimeNetRegistryError("file:// registry URI must name an absolute path") + raise TimeNetRegistryError(f"{label} {text!r} must name an absolute path") return Path(url2pathname(parsed.path)).expanduser() if "://" in text: - raise TimeNetRegistryError(f"unsupported registry scheme in {text!r}") + raise TimeNetRegistryError(f"{label} {text!r} has an unsupported registry scheme. {_SCHEMES}") return Path(text).expanduser() diff --git a/packages/timenet/src/timenet/registry/lazy_read.py b/packages/timenet/src/timenet/registry/lazy_read.py index 9368a596e..32654c6f6 100644 --- a/packages/timenet/src/timenet/registry/lazy_read.py +++ b/packages/timenet/src/timenet/registry/lazy_read.py @@ -52,8 +52,10 @@ def put_dataset( A reference to the stored release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeFValidationError: If ``force`` is set. - """ + """ # ruff: ignore[docstring-extraneous-exception] - raised by check_writable + self.check_writable("put_dataset()") if force: raise TimeFValidationError( f"force=True needs a local registry: {self} never writes a published version again" diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index ab4e8fb43..22dc22664 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -24,15 +24,17 @@ class LocalRegistry(WritableRegistry): - """A registry that reads and writes datasets in a local directory.""" + """Read and write dataset versions in a local directory.""" - def __init__(self, root: Path) -> None: - """Open a local registry rooted at a directory. + def __init__(self, root: str | Path, *, read_only: bool = False) -> None: + """Open a registry in a local directory. Args: - root: The directory containing ``//`` layouts. This expands ``~``. + root: The directory containing ``//`` folders, with ``~`` expanded. + read_only: Refuse every write to this registry. """ self._root = Path(root).expanduser() + self._read_only = read_only # (dataset_id, version) -> (manifest mtime, parsed manifest). This avoids re-parsing on repeat # reads, for example list_datasets then search's schema filter. An mtime change invalidates it. self._manifest_cache: dict[tuple[str, str], tuple[float, Manifest]] = {} @@ -200,9 +202,11 @@ def put_dataset( A reference to the stored release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeNetDatasetNotFoundError: If a release that stores reused data is missing from this registry. TimeFFormatError: If a source release or the new output fails a file pin check. """ # ruff: ignore[docstring-extraneous-exception] - the writer checks payload sources before it publishes + self.check_writable("put_dataset()") if dataset.schema is None: dataset.derive_schema() version = str(dataset.metadata.dataset_version) @@ -236,10 +240,12 @@ def put_release(self, from_directory: Path) -> DatasetRef: A reference to the published release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. OSError: If copying or renaming fails, with the backup path if restoring the old directory also fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release and publish_directory + self.check_writable("put_release()") manifest = self._read_release(from_directory) reference = DatasetRef(dataset_id=manifest.dataset_id, version=manifest.metadata.dataset_version) with staged_directory(release_dir(self._root, reference)) as staging: diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index ad093591c..692e3d8a7 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -41,6 +41,7 @@ def __init__( token: str | None = None, cache_dir: str | Path | None = None, transport: httpx.BaseTransport | None = None, + read_only: bool = False, ) -> None: """Open a remote registry. @@ -49,8 +50,10 @@ def __init__( token: Bearer token. Defaults to ``$TIMENET_TOKEN`` (``None`` is anonymous). cache_dir: Where downloads are cached; defaults to the configured storage directory. transport: An httpx transport for testing; ``None`` uses the network. + read_only: Refuse every write to this registry. """ cfg = settings() + self._read_only = read_only self._base_url = base_url self._http = RegistryHttpClient(base_url, token=token if token is not None else cfg.token, transport=transport) self._cache_dir = Path(cache_dir) if cache_dir is not None else cfg.storage_dir @@ -175,8 +178,10 @@ def put_release(self, from_directory: Path) -> DatasetRef: Raises: TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. - TimeNetRegistryError: If the service skips a listed file or requests a missing file. + TimeNetRegistryError: If this registry is read-only, or the service skips a listed file or + requests a missing file. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release + self.check_writable("put_release()") manifest = self._read_release(from_directory) dataset_id = manifest.dataset_id version = str(manifest.metadata.dataset_version) diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 4bdf97d5f..1ea8a46fe 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -42,12 +42,13 @@ class S3Registry(LazyReadRegistry): _staging_prefix = "timenet-s3-" - def __init__(self, uri: str, *, cache_dir: str | Path | None = None) -> None: + def __init__(self, uri: str, *, cache_dir: str | Path | None = None, read_only: bool = False) -> None: """Open an S3-backed registry. Args: uri: The bucket or prefix root, for example ``s3://my-bucket/registry``. cache_dir: Where downloads are cached; defaults to the configured storage directory. + read_only: Refuse every write to this registry. Raises: TimeNetRegistryError: If ``uri`` is not an ``s3://bucket[/prefix]`` URL. @@ -58,6 +59,7 @@ def __init__(self, uri: str, *, cache_dir: str | Path | None = None) -> None: self._bucket = parsed.netloc self._prefix = parsed.path.strip("/") self._cache_dir = Path(cache_dir) if cache_dir is not None else settings().storage_dir + self._read_only = read_only def __str__(self) -> str: return f"S3 registry {self._display_root()}" @@ -168,9 +170,11 @@ def put_release(self, from_directory: Path) -> DatasetRef: A reference to the published release. Raises: + TimeNetRegistryError: If this registry is read-only. TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release + self.check_writable("put_release()") manifest = self._read_release(from_directory) client = self._client() final_base = self._key(manifest.dataset_id, str(manifest.metadata.dataset_version)) diff --git a/packages/timenet/tests/coordinator/test_build.py b/packages/timenet/tests/coordinator/test_build.py index 7f8e95ace..c957ffd8c 100644 --- a/packages/timenet/tests/coordinator/test_build.py +++ b/packages/timenet/tests/coordinator/test_build.py @@ -5,9 +5,9 @@ from timenet import builders from timenet.builders import BuildOptions from timenet.coordinator.build import plan_release_build, run_plan -from timenet.coordinator.plan import BuildStep, CopyStep +from timenet.coordinator.plan import BuildStep, CopyStep, Plan from timenet.dataset import TimeFDataset -from timenet.errors import TimeNetAccessError, TimeNetVersionUnavailableError +from timenet.errors import TimeNetAccessError, TimeNetRegistryError, TimeNetVersionUnavailableError from timenet.progress import StepFinished, StepStarted from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version @@ -125,6 +125,13 @@ def test_plan_release_build_rejects_a_version_the_connector_does_not_declare(sto assert (caught.value.release, caught.value.declared) == (DatasetRef.parse("a/b@2.0.0"), REF_AB.version) +def test_a_plan_that_builds_refuses_a_read_only_local_registry_before_any_step(tmp_path, fake): + store = LocalRegistry(tmp_path / "store", read_only=True) + with pytest.raises(TimeNetRegistryError, match=r"read_only=True, so it refuses the copies and builds of a plan\."): + run_plan(Plan((_build(REF_AB),)), store) + assert fake.calls == [] + + def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, store, fake_parent, fake_child): origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) diff --git a/packages/timenet/tests/reader/test_reader.py b/packages/timenet/tests/reader/test_reader.py index 473138274..e7a0559e7 100644 --- a/packages/timenet/tests/reader/test_reader.py +++ b/packages/timenet/tests/reader/test_reader.py @@ -374,7 +374,7 @@ def counting_open(*args, **kwargs): if entry_point == "reader": reader = TimeFReader(open_local_version(version_dir)) elif entry_point == "client": - reader = TimeNet(tmp_path, storage_path=tmp_path / "store").open("timenet/hello-world") + reader = TimeNet(sources=[str(tmp_path)]).open("timenet/hello-world") else: reader = TimeFReader.open(version_dir) with reader: diff --git a/packages/timenet/tests/registry/test_registry.py b/packages/timenet/tests/registry/test_registry.py index 7459a30ba..d7eeda740 100644 --- a/packages/timenet/tests/registry/test_registry.py +++ b/packages/timenet/tests/registry/test_registry.py @@ -13,11 +13,10 @@ LocalRegistry, RemoteRegistry, S3Registry, - WritableRegistry, + copy_release, local_registry_path, open_local_version, open_registry, - open_writable_registry, ) from timenet.testing import make_dataset from timenet.types import Annotation, AnswerTask, Domain, License @@ -56,7 +55,7 @@ def test_open_registry_timenet_scheme_aliases_hosted_remote(): def test_open_registry_timenet_scheme_rejects_a_path(): # A path after timenet:// used to be dropped silently; now it is rejected, so a mistyped registry # URI fails loudly instead of hitting the default host. - with pytest.raises(ValueError, match="timenet:// takes no path"): + with pytest.raises(TimeNetRegistryError, match="timenet:// takes no path"): open_registry("timenet://chengsenwang/tsqa") @@ -66,21 +65,22 @@ def test_open_registry_bare_timenet_scheme(): assert registry._base_url == TIMENET_REGISTRY_URL -def test_open_writable_registry_returns_writable(registry_root): - assert isinstance(open_writable_registry(registry_root), WritableRegistry) - - def test_open_registry_unknown_scheme_rejected(): # must not fall through to a LocalRegistry rooted at the literal "gs://bucket" string - with pytest.raises(ValueError, match="unsupported registry scheme"): + with pytest.raises(TimeNetRegistryError, match="unsupported registry scheme"): open_registry("gs://bucket/registry") def test_open_registry_file_uri_with_host_rejected(): - with pytest.raises(ValueError, match="absolute"): + with pytest.raises(TimeNetRegistryError, match="absolute"): open_registry("file://home/timo/registry") +def test_open_registry_rejects_a_blank_value_instead_of_opening_the_current_directory(): + with pytest.raises(TimeNetRegistryError, match=r"^--registry is blank; give a registry URL or a local path$"): + open_registry(" ", label="--registry") + + def test_open_registry_expands_user(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) registry = open_registry("~/registry") @@ -338,3 +338,16 @@ def test_s3_registry_has_no_catalog(): s3.list_datasets() with pytest.raises(NotImplementedError): s3.search() + + +def test_a_read_only_registry_refuses_put_dataset_and_a_copy_into_it(tmp_path): + origin = LocalRegistry(tmp_path / "origin") + release = origin.put_dataset(make_dataset()) + registry = LocalRegistry(tmp_path / "read-only", read_only=True) + refused = rf"^local registry {re.escape(str(registry.root))} has read_only=True, so it refuses " + + with pytest.raises(TimeNetRegistryError, match=refused + r"put_dataset\(\)\."): + registry.put_dataset(make_dataset()) + with pytest.raises(TimeNetRegistryError, match=refused + r"a copy of timenet/hello-world@1\.0\.0\."): + copy_release(release, origin, registry) + assert not registry.root.exists() diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 1190d58ff..59c182c16 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -296,7 +296,7 @@ def test_load_full_round_trips(version_dir, tmp_path, entry_point): directory, manifest = version_dir with registry_server(directory) as server: registry = RemoteRegistry(server.url, cache_dir=tmp_path / "cache") - client = TimeNet(registry=registry, storage_path=tmp_path / "storage") + client = TimeNet(sources=[registry]) if entry_point == "open": with client.open(manifest.metadata.dataset_id) as reader: assert_datasets_equal(make_dataset(), reader.read()) @@ -321,7 +321,7 @@ def single_use_download(request): with registry_server(directory, serve_file=single_use_download if expire_urls else None) as server: registry = RemoteRegistry(server.url, cache_dir=tmp_path / "cache") - client = TimeNet(registry=registry, storage_path=tmp_path / "storage") + client = TimeNet(sources=[registry]) with client.open(manifest.metadata.dataset_id) as reader: # A worker must reopen its own clients, including after the main process has read values. reader.get(Signal, make_dataset().records[0].signals[0].id).to_arrow() @@ -355,7 +355,7 @@ def serve_file(request): with registry_server(directory, serve_file=serve_file) as server: remote = RemoteRegistry(server.url, token="private-token", cache_dir=tmp_path / "cache") - with TimeNet(remote).open(dataset.metadata.dataset_id, "1.0.0") as reader: + with TimeNet(sources=[remote]).open(dataset.metadata.dataset_id, "1.0.0") as reader: assert all(response.request.url.path.startswith("/api/v1/") for response in server.responses) if not support_ranges: # The get reads the catalog, which already needs range requests. @@ -484,12 +484,13 @@ def handler(request): registry.get_manifest("demo/example") -def test_remote_registry_uses_client_storage_path(tmp_path): - # A URL-configured remote registry must cache under the client's storage_path, so download() and - # load() share one cache-first directory. - client = TimeNet(registry="https://registry.example.test", storage_path=tmp_path / "store") - assert isinstance(client._registry, RemoteRegistry) - assert client._registry._cache_dir == client._storage == tmp_path / "store" +def test_a_registry_url_in_the_sources_caches_under_the_storage_directory(tmp_path, monkeypatch): + # A remote registry that the sources name by URL caches under the storage directory, where + # download() copies to, so download() and load() share one cache-first directory. + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "store")) + (registry,) = TimeNet(sources=["https://registry.example.test"])._sources + assert isinstance(registry, RemoteRegistry) + assert registry._cache_dir == tmp_path / "store" def test_put_dataset_without_write_token_is_rejected(tmp_path): @@ -538,14 +539,14 @@ def handler(request): cache = tmp_path / "cache" remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=cache) with range_server(origin) as blob_url: - with TimeNet(remote).open("test/child", "1.0.0") as reader: + with TimeNet(sources=[remote]).open("test/child", "1.0.0") as reader: assert all("/download/" not in path for path in requests) assert reader.get(Signal, signal_id).to_arrow().equals(expected) assert any(parent_id + "/1.0.0/download/" in path for path in requests) assert not any(parent_id + "/1.0.0/download/control.duckdb" in path for path in requests) assert not (cache / parent_id / "1.0.0").exists() remote.download_version("test/child", "1.0.0", cache / "test/child/1.0.0") - with TimeNet(remote).open("test/child", "1.0.0") as reader: + with TimeNet(sources=[remote]).open("test/child", "1.0.0") as reader: assert reader.get(Signal, signal_id).to_arrow().equals(expected) assert not any("test/build-only" in path for path in requests) assert (cache / parent_id / "1.0.0/control.duckdb").exists() @@ -577,17 +578,19 @@ def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: return dataset.metadata.dataset_id, release -@pytest.mark.parametrize("page", ["own", "ancestor"]) -def test_a_local_client_checks_access_before_it_reads_from_a_remote_registry(tmp_path, page): - dataset_id, release = _gated_release(tmp_path / "origin", page) +def test_a_read_checks_access_before_it_reads_from_a_remote_registry(tmp_path): + dataset_id, release = _gated_release(tmp_path / "origin", "ancestor") transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") with pytest.raises(TimeNetAccessError) as caught: - TimeNet(tmp_path / "local").open(dataset_id, auto_build=False, fetch_from=remote) + TimeNet(sources=[LocalRegistry(tmp_path / "local"), remote]).open(dataset_id) assert caught.value.requirements == (AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua"),) - assert caught.value.hint == "Build it from its connector with auto_build=True." + assert caught.value.hint == ( + "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " + f"`timenet-build build {dataset_id}`." + ) assert not any("/download/" in request.url.path or request.url.host == "blob.local" for request in requests) assert not (tmp_path / "local").exists() diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index 96f2f30a2..4797ab5bc 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -51,7 +51,7 @@ def test_put_dataset_and_remote_load_round_trip(s3_root, tmp_path, backend, entr registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") reference = registry.put_dataset(make_dataset(), values_backend=backend) assert reference == DatasetRef.parse(f"{make_dataset().metadata.dataset_id}@1.0.0") - client = TimeNet(registry=registry, storage_path=tmp_path / "cache") + client = TimeNet(sources=[registry]) if entry_point == "open": with client.open(make_dataset().metadata.dataset_id) as reader: # The reader reads the release from S3, not from a downloaded copy. @@ -99,11 +99,12 @@ def test_objects_live_under_a_datasets_prefix(s3_root, tmp_path): assert all(key.startswith("registry/datasets/") for key in keys) -def test_download_then_load_serves_from_cache(s3_root, tmp_path): +def test_download_then_load_serves_from_cache(s3_root, tmp_path, monkeypatch): registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") dataset_id = make_dataset().metadata.dataset_id registry.put_dataset(make_dataset()) - client = TimeNet(registry=registry, storage_path=tmp_path / "cache") + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "cache")) + client = TimeNet(sources=[registry]) path = client.download(dataset_id) # boto3 GetObject per file, into the cache assert (path / "manifest.json").exists() handle = registry.open_release(DatasetRef.parse(f"{dataset_id}@1.0.0")) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index d972b8c3a..8e82e141a 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -2,13 +2,13 @@ import subprocess import sys +import click import pytest import typer from typer.testing import CliRunner from timenet.cache import cached_datasets, raw_cache_size from timenet.cli.app import _enum_list, app -from timenet.client import TimeNet from timenet.dataset import TimeFDataset from timenet.errors import TimeNetDatasetNotFoundError from timenet.reader import TimeFReader @@ -80,24 +80,27 @@ def test_cache_clear_all_removes_everything(home): def test_list(registry_root): - assert runner.invoke(app, ["list", "--registry", str(registry_root)]).exit_code == 0 - assert any(d.dataset_id == "timenet/hello-world" for d in TimeNet(str(registry_root)).list()) + result = runner.invoke(app, ["list", "--registry", str(registry_root)]) + assert result.exit_code == 0, result.output + assert "timenet/hello-world" in click.unstyle(result.output) def test_search_by_domain(registry_root): - assert runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "general"]).exit_code == 0 - results = TimeNet(str(registry_root)).search(domain=[Domain("general")]) - assert any(d.dataset_id == "timenet/hello-world" for d in results) + result = runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "general"]) + assert result.exit_code == 0, result.output + assert "timenet/hello-world" in click.unstyle(result.output) def test_search_no_match(registry_root): - assert runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "cardiology"]).exit_code == 0 - assert TimeNet(str(registry_root)).search(domain=[Domain("cardiology")]) == [] + result = runner.invoke(app, ["search", "--registry", str(registry_root), "--domain", "cardiology"]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.output) == "No datasets match.\n" def test_info(registry_root): - assert runner.invoke(app, ["info", "timenet/hello-world", "--registry", str(registry_root)]).exit_code == 0 - assert TimeNet(str(registry_root)).get("timenet/hello-world").dataset_id == "timenet/hello-world" + result = runner.invoke(app, ["info", "timenet/hello-world", "--registry", str(registry_root)]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.output).startswith("timenet/hello-world") def _write(root, dataset): @@ -148,12 +151,11 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): assert cells[start + 3][0] == "domains" -def test_download(registry_root, tmp_path): - result = runner.invoke( - app, - ["download", "timenet/hello-world", "--registry", str(registry_root), "--storage", str(tmp_path / "store")], - ) - assert result.exit_code == 0 +def test_download(registry_root, tmp_path, monkeypatch): + monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "store")) + result = runner.invoke(app, ["download", "timenet/hello-world", "--registry", str(registry_root)]) + assert result.exit_code == 0, result.output assert (tmp_path / "store" / "timenet/hello-world" / "1.0.0" / "manifest.json").exists() diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 955e02556..36ba34bd7 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -2,13 +2,12 @@ from types import SimpleNamespace from _demo_child import _child_of_the_demo -from _fake_registry import build_service_fake import httpx from pydantic import ValidationError import pytest from timenet import builders as builders_module -from timenet.client import TimeNet +from timenet.client import TimeNet, default_sources from timenet.connectors import BaseConnector from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset from timenet.engine import open_parents, run_pipeline @@ -17,16 +16,16 @@ TimeNetAccessError, TimeNetDatasetNotFoundError, TimeNetMissingParentError, + TimeNetRegistryError, TimeNetVersionUnavailableError, ) from timenet.format.stored_values import StoredText -from timenet.manifest import Manifest from timenet.manifest.files import FilePart from timenet.reader import Loading, LoadPlan, TimeFReader from timenet.registry import TIMENET_REGISTRY_URL, LocalRegistry, RemoteRegistry, open_local_version from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal, install_backends, make_dataset -from timenet.types import Access, DatasetMetadata, Domain, License, Version +from timenet.types import Access, DatasetMetadata, License, Version from timenet.types.access import AccessRequirement from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.reference_keys import ParentAlias @@ -61,48 +60,27 @@ def build(self, release, store, *, force, pinned_by, options): store.put_dataset(dataset, force=force, values_backend=self.BACKEND, pinned_by=pinned_by or None) -def test_a_local_child_reads_its_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): +def test_a_child_reads_its_missing_payload_source_from_a_later_registry(tmp_path, monkeypatch): monkeypatch.setattr(builders_module, "entry_points", lambda group: []) origin, reference = _child_of_the_demo(tmp_path / "origin") cache = tmp_path / "cache" shutil.copytree(origin.root / "test/child/1.0.0", cache / "test/child/1.0.0") - client = TimeNet(cache) + store = LocalRegistry(cache) with pytest.raises(TimeNetMissingParentError) as caught: - client.open("test/child", auto_build=False) + TimeNet(sources=[store]).open("test/child") assert (caught.value.missing, caught.value.needed_by) == (reference, DatasetRef.parse("test/child@1.0.0")) assert str(caught.value).endswith( - ". Read it from a registry that holds it with fetch_from= or build it from its connector with " - "auto_build=True." + ". Add a registry that holds it to the sources, or add InstalledConnectors() to build it." ) - # No installed connector builds the payload source, so with auto_build the hint names only fetch_from. - with pytest.raises( - TimeNetMissingParentError, match=r"\. Read it from a registry that holds it with fetch_from=\.$" - ): - client.open("test/child") - with client.open("test/child", fetch_from=str(origin.root)) as reader: + # No installed connector builds the payload source, so with InstalledConnectors() the hint names only a registry. + with pytest.raises(TimeNetMissingParentError, match=r"\. Add a registry that holds it to the sources\.$"): + TimeNet(sources=[store, InstalledConnectors()]).open("test/child") + with TimeNet(sources=[store, str(origin.root), InstalledConnectors()]).open("test/child") as reader: assert reader.read().records[0].signals[0].to_arrow().equals(make_dataset().records[0].signals[0].to_arrow()) # The read opens the payload source in origin, where it is, and copies nothing. assert not (cache / reference.dataset_id).exists() -@pytest.mark.parametrize("in_fetch_from", [True, False]) -def test_a_missing_version_is_read_from_fetch_from_before_it_is_built(tmp_path, monkeypatch, in_fetch_from): - origin = LocalRegistry(tmp_path / "origin") - if in_fetch_from: - _child_of_the_demo(origin.root) - calls = [] - install_backends(monkeypatch, _Builder(calls)) - - with TimeNet(tmp_path / "local").open("test/child", fetch_from=origin) as reader: - assert reader.read().records - - if in_fetch_from: - assert calls == [] - assert not (tmp_path / "local").exists() - return - assert calls == [(DatasetRef.parse("test/child@1.0.0"), False, {})] - - class _Resampled(BaseConnector[str]): """Writes the first demo Signal at half its rate, from its declared parent, into a Record of its own.""" @@ -127,7 +105,7 @@ def compose(self, raw_refs, context): return child -def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_path): +def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_path, monkeypatch): origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) metadata = _Resampled().metadata() @@ -139,15 +117,16 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p expected = make_dataset().records[0].signals[0].to_arrow()[::2] shutil.rmtree(origin.root / reference.dataset_id) - # Reading the compiled result needs no original samples, even when fetching is enabled. - loaded = TimeNet(origin).load(metadata.dataset_id, fetch_from=str(origin.root)) + # Reading the compiled result needs no original samples. + loaded = TimeNet(sources=[origin]).load(metadata.dataset_id) assert loaded.records[0].signals[0].to_arrow().equals(expected) # Running the connector again still requires its declared parent. with pytest.raises(TimeNetDatasetNotFoundError, match=reference.dataset_id): open_parents(metadata, origin) cache = tmp_path / "download" - directory = TimeNet(origin, storage_path=cache).download(metadata.dataset_id) + monkeypatch.setenv("TIMENET_STORAGE", str(cache)) + directory = TimeNet(sources=[origin]).download(metadata.dataset_id) assert not (cache / reference.dataset_id).exists() shutil.rmtree(origin.root) version = open_local_version(directory) @@ -157,28 +136,6 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p assert reader.read().records[0].signals[0].to_arrow().equals(expected) -def test_a_remote_client_takes_no_fetch_from_and_never_builds(tmp_path, monkeypatch): - calls = [] - install_backends(monkeypatch, _Builder(calls)) - # The remote registry holds the child, but not its payload source, which the installed backend builds. - origin, reference = _child_of_the_demo(tmp_path / "origin") - shutil.rmtree(origin.root / reference.dataset_id) - transport, _ = build_service_fake(origin.root) - remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") - - message = "fetch_from adds a registry to the reads of a local registry; a remote client reads from its own registry" - with pytest.raises(TimeFValidationError, match=message): - TimeNet(remote).open("test/child", fetch_from=str(tmp_path)) - # Neither fetch_from nor auto_build helps a remote client, so the messages name no keyword. - with pytest.raises(TimeNetMissingParentError) as missing: - TimeNet(remote).open("test/child") - assert str(missing.value) == f"payload source {reference} of test/child@1.0.0 is not in registry http://api.local." - with pytest.raises(TimeNetDatasetNotFoundError) as caught: - TimeNet(remote).open(reference.dataset_id) - assert str(caught.value) == f"{reference.dataset_id} is in none of the sources: registry http://api.local." - assert calls == [] - - @pytest.fixture(autouse=True) def _clean_env(monkeypatch): for var in ("TIMENET_HOME", "TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): @@ -194,11 +151,41 @@ def registry_root(tmp_path): return tmp_path / "reg" -def test_no_args_defaults_to_the_hosted_registry(monkeypatch): - monkeypatch.delenv("TIMENET_REGISTRY", raising=False) - client = TimeNet() # no registry, no storage - assert isinstance(client._registry, RemoteRegistry) - assert client._registry._base_url == TIMENET_REGISTRY_URL +@pytest.fixture +def storage(tmp_path, monkeypatch): + """The directory that ``download()`` copies into.""" + monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "storage")) + return tmp_path / "storage" + + +def _entry(source): + """Describe one source of a list: a local registry by its root, another source by its text.""" + if isinstance(source, LocalRegistry): + return ("local", source.root) + if isinstance(source, InstalledConnectors): + return "installed connectors" + return str(source) + + +@pytest.mark.parametrize("registry", [None, "https://registry.example.test", "local"]) +def test_the_default_list_follows_timenet_registry(tmp_path, monkeypatch, registry): + monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) + store = ("local", tmp_path / "home" / "registry") + if registry == "local": + monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "elsewhere")) + expected = [("local", tmp_path / "elsewhere"), "installed connectors"] + elif registry is not None: + monkeypatch.setenv("TIMENET_REGISTRY", registry) + expected = [store, f"registry {registry}", "installed connectors"] + else: + expected = [store, f"registry {TIMENET_REGISTRY_URL}", "installed connectors"] + client = TimeNet() + + assert [_entry(source) for source in client._sources] == expected + assert client._store is client._sources[0] + # The argument comes before $TIMENET_REGISTRY, and adds a remote to a local $TIMENET_REGISTRY. + with_remote = [_entry(source) for source in default_sources(remote_registry="https://other.example.test")] + assert with_remote == [expected[0], "registry https://other.example.test", "installed connectors"] def test_timenet_registry_env_selects_a_local_registry(tmp_path, monkeypatch): @@ -208,38 +195,111 @@ def test_timenet_registry_env_selects_a_local_registry(tmp_path, monkeypatch): dataset.derive_schema() with TimeFWriter(tmp_path / "home" / "registry", dataset) as writer: writer.write() - client = TimeNet() # registry comes from $TIMENET_REGISTRY + client = TimeNet() # the store comes from $TIMENET_REGISTRY + assert [_entry(source) for source in client._sources] == [ + ("local", tmp_path / "home" / "registry"), + "installed connectors", + ] assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} assert client.download("timenet/hello-world") == tmp_path / "home" / "storage" / "timenet/hello-world" / "1.0.0" -def test_list(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") - assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} +def test_a_client_takes_an_ordered_list_of_sources(tmp_path): + store = LocalRegistry(tmp_path / "store") + client = TimeNet(sources=[store, str(tmp_path / "a"), tmp_path / "b", InstalledConnectors()]) + + first, second, third, fourth = client._sources + assert first is store + for source, root in ((second, tmp_path / "a"), (third, tmp_path / "b")): + assert isinstance(source, LocalRegistry) + assert source.root == root + assert fourth == InstalledConnectors() + + +@pytest.mark.parametrize("writable", ["class", "path"]) +def test_builds_land_in_the_first_local_registry_that_is_not_read_only(tmp_path, monkeypatch, writable): + install_backends(monkeypatch, _Builder([])) + roots = [tmp_path / name for name in ("read-only", "first", "later")] + first = LocalRegistry(roots[1]) if writable == "class" else str(roots[1]) + sources = [LocalRegistry(roots[0], read_only=True), first, LocalRegistry(roots[2]), InstalledConnectors()] + + TimeNet(sources=sources).load("timenet/hello-world") + + built = [root.name for root in roots if LocalRegistry(root).exists("timenet/hello-world", "1.0.0")] + assert built == ["first"] + + +def test_a_client_rejects_one_source_given_as_a_string(tmp_path): + for value in (str(tmp_path), tmp_path): + with pytest.raises(TimeFValidationError) as caught: + TimeNet(value) + assert str(caught.value) == ( + f"sources takes a list of sources. Pass sources=[{str(tmp_path)!r}], or " + f"sources=[{str(tmp_path)!r}, InstalledConnectors()] " + "to build what the path lacks" + ) + +def test_a_client_rejects_one_registry_given_without_a_list(tmp_path): + with pytest.raises(TimeFValidationError, match=r"^sources takes a list of sources\. Pass sources=\["): + TimeNet(LocalRegistry(tmp_path)) # ty: ignore[invalid-argument-type] -def test_get_returns_manifest(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") - assert isinstance(client.get("timenet/hello-world"), Manifest) +def test_a_client_rejects_a_string_that_is_not_a_registry(): + with pytest.raises(TimeNetRegistryError) as caught: + TimeNet(sources=["ftp://x"]) + assert str(caught.value) == ( + "sources[0] 'ftp://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + ) + + +def test_a_client_rejects_an_empty_list(): + with pytest.raises(TimeFValidationError, match=r"^sources is empty\."): + TimeNet(sources=[]) + + +def test_a_client_rejects_an_item_that_is_not_a_source(tmp_path): + with pytest.raises(TimeFValidationError, match=r"^sources\[1\] is not a source, a registry URL or a path: 7$"): + TimeNet(sources=[str(tmp_path), 7]) # ty: ignore[invalid-argument-type] + + +def test_installed_connectors_need_a_local_registry_that_is_not_read_only(tmp_path): + with pytest.raises(TimeFValidationError) as caught: + TimeNet(sources=[LocalRegistry(tmp_path, read_only=True), InstalledConnectors()]) + assert str(caught.value) == ( + "InstalledConnectors() needs a writable local registry to build into. " + "Add a LocalRegistry without read_only=True to the sources" + ) + + +def test_get_list_and_search_ask_the_first_registry_that_is_not_the_store(registry_root, tmp_path): + store = LocalRegistry(tmp_path / "store") + other = make_dataset() + other._metadata = other.metadata.model_copy(update={"dataset_id": "test/other"}) + store.put_dataset(other) + client = TimeNet(sources=[store, str(registry_root), InstalledConnectors()]) -def test_search(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") - assert {m.dataset_id for m in client.search(domain=Domain.GENERAL)} == {"timenet/hello-world"} - assert client.search(domain=Domain.CARDIOLOGY) == [] + assert [metadata.dataset_id for metadata in client.list()] == ["timenet/hello-world"] + assert [metadata.dataset_id for metadata in client.search(query="hello")] == ["timenet/hello-world"] + assert client.get("timenet/hello-world").dataset_id == "timenet/hello-world" + with pytest.raises(TimeNetDatasetNotFoundError): + client.get("test/other") + # Without such a registry, they ask the store. + alone = TimeNet(sources=[store, InstalledConnectors()]) + assert [metadata.dataset_id for metadata in alone.list()] == ["test/other"] -def test_download_copies_into_storage(registry_root, tmp_path): - storage = tmp_path / "store" - client = TimeNet(registry_root, storage_path=storage) +def test_download_copies_into_storage(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) version_dir = client.download("timenet/hello-world") assert version_dir == storage / "timenet/hello-world" / "1.0.0" assert (version_dir / "manifest.json").exists() assert list(version_dir.glob("time_series/part-*.parquet")) -def test_download_reports_progress(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_download_reports_progress(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) reported: list[int] = [] client.download("timenet/hello-world", progress_cb=reported.append) expected = sum(part.size for part in client.get("timenet/hello-world").files.all_files()) @@ -252,15 +312,15 @@ def test_manifest_file_rejects_path_traversal(): FilePart(path="../../escape.txt", checksum="sha256:" + "0" * 64, size=0) -def test_download_is_idempotent(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_download_is_idempotent(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) first = client.download("timenet/hello-world") second = client.download("timenet/hello-world") assert first == second -def test_force_redownload_replaces_atomically(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_force_redownload_replaces_atomically(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) first = client.download("timenet/hello-world") (first / "stale.txt").write_text("left over") # a marker that a clean replace should remove @@ -271,9 +331,8 @@ def test_force_redownload_replaces_atomically(registry_root, tmp_path): assert not list(again.parent.glob("*.tmp-*")) # staging dir cleaned up -def test_download_leaves_sibling_staging_dirs_untouched(registry_root, tmp_path): - storage = tmp_path / "store" - client = TimeNet(registry_root, storage_path=storage) +def test_download_leaves_sibling_staging_dirs_untouched(registry_root, storage): + client = TimeNet(sources=[str(registry_root)]) # A concurrent download of the same version has a live .tmp-* dir. download() must not # delete a sibling staging dir: sweeping siblings would corrupt that other download mid-write. sibling = storage / "timenet/hello-world" / "1.0.0.tmp-deadbeef" @@ -285,8 +344,8 @@ def test_download_leaves_sibling_staging_dirs_untouched(registry_root, tmp_path) assert (version_dir / "manifest.json").exists() # the download still completed -def test_open_returns_a_lazy_reader(registry_root, tmp_path): - client = TimeNet(registry_root, storage_path=tmp_path / "store") +def test_open_returns_a_lazy_reader(registry_root): + client = TimeNet(sources=[str(registry_root)]) with client.open("timenet/hello-world") as reader: assert isinstance(reader, TimeFReader) assert reader._control is None # the catalog opens on the first read @@ -315,7 +374,7 @@ def counting_close(reader): close(reader) monkeypatch.setattr(TimeFReader, "close", counting_close) - restored = TimeNet(tmp_path / "reg", storage_path=tmp_path / "store").load("timenet/hello-world") + restored = TimeNet(sources=[str(tmp_path / "reg")]).load("timenet/hello-world") assert isinstance(restored, TimeFDataset) assert len(closed) == 1 signals = [signal for record in restored.records for signal in record.signals] @@ -332,12 +391,16 @@ def counting_close(reader): assert_datasets_equal(expected, restored) -def test_load_reads_in_place_without_downloading(registry_root, tmp_path): - # load() reads the release where it is, not download-then-read, so it must not - # write a copy into local storage the way download() does. - storage = tmp_path / "store" - client = TimeNet(registry_root, storage_path=storage) +def test_load_reads_in_place_without_downloading(registry_root, tmp_path, storage, monkeypatch): + # load() reads the release where it is, not download-then-read, so it must not write a copy into + # the store or into local storage the way download() does. + calls = [] + install_backends(monkeypatch, _Builder(calls)) + store = tmp_path / "store" + client = TimeNet(sources=[LocalRegistry(store), str(registry_root), InstalledConnectors()]) client.load("timenet/hello-world") + assert calls == [] + assert not store.exists() # nothing copied into the store assert not (storage / "timenet/hello-world").exists() # nothing fetched into the download cache @@ -357,7 +420,7 @@ def counting_close(reader): monkeypatch.setattr(TimeFReader, "read", read_everything) monkeypatch.setattr(TimeFReader, "close", counting_close) - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[str(registry_root)]) with client.open_torch("timenet/hello-world") as ds: assert isinstance(ds, Dataset) assert len(ds) == len(make_dataset().records) @@ -368,9 +431,9 @@ def counting_close(reader): @pytest.mark.parametrize("arrays_loaded", [True, False], ids=["default-plan", "text-only-plan"]) -def test_open_torch_passes_its_load_plan_to_the_view(registry_root, tmp_path, arrays_loaded): +def test_open_torch_passes_its_load_plan_to_the_view(registry_root, arrays_loaded): pytest.importorskip("torch") - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[str(registry_root)]) view = ( client.open_torch("timenet/hello-world") if arrays_loaded @@ -383,12 +446,6 @@ def test_open_torch_passes_its_load_plan_to_the_view(registry_root, tmp_path, ar assert {signal._values.is_loaded for signal in record.signals} == {arrays_loaded} -def test_registry_env_var(registry_root, tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_REGISTRY", str(registry_root)) - client = TimeNet(storage_path=tmp_path / "store") - assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} - - @pytest.fixture def versioned_registry(tmp_path): """A registry holding hello_world at both 1.0.0 and 1.1.0.""" @@ -402,22 +459,22 @@ def versioned_registry(tmp_path): return root -def test_version_ref_pins_and_defaults_to_latest(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_version_ref_pins_and_defaults_to_latest(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) assert str(client.get("timenet/hello-world").metadata.dataset_version) == "1.1.0" # no ref -> latest assert str(client.get("timenet/hello-world@latest").metadata.dataset_version) == "1.1.0" assert str(client.get("timenet/hello-world@1.0.0").metadata.dataset_version) == "1.0.0" # pinned assert str(client.get("timenet/hello-world", "1.0.0").metadata.dataset_version) == "1.0.0" # explicit arg -def test_open_and_load_read_the_version_asked_for(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_open_and_load_read_the_version_asked_for(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) with client.open("timenet/hello-world", version="1.0.0") as reader: assert str(reader.metadata.dataset_version) == "1.0.0" assert str(client.load("timenet/hello-world", version="1.0.0").metadata.dataset_version) == "1.0.0" -def test_open_torch_reads_the_version_asked_for(versioned_registry, tmp_path, monkeypatch): +def test_open_torch_reads_the_version_asked_for(versioned_registry, monkeypatch): pytest.importorskip("torch") opened: list[TimeFReader] = [] original_open = TimeNet.open @@ -427,55 +484,56 @@ def recording_open(client, *args, **kwargs): return opened[-1] monkeypatch.setattr(TimeNet, "open", recording_open) - TimeNet(versioned_registry, storage_path=tmp_path / "store").open_torch("timenet/hello-world", version="1.0.0") + TimeNet(sources=[str(versioned_registry)]).open_torch("timenet/hello-world", version="1.0.0") (reader,) = opened assert str(reader.metadata.dataset_version) == "1.0.0" -def test_version_ref_download_pins(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") - assert client.download("timenet/hello-world@1.0.0") == tmp_path / "store" / "timenet/hello-world" / "1.0.0" +def test_version_ref_download_pins(versioned_registry, storage): + client = TimeNet(sources=[str(versioned_registry)]) + assert client.download("timenet/hello-world@1.0.0") == storage / "timenet/hello-world" / "1.0.0" -def test_version_ref_missing_pin_raises(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_version_ref_missing_pin_raises(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) with pytest.raises(TimeNetDatasetNotFoundError): client.get("timenet/hello-world@9.9.9") -def test_version_given_twice_raises(versioned_registry, tmp_path): - client = TimeNet(versioned_registry, storage_path=tmp_path / "store") +def test_version_given_twice_raises(versioned_registry): + client = TimeNet(sources=[str(versioned_registry)]) with pytest.raises(ValueError, match="twice"): client.get("timenet/hello-world@1.0.0", "1.1.0") -def test_load_names_the_buildable_version_when_a_pin_cannot_be_built(registry_root, tmp_path, monkeypatch): +def test_load_names_the_buildable_version_when_a_pin_cannot_be_built(registry_root, monkeypatch): # The builder declares only one version, so a pin it cannot satisfy must say which version it # builds, and must fail before the expensive build rather than after it. calls = [] install_backends(monkeypatch, _Builder(calls)) - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[LocalRegistry(registry_root), InstalledConnectors()]) with pytest.raises(TimeNetVersionUnavailableError, match=r"builds version 1\.0\.0, not the requested 9\.9\.9"): client.load("timenet/hello-world@9.9.9") assert calls == [] # the pin was rejected before the build ran -@pytest.mark.parametrize("fetch_from", [None, "empty"]) -def test_a_missing_version_that_no_connector_builds_names_the_keyword_that_can_get_it( - tmp_path, monkeypatch, fetch_from -): +@pytest.mark.parametrize("connectors", [True, False]) +def test_a_release_that_no_source_provides_names_how_to_get_it(tmp_path, monkeypatch, connectors): monkeypatch.setattr(builders_module, "entry_points", lambda group: []) - client = TimeNet(tmp_path / "reg", storage_path=tmp_path / "store") - source = None if fetch_from is None else LocalRegistry(tmp_path / fetch_from) + sources = [ + LocalRegistry(tmp_path / "reg"), + LocalRegistry(tmp_path / "empty"), + *([InstalledConnectors()] if connectors else []), + ] with pytest.raises(TimeNetDatasetNotFoundError) as caught: - client.open("x/y", fetch_from=source) + TimeNet(sources=sources).open("x/y") - sources = [LocalRegistry(tmp_path / "reg"), *([] if source is None else [source]), InstalledConnectors()] message = f"x/y is in none of the sources: {', '.join(str(entry) for entry in sources)}." - # The hint names only the keyword that the call did not use. - hint = " Read it from a registry that holds it with fetch_from=." if source is None else "" + # The hint names InstalledConnectors() only if the list does not have it. + hint = " Add a registry that holds it to the sources" + hint += "." if connectors else ", or add InstalledConnectors() to build it." assert str(caught.value) == message + hint @@ -484,31 +542,25 @@ def test_load_accepts_the_latest_sentinel_after_a_build(tmp_path, monkeypatch): empty_root = tmp_path / "reg" empty_root.mkdir() install_backends(monkeypatch, _Builder([])) - client = TimeNet(empty_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[LocalRegistry(empty_root), InstalledConnectors()]) assert str(client.load("timenet/hello-world", "latest").metadata.dataset_version) == "1.0.0" -@pytest.mark.parametrize("entry_point", ["open", "load", "open_torch"]) -def test_load_does_not_build_when_auto_build_is_false(registry_root, tmp_path, monkeypatch, entry_point): - # A caller can opt out of the build-on-miss, so a notebook load fails fast instead of starting a - # multi-GB download and a build. The builder declares the requested version, so only - # auto_build=False keeps it from running. - if entry_point == "open_torch": - pytest.importorskip("torch") +def test_a_list_without_installed_connectors_does_not_build_a_missing_version(registry_root, monkeypatch): + # A list without InstalledConnectors() does not build on a miss, so a notebook load fails fast + # instead of starting a multi-GB download and a build. The builder declares the requested + # version, so only the list keeps it from running. calls = [] monkeypatch.setattr(_Builder, "VERSION", "9.9.9") install_backends(monkeypatch, _Builder(calls)) - client = TimeNet(registry_root, storage_path=tmp_path / "store") + client = TimeNet(sources=[LocalRegistry(registry_root)]) - # The hint names the keywords that could obtain the release, in a sentence of its own. - hint = ( - r"\. Read it from a registry that holds it with fetch_from= or build it from its connector with " - r"auto_build=True\.$" - ) + # The hint names the sources that could provide the release, in a sentence of its own. + hint = r"\. Add a registry that holds it to the sources, or add InstalledConnectors\(\) to build it\.$" with pytest.raises(TimeNetDatasetNotFoundError, match=hint): - getattr(client, entry_point)("timenet/hello-world@9.9.9", auto_build=False) - assert calls == [] # auto_build=False skips the connector entirely + client.load("timenet/hello-world@9.9.9") + assert calls == [] # the list skips the connector entirely def _card(dataset_id, access, access_url=None): @@ -523,41 +575,57 @@ def _card(dataset_id, access, access_url=None): ) -def _hosted(client, metadata, lineage=()): - """Serve one manifest from a registry that is not local, so the build-your-own gate applies.""" +def _hosted(tmp_path, metadata, lineage=()): + """Return a registry that is not local and serves one manifest, so the build-your-own gate applies.""" class _Hosted(RemoteRegistry): def get_manifest(self, dataset_id, version=None): return SimpleNamespace(metadata=metadata, lineage=lineage) - client._registry = _Hosted( + return _Hosted( "http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404)), - cache_dir=client._storage, + cache_dir=tmp_path / "cache", ) def test_load_rejects_a_credentialed_dataset_from_a_hosted_registry(tmp_path): - client = TimeNet(tmp_path / "registry", storage_path=tmp_path / "store") - _hosted(client, _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua")) + client = TimeNet( + sources=[_hosted(tmp_path, _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua"))] + ) message = ( "org/gated@1.0.0 is credentialed. Get access at https://physionet.example/dua for org/gated@1.0.0. " - "TimeNet does not host its data, so build it locally with `timenet-build build org/gated`." + "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " + "`timenet-build build org/gated`." + ) + with pytest.raises(TimeNetAccessError) as caught: + client.load("org/gated") + assert str(caught.value) == message + + +def test_the_access_hint_says_that_no_installed_connector_builds_the_release(tmp_path, monkeypatch): + monkeypatch.setattr(builders_module, "entry_points", lambda group: []) + card = _card("org/gated", Access.CREDENTIALED, "https://physionet.example/dua") + client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), _hosted(tmp_path, card), InstalledConnectors()]) + + with pytest.raises(TimeNetAccessError) as caught: + client.open("org/gated") + + assert str(caught.value) == ( + "org/gated@1.0.0 is credentialed. Get access at https://physionet.example/dua for org/gated@1.0.0. " + "No source serves its files, and no installed connector builds this release. " + "Install its connector package and build it locally with `timenet-build build org/gated`." ) - for entry_point in (client.open, client.load, client.download): - with pytest.raises(TimeNetAccessError) as caught: - entry_point("org/gated") - assert str(caught.value) == message def test_the_hint_for_a_hosted_child_names_the_access_page_of_each_ancestor(tmp_path): - client = TimeNet(tmp_path / "registry", storage_path=tmp_path / "store") # The card of the child stays open and has no access page. Its parents set the access. parents = ( _card("org/a", Access.CREDENTIALED, "https://a.example"), _card("org/b", Access.RESTRICTED, "https://b.example"), ) - _hosted(client, _card("org/child", Access.OPEN), tuple(ParentDatasetCard(metadata=card) for card in parents)) + lineage = tuple(ParentDatasetCard(metadata=card) for card in parents) + client = TimeNet(sources=[_hosted(tmp_path, _card("org/child", Access.OPEN), lineage)]) with pytest.raises(TimeNetAccessError) as caught: client.open("org/child") @@ -569,5 +637,6 @@ def test_the_hint_for_a_hosted_child_names_the_access_page_of_each_ancestor(tmp_ assert str(caught.value) == ( "org/child@1.0.0 is restricted. " "Get access at https://a.example for org/a@1.0.0 and at https://b.example for org/b@1.0.0. " - "TimeNet does not host its data, so build it locally with `timenet-build build org/child`." + "No source serves its files. Add InstalledConnectors() to the sources, or build it locally with " + "`timenet-build build org/child`." )