diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 9e34e972b..3dbdd1a3d 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -1,6 +1,6 @@ """The consumer command-line interface, the command-line mirror of the :class:`~timenet.client.TimeNet` SDK.""" -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Sequence from functools import partial from pathlib import Path import shlex @@ -16,12 +16,13 @@ from timenet.cli.ui import console as ui, progress_renderer from timenet.client import TimeNet, default_local_registry, default_sources from timenet.config import settings -from timenet.errors import TimeNetDatasetNotFoundError +from timenet.errors import TimeNetDatasetNotFoundError, TimeNetNotBuiltError from timenet.producer import publish as publish_release from timenet.progress import ProgressCallback, ProgressEvent, StepStarted from timenet.registry import open_registry from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access +from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.references import split_ref from timenet.types.tasks import TASKS @@ -157,27 +158,25 @@ def info( remote_registry: str | None = REMOTE_REGISTRY, auto_build: bool | None = AUTO_BUILD, ) -> None: - """Show a dataset's metadata, schema, and counts.""" - manifest = _client(local_registry, remote_registry, auto_build).get(dataset_id) + """Print a dataset's metadata, schema, and counts. + + For an unbuilt release, show its metadata card and build command instead. + """ + client = _client(local_registry, remote_registry, auto_build) + try: + manifest = client.get_manifest(dataset_id) + except TimeNetNotBuiltError as error: + console.print(_card_table(error.card, lineage=None)) + build = f"timenet-build build {split_ref(dataset_id)[0]}" + if local_registry is not None: + build += f" --local-registry {shlex.quote(local_registry)}" + if remote_registry is not None: + build += f" --remote-registry {shlex.quote(remote_registry)}" + # A plain line, so that a long path does not wrap. + typer.echo(f"The schema and the counts appear after a build: `{build}`.") + return counts = manifest.counts - table = Table(title=manifest.dataset_id, title_justify="left", title_style="bold", show_header=False) - table.add_column("Field", style="cyan") - table.add_column("Value") - table.add_row("version", str(manifest.metadata.dataset_version)) - table.add_row("name", manifest.metadata.name) - table.add_row("license", str(manifest.metadata.license)) - if manifest.metadata.license_url: - table.add_row("license_url", manifest.metadata.license_url) - # The strictest level of the card and its lineage applies, also when the card itself says open. - table.add_row("access", str(effective_access(manifest.metadata, manifest.lineage))) - # Each card whose access is not open, the dataset or an ancestor, names its own page. - requirements = access_requirements(manifest.metadata, manifest.lineage) - if requirements: - pages = (f"{item.release} ({item.access}): {item.access_url}" for item in requirements) - table.add_row("access_urls", "\n".join(pages)) - if manifest.metadata.citation: - table.add_row("citation", manifest.metadata.citation) - table.add_row("domains", ", ".join(str(domain) for domain in manifest.metadata.domains) or "-") + table = _card_table(manifest.metadata, manifest.lineage) table.add_row("specs", ", ".join(spec.spec_type for spec in manifest.dataset_schema.time_series_specs) or "-") table.add_row("tasks", ", ".join(str(task.task_type) for task in manifest.dataset_schema.tasks) or "-") table.add_section() @@ -197,6 +196,42 @@ def info( console.print(tasks) +def _card_table(card: DatasetMetadata, lineage: Sequence[ParentDatasetCard] | None) -> Table: + """Create a table of dataset metadata and access requirements. + + Args: + card: The dataset card or manifest metadata. + lineage: Ancestor cards, or None for an unbuilt release. + + Returns: + A table without a header. + """ + table = Table(title=card.dataset_id, title_justify="left", title_style="bold", show_header=False) + table.add_column("Field", style="cyan") + table.add_column("Value") + table.add_row("version", str(card.dataset_version)) + table.add_row("name", card.name) + table.add_row("license", str(card.license)) + if card.license_url: + table.add_row("license_url", card.license_url) + # The strictest level of the card and its lineage applies, also when the card itself says open. + access = str(effective_access(card, lineage or ())) + if lineage is None and card.parents: + # Only a build reads the parent cards, so the level above is the level of this card alone. + parents = "its parent" if len(card.parents) == 1 else f"its {len(card.parents)} parents" + access += f" (this card only: the access of {parents} is known after a build)" + table.add_row("access", access) + # Each card whose access is not open, the dataset or an ancestor, names its own page. + requirements = access_requirements(card, lineage or ()) + if requirements: + pages = (f"{item.release} ({item.access}): {item.access_url}" for item in requirements) + table.add_row("access_urls", "\n".join(pages)) + if card.citation: + table.add_row("citation", card.citation) + table.add_row("domains", ", ".join(str(domain) for domain in card.domains) or "-") + return table + + @app.command() def download( dataset_id: str, diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 751399033..ad257451e 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -1,14 +1,15 @@ -"""Find, read, and download datasets with :class:`TimeNet`. +"""Find, inspect, download, and read datasets with :class:`TimeNet`. -Clients search their sources in order. Builds and downloads go into the first local registry that is -not read-only. +A client checks sources in order and builds releases from installed connectors into its first local +registry that is not read-only. """ # The public API has a method named ``list``. Deferred annotations keep the type hint # ``list[str]`` resolved to the builtin type, not to the method. from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Generator, Sequence +from contextlib import contextmanager import os from pathlib import Path import shlex @@ -27,6 +28,7 @@ TimeNetError, TimeNetFileError, TimeNetMissingParentError, + TimeNetNotBuiltError, TimeNetVersionUnavailableError, ) from timenet.manifest import Manifest @@ -204,20 +206,56 @@ def list(self) -> _Metadatas: """ return self._catalog().list_datasets() - def get(self, dataset_id: str, version: str | None = None) -> Manifest: - """Return a dataset version's manifest from the selected registry. + def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest: + """Return the manifest of the release that :meth:`open` reads, without a copy or a build. - The client uses the first registry other than the local registry, or the local registry if it is the only one. + For a release that a registry lists but does not serve, return its listed manifest with access links. Args: - dataset_id: The dataset ID. - version: The version string, or ``None`` for the latest. + dataset_id: The dataset ID, optionally ending with ``@`` or ``@latest``. + version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. Returns: - The dataset's manifest. - """ + The release's manifest. + + Raises: + TimeNetNotBuiltError: If the release needs a build to create its manifest. + TimeNetDatasetNotFoundError: If no source provides the release. + TimeNetVersionUnavailableError: If the version is unavailable and a connector declares another version. + TimeNetAccessError: If a registry only lists the release and provides no manifest. + TimeNetInvalidCardError: If the connector's card is unreadable or invalid. + TimeFValidationError: If ``dataset_id`` and ``version`` both give a version. + """ # ruff: ignore[docstring-extraneous-exception] (raised by the resolution of the release) dataset_id, version = _resolve_ref(dataset_id, version) - return self._catalog().get_manifest(dataset_id, version) + with self._hints(dataset_id): + found = self._manifest_or_connector(dataset_id, version) + if isinstance(found, FoundInConnector): + raise TimeNetNotBuiltError(found.builder.release, found.builder.card) + return found + + def get_card(self, dataset_id: str, version: str | None = None) -> DatasetMetadata: + """Return the card of the release that :meth:`open` reads, without a copy or a build. + + The card comes from the manifest or, for an unbuilt release, from the installed connector. + + Args: + dataset_id: The dataset ID, optionally ending with ``@`` or ``@latest``. + version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. + + Returns: + The release's metadata card. + + Raises: + TimeNetDatasetNotFoundError: If no source provides the release. + TimeNetVersionUnavailableError: If the version is unavailable and a connector declares another version. + TimeNetAccessError: If a registry only lists the release and provides no manifest. + TimeNetInvalidCardError: If the connector's card is unreadable or invalid. + TimeFValidationError: If ``dataset_id`` and ``version`` both give a version. + """ # ruff: ignore[docstring-extraneous-exception] (raised by the resolution of the release) + dataset_id, version = _resolve_ref(dataset_id, version) + with self._hints(dataset_id): + found = self._manifest_or_connector(dataset_id, version) + return found.builder.card if isinstance(found, FoundInConnector) else found.metadata def search( # ruff: ignore[too-many-arguments] self, @@ -277,20 +315,21 @@ def download( dataset_id: The dataset ID, optionally followed by ``@version``. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. force: Copy the release again, even if the local registry holds a complete copy. - progress: Receives step events and byte counts for each copied release. + progress: A callback for copy events and byte counts. Returns: The release directory in the local registry. Raises: - TimeFValidationError: If the sources lack a writable local registry, or both arguments specify a version. - TimeNetAccessError: If a required release is only listed or a remote release is not open. + TimeFValidationError: If no source is a writable local registry. + TimeNetAccessError: If a required release is only listed or requires access in a remote registry. + TimeNetNotBuiltError: If only an installed connector provides the release. TimeNetDatasetNotFoundError: If no registry other than the local registry provides the release. TimeNetVersionUnavailableError: If the version is missing and its connector builds another version. TimeNetMissingParentError: If no registry provides a missing payload source. - TimeNetVersionConflictError: If the local registry holds a release of the copy with different files. - TimeNetFileError: If a copied file is damaged or no registry can replace a damaged local copy. - TimeFFormatError: If payload sources have conflicting file records or form a dependency cycle. + TimeNetVersionConflictError: If the local registry holds the same release with different files. + TimeNetFileError: If no registry can replace a damaged copy, or a copied file is damaged. + TimeFFormatError: If a payload source differs from its recorded files or the source links form a cycle. """ # ruff: ignore[docstring-extraneous-exception] (raised by the plan and the copies) dataset_id, version = _resolve_ref(dataset_id, version) store = self._store @@ -378,24 +417,8 @@ def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: TimeNetBuildError: If a build runs but fails. """ # ruff: ignore[docstring-extraneous-exception] (raised by read_release and the builder) dataset_id, version = _resolve_ref(dataset_id, version) - try: + with self._hints(dataset_id): return TimeFReader(read_release(dataset_id, version, sources=self._sources, store=self._store)) - except TimeNetAccessError as error: - error.hint = _access_hint(dataset_id, self._sources) - raise - except TimeNetFileError as error: - error.hint = self._file_hint(error) - raise - except TimeNetMissingParentError as error: - error.hint = _not_provided_hint(self._sources) - raise - except TimeNetDatasetNotFoundError as error: - # A subclass, such as TimeNetVersionUnavailableError, names its own fix. - if type(error) is not TimeNetDatasetNotFoundError: - raise - raise TimeNetDatasetNotFoundError( - f"{str(error).rstrip('.')}. {_not_provided_hint(self._sources)}" - ) from error def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: """Read a whole dataset version into memory, with every stored value loaded. @@ -428,13 +451,13 @@ def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: return reader.read(load=LoadPlan.ALL_EAGER) def _catalog(self) -> BaseRegistry: - """Select a registry from the sources. + """Choose the registry to browse. Returns: - The first registry other than the local registry, or the local registry if no other registry exists. + The first registry other than the local registry, or the local registry if it is the only one. Raises: - TimeFValidationError: If no source is a registry. + TimeFValidationError: If the sources contain no registry. """ # TODO(adrian): Remove temporary glue which maps these methods onto one registry until # list() and search() ask every source. This registry gives the results of the client of @@ -442,9 +465,73 @@ def _catalog(self) -> BaseRegistry: registries = [source for source in self._sources if isinstance(source, BaseRegistry)] registry = next((registry for registry in registries if registry is not self._store), self._store) if registry is None: - raise TimeFValidationError("list(), get() and search() need a registry in the sources") + raise TimeFValidationError("list() and search() need a registry in the sources") return registry + @contextmanager + def _hints(self, dataset_id: str) -> Generator[None]: + """Add recovery hints to errors raised inside the block. + + Args: + dataset_id: The dataset ID without a version suffix. + + Yields: + Nothing. + + Raises: + TimeNetAccessError: If access fails. + TimeNetFileError: If a file is missing or damaged. + TimeNetMissingParentError: If a required release is missing. + TimeNetDatasetNotFoundError: If the dataset is missing. + """ + try: + yield + except TimeNetAccessError as error: + error.hint = _access_hint(dataset_id, self._sources) + raise + except TimeNetFileError as error: + error.hint = self._file_hint(error) + raise + except TimeNetMissingParentError as error: + error.hint = _not_provided_hint(self._sources) + raise + except TimeNetDatasetNotFoundError as error: + if type(error) is not TimeNetDatasetNotFoundError: + raise + raise TimeNetDatasetNotFoundError( + f"{str(error).rstrip('.')}. {_not_provided_hint(self._sources)}" + ) from error + + def _manifest_or_connector(self, dataset_id: str, version: str | None) -> Manifest | FoundInConnector: + """Resolve a release to its manifest, or to the answer of the connector that provides it. + + Args: + dataset_id: The dataset ID without a version suffix. + version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. + + Returns: + The manifest, or the connector answer if only an installed connector provides the release. + + Raises: + TimeNetDatasetNotFoundError: If no source provides the release. + TimeNetAccessError: If a registry only lists the release and provides no manifest. + """ # ruff: ignore[docstring-extraneous-exception] (raised by raise_not_found) + lookups = Lookups() + answer = lookups.resolve(self._sources, dataset_id, version) + if isinstance(answer, NotFound): + raise_not_found(dataset_id, version, self._sources) + if isinstance(answer, NotServed): + if answer.manifest is None: + raise answer.access_error(dataset_id) + return answer.manifest + if isinstance(answer, FoundInConnector): + return answer + manifest = lookups.manifest(answer.registry, DatasetRef(dataset_id=dataset_id, version=answer.version)) + if manifest is None: + # The registry no longer holds the release that it listed. + raise_not_found(dataset_id, version, self._sources) + return manifest + def _file_hint(self, error: TimeNetFileError) -> str | None: """Suggest how to replace a damaged release in the local registry. diff --git a/packages/timenet/src/timenet/coordinator/plan.py b/packages/timenet/src/timenet/coordinator/plan.py index 4140ec3cc..69ef4bd56 100644 --- a/packages/timenet/src/timenet/coordinator/plan.py +++ b/packages/timenet/src/timenet/coordinator/plan.py @@ -13,6 +13,7 @@ TimeNetDatasetNotFoundError, TimeNetFileError, TimeNetMissingParentError, + TimeNetNotBuiltError, ) from timenet.manifest import FilePart, Manifest, PayloadSource from timenet.registry.base import BaseRegistry @@ -233,7 +234,7 @@ def visit( # ruff: ignore[too-many-arguments] answer: ReleaseLookup | None = None, force: bool = False, ) -> None: - """Add a release and its dependencies to the graph. + """Add a required release and its dependencies to the graph. A node is planned again when a later visit needs more of it: a dataset instead of a payload source, or a copy in the output instead of a read. @@ -241,7 +242,7 @@ def visit( # ruff: ignore[too-many-arguments] Args: reference: The release to add. needed_by: The release that requires ``reference``, or ``None`` for the requested release. - pinned: The file records that ``needed_by`` requires, or ``None`` for a parent dataset. + pinned: Required file records, or ``None`` for a build parent. needed_in_local_registry: True if the plan needs a complete copy in the output. answer: The answer of the sources for ``reference``, or ``None`` to ask them. force: Copy the release again, even if the output holds a complete copy. @@ -319,8 +320,9 @@ def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, Raises: TimeFValidationError: If the plan has no output. - TimeNetDatasetNotFoundError: If the plan builds nothing and only installed connectors provide the release, - no source provides the requested release, or ``force`` is set and only the output holds it. + TimeNetNotBuiltError: If the plan builds nothing and only installed connectors provide the release. + TimeNetDatasetNotFoundError: If no source provides the requested release, or ``force`` is set and only + the output holds it. TimeNetFileError: If the output holds a damaged copy that no other registry provides. TimeNetAccessError: If a source lists the release but no source serves it. TimeNetMissingParentError: If no source provides the release. @@ -352,7 +354,7 @@ def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, elif isinstance(answer, FoundInConnector): if not self.builds: # A download builds nothing. - raise TimeNetDatasetNotFoundError(_not_built(reference)) + raise TimeNetNotBuiltError(reference, answer.builder.card) self.build(reference, role, answer.builder) elif isinstance(answer, NotServed): raise answer.access_error(reference.dataset_id) @@ -653,13 +655,14 @@ def plan_download( store: The local registry to copy into. sources: The sources to search in order, excluding registries at the directory of ``store``. force: Copy the release again, even if the local registry holds a complete copy. - lookups: The answers and manifests that resolved ``release``, or ``None`` to ask the sources. + lookups: Cached source answers and manifests, or ``None`` to query sources. Returns: The copy steps, in the order in which they run. Raises: - TimeNetDatasetNotFoundError: If a needed release requires a build or no registry can supply a forced copy. + TimeNetNotBuiltError: If only installed connectors provide a release that the local registry lacks. + TimeNetDatasetNotFoundError: If ``force`` is set and only the local registry holds the release. TimeNetFileError: If the local registry holds a damaged copy that no other registry provides. TimeNetAccessError: If a source lists a release that the local registry lacks but no source serves it. TimeNetMissingParentError: If no registry provides a payload source that the local registry lacks. @@ -672,21 +675,6 @@ def plan_download( return Plan(tuple(graph.create_ordered_steps())) -def _not_built(release: DatasetRef) -> str: - """Describe a release that only installed connectors provide, in a plan that builds nothing. - - Args: - release: The release. - - Returns: - The error message. - """ - return ( - f"{release} is not built yet: only an installed connector provides it. open() or load() builds it into " - "the local registry." - ) - - def describe_searched_sources(sources: Iterable[ReleaseSource], reference: DatasetRef) -> str: """Describe the sources searched for a missing release. diff --git a/packages/timenet/src/timenet/coordinator/read.py b/packages/timenet/src/timenet/coordinator/read.py index 560dcd0fd..f5dc30a11 100644 --- a/packages/timenet/src/timenet/coordinator/read.py +++ b/packages/timenet/src/timenet/coordinator/read.py @@ -87,9 +87,9 @@ def read_release( def raise_not_found(dataset_id: str, version: str | None, sources: Sequence[ReleaseSource]) -> NoReturn: - """Raise the error for a release that no source provides. + """Raise an error for an unavailable release. - For an exact version, an installed connector that builds another version gives a more exact error. + For an exact version, check the installed connector if the sources include it. Args: dataset_id: The dataset ID. @@ -97,9 +97,9 @@ def raise_not_found(dataset_id: str, version: str | None, sources: Sequence[Rele sources: The sources searched for the release. Raises: - TimeNetVersionUnavailableError: If the installed connector declares a different version. - TimeNetInvalidCardError: If the card of the connector cannot be read. - TimeNetDatasetNotFoundError: Otherwise, with a message that names the searched sources. + TimeNetVersionUnavailableError: If the installed connector declares a different version from the exact request. + TimeNetInvalidCardError: If the connector's card is unreadable or invalid. + TimeNetDatasetNotFoundError: Otherwise, with a message listing the sources searched. """ # ruff: ignore[docstring-extraneous-exception] - raised by declared requested = None if is_latest(version) else DatasetRef.parse(f"{dataset_id}@{version}") connectors = any(isinstance(source, InstalledConnectors) for source in sources) diff --git a/packages/timenet/src/timenet/errors.py b/packages/timenet/src/timenet/errors.py index 970d9d509..59677c185 100644 --- a/packages/timenet/src/timenet/errors.py +++ b/packages/timenet/src/timenet/errors.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from timenet.types.access import Access, AccessRequirement + from timenet.types.metadata import DatasetMetadata from timenet.types.references import DatasetRef from timenet.types.version import Version @@ -34,6 +35,25 @@ class TimeNetDatasetNotFoundError(TimeNetError): """A requested dataset id or version is not present in a registry or on disk.""" +class TimeNetNotBuiltError(TimeNetDatasetNotFoundError): + """Only an installed connector provides a release, so it has no manifest yet. + + Build the release to create its manifest, or read the connector's card for metadata. + + Attributes: + release: The unbuilt release. + card: The connector's card of the release. + """ + + def __init__(self, release: "DatasetRef", card: "DatasetMetadata") -> None: + super().__init__(release, card) + self.release = release + self.card = card + + def __str__(self) -> str: + return f"{self.release} is not built yet: only an installed connector provides it." + + class TimeNetMissingParentError(TimeNetDatasetNotFoundError): """A required parent release or payload source is missing. diff --git a/packages/timenet/src/timenet/testing/builders.py b/packages/timenet/src/timenet/testing/builders.py index 6fa9ae830..2a9c81cdb 100644 --- a/packages/timenet/src/timenet/testing/builders.py +++ b/packages/timenet/src/timenet/testing/builders.py @@ -100,7 +100,7 @@ def _series(spec, signal, n, phase=0.0): ) -def make_dataset() -> TimeFDataset: +def make_dataset(**card: object) -> TimeFDataset: """Build a deterministic dataset that exercises every TimeF feature. The dataset covers two modalities over a shared data source. Every signal has one owning source. @@ -111,19 +111,21 @@ def make_dataset() -> TimeFDataset: and an interval. The dataset numbers every id, so two calls build equal datasets. This is the canonical writer and reader round-trip fixture. + Args: + **card: Card fields that replace those of the demo card, for example ``dataset_id="org/name"``. + Returns: The populated :class:`TimeFDataset`. """ - dataset = TimeFDataset( - metadata=DatasetMetadata( - dataset_id="timenet/hello-world", - dataset_version=Version(1, 0, 0), - name="Hello World", - description="A synthetic demo dataset.", - license=License.CC_BY_4_0, - domains=(Domain.GENERAL,), - ) - ) + fields: dict[str, object] = { + "dataset_id": "timenet/hello-world", + "dataset_version": Version(1, 0, 0), + "name": "Hello World", + "description": "A synthetic demo dataset.", + "license": License.CC_BY_4_0, + "domains": (Domain.GENERAL,), + } + dataset = TimeFDataset(metadata=DatasetMetadata.model_validate({**fields, **card})) shared = _series(_SINE, "a", 16) record0 = dataset.add_record( diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index c63974227..4eafe0488 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -22,7 +22,7 @@ ) from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version -from timenet.testing import make_dataset +from timenet.testing import install_backends, make_dataset from timenet.types import Access, Domain from timenet.types.reference_keys import ParentAlias from timenet.types.references import DatasetRef @@ -50,6 +50,11 @@ def _downloads_size(cache): return sum(path.stat().st_size for path in (cache / "downloads").rglob("*") if path.is_file()) +def _table_cells(output: str) -> list[list[str]]: + """Return the cells of each table row in ``output``, without the borders.""" + return [[cell.strip() for cell in line.split("│")[1:-1]] for line in output.splitlines() if "│" in line] + + @pytest.fixture def state(tmp_path, monkeypatch): """Put a release in the store and 10 bytes of raw downloads in the cache. Returns both paths.""" @@ -158,6 +163,89 @@ def test_search_no_match(registry_root): assert click.unstyle(result.output) == "No datasets match.\n" +class _Backend: + """A builder backend with one connector, whose card declares ``test/gated@1.0.0``.""" + + def dataset_ids(self): + return ["test/gated"] + + def card(self, dataset_id): + return make_dataset().metadata.model_copy( + update={ + "dataset_id": dataset_id, + "access": Access.CREDENTIALED, + "access_url": "https://gated.example", + "citation": "A citation.", + } + ) + + +def test_info_shows_the_card_of_a_dataset_that_only_a_connector_has(tmp_path, monkeypatch): + install_backends(monkeypatch, _Backend()) + store = tmp_path / "store" + flags = ["--local-registry", str(store), "--remote-registry", str(tmp_path / "empty"), "--auto-build"] + + result = runner.invoke(app, ["info", "test/gated", *flags]) + + assert result.exit_code == 0, result.output + cells = _table_cells(result.stdout) + card = _Backend().card("test/gated") + assert cells == [ + ["version", "1.0.0"], + ["name", card.name], + ["license", str(card.license)], + ["access", "credentialed"], + ["access_urls", "test/gated@1.0.0 (credentialed): https://gated.example"], + ["citation", "A citation."], + ["domains", "general"], + ] + assert result.stdout.splitlines()[-1] == ( + "The schema and the counts appear after a build: " + f"`timenet-build build test/gated --local-registry {store} --remote-registry {tmp_path / 'empty'}`." + ) + # The command reads the card and builds nothing. + assert not store.exists() + + +def test_info_of_an_unbuilt_child_says_that_its_access_leaves_out_its_parents(tmp_path, monkeypatch): + class _Child(_Backend): + def dataset_ids(self): + return ["test/child"] + + def card(self, dataset_id): + parents = {"a": "test/a@1.0.0", "b": "test/b@1.0.0"} + return make_dataset(dataset_id=dataset_id, parents=parents).metadata + + install_backends(monkeypatch, _Child()) + # The access row is long, so the console must not wrap it. + monkeypatch.setattr(_cli_module, "console", Console(width=1000)) + flags = ["--local-registry", str(tmp_path / "store"), "--remote-registry", str(tmp_path / "empty"), "--auto-build"] + + result = runner.invoke(app, ["info", "test/child", *flags]) + + assert result.exit_code == 0, result.output + access = "open (this card only: the access of its 2 parents is known after a build)" + assert ["access", access] in _table_cells(click.unstyle(result.stdout)) + + +def test_info_of_a_dataset_that_only_a_connector_has_reads_its_card_once(tmp_path, monkeypatch): + reads = [] + + class _Counting(_Backend): + def card(self, dataset_id): + reads.append(dataset_id) + return super().card(dataset_id) + + install_backends(monkeypatch, _Counting()) + flags = ["--local-registry", str(tmp_path / "store"), "--remote-registry", str(tmp_path / "empty"), "--auto-build"] + + result = runner.invoke(app, ["info", "test/gated", *flags]) + + assert result.exit_code == 0, result.output + # Each resolution of the release reads the card, so one read means one resolution. + assert reads == ["test/gated"] + + def _write(root, dataset): with TimeFWriter(root, dataset) as writer: writer.write() @@ -192,11 +280,10 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): child.import_from(reader) _write(root, child) - result = runner.invoke(app, ["info", "test/child", "--remote-registry", str(root)]) + result = runner.invoke(app, ["info", "test/child", "--remote-registry", str(root), "--no-auto-build"]) assert result.exit_code == 0, result.output - lines = click.unstyle(result.output).splitlines() - cells = [[cell.strip() for cell in line.split("│")[1:-1]] for line in lines if "│" in line] + cells = _table_cells(click.unstyle(result.output)) start = cells.index(["access", "credentialed"]) # The card of the child stays open. Its effective level comes from its parents, and each parent # names its own page. @@ -205,6 +292,7 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): ["", "test/b@1.0.0 (credentialed): https://b.example"], ] assert cells[start + 3][0] == "domains" + assert ["records", str(len(child.records))] in cells def test_download_command_copies_into_the_store_and_then_has_nothing_to_do(registry_root, tmp_path, monkeypatch): diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 25ea84669..967c013e5 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -19,6 +19,7 @@ TimeNetDatasetNotFoundError, TimeNetFileError, TimeNetMissingParentError, + TimeNetNotBuiltError, TimeNetRegistryError, TimeNetVersionConflictError, TimeNetVersionUnavailableError, @@ -308,7 +309,7 @@ def test_installed_connectors_need_a_local_registry_that_is_not_read_only(tmp_pa ) -def test_get_list_and_search_ask_the_first_registry_that_is_not_the_store(registry_root, tmp_path): +def test_list_and_search_ask_the_first_registry_that_is_not_the_store(registry_root, tmp_path): store = LocalRegistry(tmp_path / "store") other = make_dataset() other._metadata = other.metadata.model_copy(update={"dataset_id": "test/other"}) @@ -317,14 +318,86 @@ def test_get_list_and_search_ask_the_first_registry_that_is_not_the_store(regist assert [metadata.dataset_id for metadata in client.list()] == ["timenet/hello-world"] assert [metadata.dataset_id for metadata in client.search(query="hello")] == ["timenet/hello-world"] - assert client.get("timenet/hello-world").dataset_id == "timenet/hello-world" - with pytest.raises(TimeNetDatasetNotFoundError): - client.get("test/other") # Without such a registry, they ask the store. alone = TimeNet(sources=[store, InstalledConnectors()]) assert [metadata.dataset_id for metadata in alone.list()] == ["test/other"] +def test_get_manifest_returns_the_manifest_of_the_release_that_open_reads(tmp_path): + store, remote = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "remote") + store.put_dataset(make_dataset()) + remote.put_dataset(make_dataset(dataset_version="1.1.0")) + client = TimeNet(sources=[store, remote]) + assert str(client.get_manifest("timenet/hello-world").metadata.dataset_version) == "1.0.0" + assert str(client.get_manifest("timenet/hello-world@latest").metadata.dataset_version) == "1.1.0" + + +def test_get_manifest_of_a_release_that_is_not_built_raises_with_the_card_of_the_connector(tmp_path, monkeypatch): + install_backends(monkeypatch, _Builder([])) + client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), InstalledConnectors()]) + # Without a version, the error names the version that the connector declares. + with pytest.raises(TimeNetNotBuiltError) as caught: + client.get_manifest("test/child") + assert caught.value.release == DatasetRef.parse("test/child@1.0.0") + assert caught.value.card == _Builder([]).card("test/child") + assert str(caught.value) == "test/child@1.0.0 is not built yet: only an installed connector provides it." + assert client.get_card("test/child").dataset_id == "test/child" + assert not (tmp_path / "store").exists() + + +def test_get_manifest_returns_the_listed_manifest_of_a_release_that_is_not_served(tmp_path): + # The open card of test/child reads a credentialed parent, test/a. + hosted = LocalRegistry(tmp_path / "hosted") + parent = make_dataset(dataset_id="test/a", access=Access.CREDENTIALED, access_url="https://a.example") + reference = hosted.put_dataset(parent) + child = TimeFDataset( + metadata=make_dataset().metadata.model_copy( + update={"dataset_id": "test/child", "parents": {ParentAlias("a"): reference}} + ) + ) + with TimeNet(sources=[hosted]).open("test/a") as reader: + child.import_from(reader) + hosted.put_dataset(child) + transport, _ = build_service_fake(tmp_path / "hosted") + remote = RemoteRegistry("http://api.local", transport=transport) + client = TimeNet(sources=[remote]) + + manifest = client.get_manifest("test/child") + + # The hosted registry lists the release without serving its files, so the manifest keeps its access pages. + assert [card.metadata.dataset_id for card in manifest.lineage] == ["test/a"] + assert manifest.lineage[0].metadata.access_url == "https://a.example" + assert client.get_card("test/child") == manifest.metadata + with pytest.raises(TimeNetAccessError): + client.open("test/child") + + +def test_get_card_reads_the_card_from_the_manifest_of_a_registry(tmp_path, monkeypatch): + # The connector declares a newer version, so the card of the exact version comes from the registry. + monkeypatch.setattr(_Builder, "VERSION", "2.0.0") + install_backends(monkeypatch, _Builder([])) + store, remote = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "remote") + remote.put_dataset(make_dataset()) + client = TimeNet(sources=[store, remote, InstalledConnectors()]) + + assert client.get_card("timenet/hello-world@1.0.0") == remote.get_manifest("timenet/hello-world").metadata + assert str(client.get_card("timenet/hello-world@latest").dataset_version) == "2.0.0" + + +def test_get_manifest_names_every_source_when_none_provides_the_release(tmp_path): + sources = [LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "empty")] + + with pytest.raises(TimeNetDatasetNotFoundError) as caught: + TimeNet(sources=sources).get_manifest("x/y") + + names = ", ".join(str(source) for source in sources) + # The hint is the hint of open(). + assert str(caught.value) == ( + f"x/y is in none of the sources: {names}. " + "Add a registry that holds it to the sources, or add InstalledConnectors() to build it." + ) + + def test_download_needs_a_local_registry_that_is_not_read_only(registry_root): with pytest.raises(TimeFValidationError) as caught: TimeNet(sources=[LocalRegistry(registry_root, read_only=True)]).download("timenet/hello-world") @@ -339,13 +412,10 @@ def test_download_of_a_release_that_only_a_connector_provides_builds_nothing(tmp install_backends(monkeypatch, _Builder(calls)) store = LocalRegistry(tmp_path / "store") - with pytest.raises(TimeNetDatasetNotFoundError) as caught: + with pytest.raises(TimeNetNotBuiltError) as caught: TimeNet(sources=[store, InstalledConnectors()]).download("timenet/hello-world") - assert str(caught.value) == ( - "timenet/hello-world@1.0.0 is not built yet: only an installed connector provides it. " - "open() or load() builds it into the local registry." - ) + assert str(caught.value) == "timenet/hello-world@1.0.0 is not built yet: only an installed connector provides it." assert calls == [] assert not store.root.exists() @@ -358,7 +428,7 @@ def test_download_of_latest_fails_when_a_connector_declares_a_newer_version(regi client = TimeNet(sources=[store, str(registry_root), InstalledConnectors()]) # "Latest" is the version of the connector, which no registry holds. - with pytest.raises(TimeNetDatasetNotFoundError, match=r"^timenet/hello-world@2\.0\.0 is not built yet: "): + with pytest.raises(TimeNetNotBuiltError, match=r"^timenet/hello-world@2\.0\.0 is not built yet: "): client.download("timenet/hello-world@latest") assert calls == [] # Without a version, the download takes the registry, the first source that serves the dataset. @@ -721,10 +791,10 @@ def versioned_registry(tmp_path): def test_version_ref_pins_and_defaults_to_latest(versioned_registry): client = TimeNet(sources=[str(versioned_registry)]) - assert str(client.get("timenet/hello-world").metadata.dataset_version) == "1.1.0" # no ref -> latest - assert str(client.get("timenet/hello-world@latest").metadata.dataset_version) == "1.1.0" - assert str(client.get("timenet/hello-world@1.0.0").metadata.dataset_version) == "1.0.0" # pinned - assert str(client.get("timenet/hello-world", "1.0.0").metadata.dataset_version) == "1.0.0" # explicit arg + assert str(client.get_manifest("timenet/hello-world").metadata.dataset_version) == "1.1.0" # no ref -> latest + assert str(client.get_manifest("timenet/hello-world@latest").metadata.dataset_version) == "1.1.0" + assert str(client.get_manifest("timenet/hello-world@1.0.0").metadata.dataset_version) == "1.0.0" # pinned + assert str(client.get_manifest("timenet/hello-world", "1.0.0").metadata.dataset_version) == "1.0.0" def test_open_and_load_read_the_version_asked_for(versioned_registry): @@ -752,13 +822,13 @@ def recording_open(client, *args, **kwargs): def test_version_ref_missing_pin_raises(versioned_registry): client = TimeNet(sources=[str(versioned_registry)]) with pytest.raises(TimeNetDatasetNotFoundError): - client.get("timenet/hello-world@9.9.9") + client.get_manifest("timenet/hello-world@9.9.9") def test_version_given_twice_raises(versioned_registry): client = TimeNet(sources=[str(versioned_registry)]) with pytest.raises(ValueError, match="twice"): - client.get("timenet/hello-world@1.0.0", "1.1.0") + client.get_manifest("timenet/hello-world@1.0.0", "1.1.0") def test_load_names_the_buildable_version_when_a_pin_cannot_be_built(registry_root, monkeypatch): diff --git a/packages/timenet/tests/test_errors.py b/packages/timenet/tests/test_errors.py index 4e3d72114..6a4883171 100644 --- a/packages/timenet/tests/test_errors.py +++ b/packages/timenet/tests/test_errors.py @@ -14,10 +14,12 @@ TimeNetError, TimeNetFileError, TimeNetMissingParentError, + TimeNetNotBuiltError, TimeNetRegistryError, TimeNetVersionConflictError, TimeNetVersionUnavailableError, ) +from timenet.testing import make_dataset from timenet.types import Access, Version from timenet.types.access import AccessRequirement from timenet.types.references import DatasetRef @@ -164,3 +166,9 @@ def test_an_error_with_facts_ends_with_the_hint_of_its_front_end(error, facts, m assert all(isinstance(restored, kind) for kind in caught_as) # The message ends with a period, and the hint follows as its own sentence. assert str(restored) == f"{message} Do this instead." + + +def test_a_not_built_error_keeps_its_release_and_card_through_pickle(): + card = make_dataset().metadata + restored = pickle.loads(pickle.dumps(TimeNetNotBuiltError(_A, card))) + assert (restored.release, restored.card) == (_A, card)