diff --git a/examples/load_tsqa.py b/examples/load_tsqa.py index 9256ee49c..31e9a5763 100644 --- a/examples/load_tsqa.py +++ b/examples/load_tsqa.py @@ -11,7 +11,7 @@ from pathlib import Path import sys -from timenet.client import TimeNet +from timenet import TimeNet def main() -> None: diff --git a/examples/test_mean_classifier.py b/examples/test_mean_classifier.py index e99f382d8..b19e027ac 100644 --- a/examples/test_mean_classifier.py +++ b/examples/test_mean_classifier.py @@ -17,13 +17,13 @@ from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split -from timenet.client import TimeNet -import timenet_connectors +import timenet +from timenet import TimeNet # Build and load through the same explicit local registry. registry = Path(".timenet-registry") -timenet_connectors.build("timenet/test-mean", out=registry) +timenet.build("timenet/test-mean", local_registry=registry) dataset = TimeNet(sources=[registry]).load("timenet/test-mean") # A quick tour of what we loaded: identity, counts, specs, and a record preview. diff --git a/packages/timenet-connectors/src/timenet_connectors/__init__.py b/packages/timenet-connectors/src/timenet_connectors/__init__.py index b650362a8..42d33ab39 100644 --- a/packages/timenet-connectors/src/timenet_connectors/__init__.py +++ b/packages/timenet-connectors/src/timenet_connectors/__init__.py @@ -5,8 +5,6 @@ (see :mod:`timenet_connectors.discovery`). Reusable bases, for example for the HuggingFace Hub, live under ``timenet_connectors.bases``. The connector contract is :class:`timenet.connectors.BaseConnector`. -The module-level helpers :func:`build` and :func:`load` (from :mod:`timenet_connectors.api`) are the -producer-side shortcuts. Use them to build and consume a dataset from local code. +To build a dataset from Python, call ``timenet.build("org/name")``, and read it with +``TimeNet().load("org/name")``. """ - -from timenet_connectors.api import build as build, load as load diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py deleted file mode 100644 index dd295cbae..000000000 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Build and load datasets with installed connectors.""" - -from pathlib import Path -from typing import TYPE_CHECKING - - -if TYPE_CHECKING: - from timenet.dataset import TimeFDataset - from timenet.registry import BaseRegistry - - -def build( # ruff: ignore[too-many-arguments] - dataset_id: str, - *, - version: str | None = None, - out: str | Path | None = None, - force: bool = False, - keep_cache: bool = False, - fetch_from: "str | BaseRegistry | None" = None, - build_parents: bool = False, -) -> Path: - # ruff: ignore[docstring-extraneous-exception] (raised by the plan and its run, not directly here) - """Build a dataset release in a local registry. - - The connector runs in an environment built from its ``requirements.txt``. - Set ``TIMENET_ISOLATION=off`` to use the current interpreter. - - Args: - dataset_id: The ``org/name`` ID declared in the connector's card. - version: The required card version, or ``None`` to accept any version. - out: The output directory, or ``None`` to use ``TIMENET_LOCAL_REGISTRY``. - force: Rebuild an existing release instead of reusing it. - keep_cache: Keep raw downloads after a successful build. - fetch_from: A registry or registry URI to copy missing parent releases from. - build_parents: Build missing parent releases from their installed connectors. - - Returns: - The release's version directory. - - Raises: - TimeNetDatasetNotFoundError: If no installed connector builds ``dataset_id``. - TimeNetVersionUnavailableError: If the card version differs from ``version``. - TimeNetMissingParentError: If a required parent release is missing. - """ - from timenet.builders import BuildOptions, declared # ruff: ignore[import-outside-top-level] - from timenet.config import settings # ruff: ignore[import-outside-top-level] - from timenet.coordinator.build import plan_release_build, run_plan # ruff: ignore[import-outside-top-level] - from timenet.errors import TimeNetMissingParentError # ruff: ignore[import-outside-top-level] - from timenet.registry import LocalRegistry, open_registry # ruff: ignore[import-outside-top-level] - from timenet.registry._paths import release_dir # ruff: ignore[import-outside-top-level, import-private-name] - from timenet.sources import InstalledConnectors # ruff: ignore[import-outside-top-level] - - release = declared(dataset_id).release - # TODO(adrian): Remove temporary glue which maps a missing out onto the local registry of the default - # list until timenet.build() replaces this function. - root = Path(out).expanduser() if out is not None else settings().local_registry_dir - source = open_registry(fetch_from) if isinstance(fetch_from, str) else fetch_from - store = LocalRegistry(root) - try: - plan = plan_release_build( - f"{dataset_id}@{version}" if version else dataset_id, - store=store, - # TODO(adrian): Remove temporary glue which maps fetch_from and build_parents onto a - # list of sources until the producer API replaces this function. - parents_from=[ - *([source] if source is not None else []), - *([InstalledConnectors()] if build_parents else []), - ], - force=force, - ) - run_plan(plan, store, options=BuildOptions(keep_cache=keep_cache)) - except TimeNetMissingParentError as error: - error.hint = _missing_hint(fetch_from=fetch_from, build_parents=build_parents) - raise - return release_dir(root, release) - - -def _missing_hint(*, fetch_from: "str | BaseRegistry | None", build_parents: bool) -> str | None: - """Suggest options to obtain a missing release. - - Args: - fetch_from: The selected registry or URI, or ``None``. - build_parents: Whether parent builds are enabled. - - Returns: - A sentence naming unused options, or ``None`` if both are enabled. - """ - if fetch_from is None and not build_parents: - return "Copy it with fetch_from= or build it with build_parents=True." - if fetch_from is None: - return "Copy it with fetch_from=." - if not build_parents: - return "Build it with build_parents=True." - return None - - -def load( - dataset_id: str, - version: str | None = None, - *, - fetch_from: "str | BaseRegistry | None" = None, - build_parents: bool = False, -) -> "TimeFDataset": - """Build a dataset if needed and load its values into memory. - - The build writes to the default local registry. - - Args: - dataset_id: The dataset ID (``org/name``). - version: The version declared in the connector's card, or ``None`` for the latest. - fetch_from: A registry or URI to copy missing dependencies from. - build_parents: Build missing parent releases from their installed connectors. - - Returns: - The dataset with all stored values in memory. - """ - from timenet.client import TimeNet # ruff: ignore[import-outside-top-level] - - build(dataset_id, version=version, fetch_from=fetch_from, build_parents=build_parents) - return TimeNet().load(dataset_id, version) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index dd79fad25..dc39a036a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -206,13 +206,13 @@ def _build_to( # ruff: ignore[too-many-arguments] ) -> str: """Build a release for a local or remote output. - Remote outputs receive a copy of a build in the local registry. If the output already holds the release and - ``force`` is not set, nothing runs and a status line says so. + Remote registries receive a copy from the local registry. + The build skips an existing release unless ``force`` is set. Args: dataset_id: The dataset ID. target: The local output directory or remote registry. - fetch_from: The registry to copy missing parents from, or ``None`` to use the remote output if present. + fetch_from: The registry for missing parents, or ``None`` to use the remote output if present. build_parents: Build missing parents with installed connectors. force: Rebuild an existing local release. options: Options for each build. diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index f69c385aa..9e7f8918b 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -30,7 +30,6 @@ from timenet.types.reference_keys import ParentAlias, SourceKey from timenet.types.references import DatasetRef from timenet.values_backends import ValuesBackend -import timenet_connectors from timenet_connectors.builder import backend as backend_module, cli as cli_module, env as env_module from timenet_connectors.builder.backend import ConnectorBuilder from timenet_connectors.builder.cli import app @@ -535,45 +534,29 @@ def test_build_prepares_parents_before_the_isolated_child(tmp_path, monkeypatch) assert result.stdout.strip() == str(out / CHILD.dataset_id / str(CHILD.version)) -@pytest.mark.parametrize("obtain", ["fetch_from", "build_parents"]) -def test_load_obtains_a_missing_parent_before_it_builds_and_reads(tmp_path, monkeypatch, obtain): +def test_every_build_entry_point_prepares_parents(tmp_path, monkeypatch, child_builds): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "out")) - calls = [] - monkeypatch.setattr(backend_module, "run_isolated_build", _prepared_builds(calls)) - origin = LocalRegistry(tmp_path / "origin") - parent = make_dataset() - parent._metadata = cards[PARENT.dataset_id] - origin.put_dataset(parent) - - with pytest.raises(TimeNetMissingParentError, match=r"^parent release test/parent@1\.0\.0 of test/child") as caught: - timenet_connectors.load(CHILD.dataset_id) - assert str(caught.value).endswith(". Copy it with fetch_from= or build it with build_parents=True.") - if obtain == "fetch_from": - dataset = timenet_connectors.load(CHILD.dataset_id, fetch_from=origin) - else: - dataset = timenet_connectors.load(CHILD.dataset_id, build_parents=True) + out = tmp_path / "out" - assert dataset.metadata.dataset_id == CHILD.dataset_id - # The child holds the Record it imported from the parent. - assert [record.id for record in dataset.records] == [make_dataset().records[0].id] - builds = [(CHILD.dataset_id, False)] - assert calls == (builds if obtain == "fetch_from" else [(PARENT.dataset_id, False), *builds]) + def build(origin=None): + # The client and timenet.build() build through the coordinator. + store = LocalRegistry(out) + run_plan( + plan_release_build(CHILD.dataset_id, store=store, parents_from=[] if origin is None else [origin]), store + ) + with pytest.raises(TimeNetMissingParentError, match=r"parent release test/parent@1\.0\.0 of test/child@1\.0\.0"): + build() + assert child_builds == [] -def test_load_copies_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): - _install_cards(monkeypatch, parent={"parent": GRAND}) - out = LocalRegistry(tmp_path / "out") - _parent_without_its_source(out) + # A registry instance to fetch from, as the SDK layers accept one. origin = LocalRegistry(tmp_path / "origin") - origin.put_dataset(make_dataset()) - monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(out.root)) - - dataset = timenet_connectors.load(PARENT.dataset_id, fetch_from=origin) - - assert len(dataset.records) == 2 - # The build copies the payload source into the store, where the read finds it. - assert out.exists(GRAND.dataset_id, str(GRAND.version)) + parent = make_dataset() + parent._metadata = cards[PARENT.dataset_id] + origin.put_dataset(parent) + build(origin) + assert child_builds == [(CHILD.dataset_id, False)] + assert LocalRegistry(out).exists(PARENT.dataset_id, str(PARENT.version)) def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch): @@ -813,26 +796,17 @@ def test_the_requested_build_waits_for_a_payload_source_of_a_held_parent(tmp_pat ) -@pytest.mark.parametrize( - ("entry", "hint"), [("cli", "Build it with --build-parents."), ("api", "Copy it with fetch_from=.")] -) -def test_the_hint_names_only_the_option_that_the_build_did_not_use(tmp_path, monkeypatch, entry, hint): - # No registry holds the parent, and no installed connector builds it. The CLI build uses - # --fetch-from, and the API build uses build_parents=True. +def test_the_hint_names_only_the_option_that_the_build_did_not_use(tmp_path, monkeypatch): + # No registry holds the parent, and no installed connector builds it. The build uses --fetch-from. cards = _install_cards(monkeypatch, child={"parent": PARENT}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) out = tmp_path / "out" - if entry == "cli": - options = ["--fetch-from", str(tmp_path / "origin")] - error = runner.invoke(app, ["build", CHILD.dataset_id, "--out", str(out), *options]).exception - else: - with pytest.raises(TimeNetMissingParentError) as caught: - timenet_connectors.build(CHILD.dataset_id, out=out, build_parents=True) - error = caught.value + options = ["--fetch-from", str(tmp_path / "origin")] + error = runner.invoke(app, ["build", CHILD.dataset_id, "--out", str(out), *options]).exception assert isinstance(error, TimeNetMissingParentError) - assert error.hint == hint + assert error.hint == "Build it with --build-parents." def _count_reads(monkeypatch, registry) -> dict[str, int]: diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 853617a91..51c8e38d5 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -6,12 +6,11 @@ from timenet.client import TimeNet from timenet.config import settings -from timenet.errors import TimeFValidationError, TimeNetRegistryError +from timenet.errors import TimeNetRegistryError from timenet.provenance import build_env from timenet.registry import LocalRegistry, RemoteRegistry from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal -from timenet_connectors import build, load from timenet_connectors.builder.cli import _resolve_target, app as build_app from timenet_connectors.datasets.timenet.hello_world import HelloWorldConnector from timenet_connectors.discovery import available, resolve @@ -63,17 +62,6 @@ def test_build_unknown_id_fails(tmp_path): assert result.exit_code != 0 -def test_build_rejects_version_mismatch(): - # The guard fires before the pipeline runs, so no registry is written. - with pytest.raises(TimeFValidationError, match="builds version"): - build("timenet/hello-world", version="9.9.9") - - -def test_load_rejects_version_mismatch(): - with pytest.raises(TimeFValidationError, match="builds version"): - load("timenet/hello-world", "9.9.9") - - # ---- where a bare `build` writes ----------------------------------------------------------------- @@ -111,17 +99,6 @@ def test_resolve_target_rejects_an_empty_out_instead_of_the_current_directory(): _resolve_target("") -def test_sdk_build_cleans_cache_but_keep_cache_retains(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_ISOLATION", "off") # the in-process path, the one that kept the cache - cache = settings().downloads_dir / "timenet" / "hello-world" - - build("timenet/hello-world", out=tmp_path / "r1") - assert not cache.exists() - - build("timenet/hello-world", out=tmp_path / "r2", keep_cache=True) - assert cache.exists() - - def test_build_cleans_cache_but_keep_flag_retains(tmp_path): cache = settings().downloads_dir / "timenet" / "hello-world" diff --git a/packages/timenet-connectors/tests/test_package_api.py b/packages/timenet-connectors/tests/test_timenet_build.py similarity index 73% rename from packages/timenet-connectors/tests/test_package_api.py rename to packages/timenet-connectors/tests/test_timenet_build.py index 1e189abc2..318467fc2 100644 --- a/packages/timenet-connectors/tests/test_package_api.py +++ b/packages/timenet-connectors/tests/test_timenet_build.py @@ -1,14 +1,18 @@ import pytest +import timenet +from timenet.client import TimeNet from timenet.errors import TimeNetVersionUnavailableError from timenet.registry import LocalRegistry from timenet.testing import make_dataset from timenet.types import ClassificationTask from timenet.types.references import DatasetRef -import timenet_connectors from timenet_connectors.builder import backend as backend_module +HELLO = DatasetRef.parse("timenet/hello-world@1.0.0") + + def _unexpected_isolation(*args, **kwargs): raise AssertionError(f"run_isolated_build should not have run: {args} {kwargs}") @@ -17,16 +21,15 @@ def _unexpected_resolve(*args, **kwargs): raise AssertionError(f"resolve should not have run: {args} {kwargs}") -def test_build_then_load_round_trip(monkeypatch): - # timenet_connectors.build/load anchor on the store of the default list, so a build here loads back. +def test_build_then_load_round_trip(monkeypatch, tmp_path): # In-process, so the round trip stays in this interpreter instead of shelling out to uv. monkeypatch.setenv("TIMENET_ISOLATION", "off") monkeypatch.setattr(backend_module, "run_isolated_build", _unexpected_isolation) + out = tmp_path / "out" - version_dir = timenet_connectors.build("timenet/test-mean") - assert (version_dir / "manifest.json").exists() + assert timenet.build("timenet/test-mean", local_registry=out) == out / "timenet/test-mean/1.0.0" - dataset = timenet_connectors.load("timenet/test-mean") + dataset = TimeNet(sources=[str(out)]).load("timenet/test-mean") assert len(dataset.records) == 1000 x, y = dataset.to_features_and_targets(task=ClassificationTask) @@ -46,11 +49,10 @@ def fake_isolated(release, store_root, *, force=False, keep_cache=False, quiet=F monkeypatch.setattr(backend_module, "run_isolated_build", fake_isolated) - version_dir = timenet_connectors.build("timenet/hello-world", out=str(tmp_path), force=True) + timenet.build("timenet/hello-world", local_registry=tmp_path / "out", force=True) # The child is not quiet: it shows its downloads and shards. - assert calls == [(DatasetRef.parse("timenet/hello-world@1.0.0"), tmp_path, True, False, False)] - assert version_dir == tmp_path / "timenet" / "hello-world" / "1.0.0" + assert calls == [(HELLO, tmp_path / "out", True, False, False)] def test_build_rejects_a_version_the_connector_does_not_declare(monkeypatch, tmp_path): @@ -60,7 +62,7 @@ def test_build_rejects_a_version_the_connector_does_not_declare(monkeypatch, tmp with pytest.raises( TimeNetVersionUnavailableError, match=r"builds version 1\.0\.0, not the requested 9\.9\.9" ) as caught: - timenet_connectors.build("timenet/hello-world", version="9.9.9", out=tmp_path) + timenet.build("timenet/hello-world@9.9.9", local_registry=tmp_path / "out") assert str(caught.value.declared) == "1.0.0" - assert not tmp_path.joinpath("timenet").exists() + assert not (tmp_path / "out").exists() diff --git a/packages/timenet/src/timenet/__init__.py b/packages/timenet/src/timenet/__init__.py index 04a914252..7bcbfb566 100644 --- a/packages/timenet/src/timenet/__init__.py +++ b/packages/timenet/src/timenet/__init__.py @@ -1 +1,56 @@ -"""TimeNet: register, query, download, convert, and explore time series datasets.""" +"""Build, publish, and read time series datasets.""" + +import importlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from timenet.client import TimeNet + from timenet.producer import build, publish + from timenet.registry import LocalRegistry + from timenet.sources import InstalledConnectors + + +__all__ = [ + "InstalledConnectors", + "LocalRegistry", + "TimeNet", + "build", + "publish", +] + + +def __getattr__(name: str) -> object: + """Load an exported name on first access, so ``import timenet`` stays cheap. + + Args: + name: The attribute name. + + Returns: + The exported object. + + Raises: + AttributeError: If the package does not export the name. + """ + module = { + "InstalledConnectors": "timenet.sources", + "LocalRegistry": "timenet.registry", + "TimeNet": "timenet.client", + "build": "timenet.producer", + "publish": "timenet.producer", + }.get(name) + if module is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(importlib.import_module(module), name) + # Later lookups find the name without this function. + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """List package attributes, including exports that have not loaded yet. + + Returns: + The attribute names in sorted order. + """ + return sorted({*globals(), *__all__}) diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 8115391de..1267d8feb 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -3,6 +3,7 @@ from collections.abc import Callable, Iterable from functools import partial from pathlib import Path +import shlex from typing import TypeVar from rich.console import Console @@ -15,7 +16,10 @@ from timenet.cli.ui import console as ui, progress_renderer from timenet.client import TimeNet, default_local_registry, default_sources from timenet.config import settings +from timenet.errors import TimeNetDatasetNotFoundError +from timenet.producer import publish as publish_release from timenet.progress import ProgressCallback, ProgressEvent, StepStarted +from timenet.registry import open_registry from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access from timenet.types.references import split_ref @@ -232,6 +236,51 @@ def _keep_steps(started: list[StepStarted], show: ProgressCallback | None, event show(event) +@app.command() +def publish( + dataset_id: str, + dest: str = typer.Option(..., "--dest", metavar="URL|PATH", help="The registry to copy to: a URL or a local path."), + source: str | None = typer.Option( + None, + "--source", + metavar="URL|PATH", + help="The registry to copy from: a URL or a local path (default: the local registry).", + ), + local_registry: str | None = LOCAL_REGISTRY, +) -> None: + """Copy a release and its payload sources to a registry, and print the release to stdout. + + The source defaults to the local registry. + The copy skips identical releases and rejects releases with conflicting files. + \f + + Raises: + TimeNetDatasetNotFoundError: If the source lacks the release. + """ # ruff: ignore[escape-sequence-in-docstring] - the form feed ends the help text of the command + store = default_local_registry(local_registry=local_registry) + origin = store if source is None else open_registry(source, label="--source") + try: + with progress_renderer() as progress: + release = publish_release(dataset_id, dest=dest, source=origin, progress=progress) + except TimeNetDatasetNotFoundError as error: + # The hint fits only a release that the local registry lacks. A subclass, such as a missing payload + # source, names its own registries. + if source is not None or type(error) is not TimeNetDatasetNotFoundError: + raise + bare_id, version = split_ref(dataset_id) + wanted = bare_id if version is None else f"{bare_id}@{version}" + # Repeat the path of --local-registry, so that each command of the hint uses it too. + path = shlex.quote(str(store.root)) + out, flag = ("", "") if local_registry is None else (f" --out {path}", f" --local-registry {path}") + hint = ( + f"Build it into the local registry with `timenet-build build {wanted}{out}`, " + f"or copy it there with `timenet download {wanted}{flag}`." + ) + raise TimeNetDatasetNotFoundError(f"{str(error).rstrip('.')}. {hint}") from error + ui.success(f"Published {release} to {dest}") + typer.echo(str(release)) + + local_app = typer.Typer(help="Inspect and clear the local files of TimeNet.", no_args_is_help=True) app.add_typer(local_app, name="local") cache_app = typer.Typer( diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 2d5a5a834..4b2ead3c7 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -124,57 +124,67 @@ def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None] return ref_id, ref_version if ref_version is not None else (version or None) -def _open_sources(sources: Sequence[ReleaseSource | str | Path] | None) -> tuple[ReleaseSource, ...]: - """Validate sources and open registry URLs and paths. +def as_release_sources(items: Sequence[ReleaseSource | str | Path], *, keyword: str) -> list[ReleaseSource]: + """Convert registry URLs and paths to release sources. + + Existing source objects keep their identity and position. A path opens a writable local registry. Args: - sources: A nonempty sequence of sources, registry URLs, or paths. + items: A sequence of sources, registry URLs, or local paths. + keyword: The argument name to show in errors. Returns: - A tuple of sources in the given order. + The sources in their original order, or an empty list for empty input. Raises: - TimeFValidationError: If sources are not a nonempty sequence of valid sources, registry URLs, or paths. + TimeFValidationError: If ``items`` is not a valid list. TimeNetRegistryError: If a registry URL or path is blank or cannot be opened. """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry - if isinstance(sources, (str, os.PathLike)): - shown = repr(str(sources)) - raise TimeFValidationError( - f"sources takes a list of sources. Pass sources=[{shown}], or " - f"sources=[{shown}, InstalledConnectors()] to build what the path lacks" - ) - if not isinstance(sources, Sequence): - raise TimeFValidationError(f"sources takes a list of sources. Pass sources=[{type(sources).__name__}(...)]") - if not sources: - raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") + if isinstance(items, (str, os.PathLike)): + raise TimeFValidationError(f"{keyword} takes a list of sources. Pass {keyword}=[{str(items)!r}].") + if not isinstance(items, Sequence): + raise TimeFValidationError(f"{keyword} takes a list of sources. Pass {keyword}=[{type(items).__name__}(...)].") opened: list[ReleaseSource] = [] - for index, item in enumerate(sources): + for index, item in enumerate(items): if isinstance(item, (str, os.PathLike)): - opened.append(open_registry(item if isinstance(item, str) else Path(item), label=f"sources[{index}]")) + path = item if isinstance(item, str) else Path(item) + opened.append(open_registry(path, label=f"{keyword}[{index}]")) elif isinstance(item, (BaseRegistry, InstalledConnectors)): opened.append(item) else: - raise TimeFValidationError(f"sources[{index}] is not a source, a registry URL or a path: {item!r}") - return tuple(opened) + raise TimeFValidationError(f"{keyword}[{index}] is not a source, a registry URL or a path: {item!r}") + return opened class TimeNet: """Browse, download, and read TimeNet datasets.""" def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) -> None: - """Create a client with an ordered list of dataset sources. + """Create a client that searches dataset sources in order. Sources accept registries, registry URLs or paths, and ``InstalledConnectors()``. A path opens a writable ``LocalRegistry``. Builds land in the first ``LocalRegistry`` without ``read_only=True``, which ``InstalledConnectors()`` requires. Args: - sources: Ordered registries, URLs, paths or connectors, or ``None`` for :func:`default_sources`. + sources: Ordered sources, registry URLs or paths, or ``None`` for :func:`default_sources`. Raises: - TimeFValidationError: If sources are invalid, or have connectors but no writable local registry. + TimeFValidationError: If ``sources`` is not a non-empty list of valid sources, or has installed + connectors but no writable local registry. """ - self._sources = tuple(default_sources()) if sources is None else _open_sources(sources) + if sources is None: + self._sources = tuple(default_sources()) + else: + if isinstance(sources, (str, os.PathLike)): + shown = repr(str(sources)) + raise TimeFValidationError( + f"sources takes a list of sources. Pass sources=[{shown}], or " + f"sources=[{shown}, InstalledConnectors()] to build what the path lacks" + ) + self._sources = tuple(as_release_sources(sources, keyword="sources")) + if not self._sources: + raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") self._store = next( (source for source in self._sources if isinstance(source, LocalRegistry) and not source.read_only), None ) diff --git a/packages/timenet/src/timenet/coordinator/__init__.py b/packages/timenet/src/timenet/coordinator/__init__.py index fce0d199d..5dd106f24 100644 --- a/packages/timenet/src/timenet/coordinator/__init__.py +++ b/packages/timenet/src/timenet/coordinator/__init__.py @@ -1,4 +1,5 @@ """Plan and run dataset builds and reads. -Not part of the user API: use ``timenet-build`` to build datasets and ``TimeNet`` to read them. +Not part of the user API: use ``timenet.build()`` or ``timenet-build`` to build datasets and +``TimeNet`` to read them. """ diff --git a/packages/timenet/src/timenet/producer.py b/packages/timenet/src/timenet/producer.py new file mode 100644 index 000000000..d32caf49d --- /dev/null +++ b/packages/timenet/src/timenet/producer.py @@ -0,0 +1,154 @@ +"""Build dataset releases in a local registry and publish them to other registries. + +The default local and remote registries come from ``TIMENET_LOCAL_REGISTRY`` and ``TIMENET_REMOTE_REGISTRY``. +``TIMENET_AUTO_BUILD`` controls whether installed connectors supply missing parents by default. +""" + +from collections.abc import Sequence +from pathlib import Path + +from timenet._registry_uri import registry_location +from timenet.builders import BuildOptions, declared +from timenet.client import as_release_sources, default_local_registry, default_sources +from timenet.coordinator.build import run_plan +from timenet.coordinator.plan import Plan, plan_steps +from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError +from timenet.progress import ProgressCallback +from timenet.registry import BaseRegistry, LocalRegistry, WritableRegistry, copy_release, open_registry +from timenet.registry._paths import release_dir +from timenet.sources import NotFound, NotServed, ReleaseSource, resolve_release +from timenet.types.references import DatasetRef, is_latest, split_ref + + +def build( # ruff: ignore[too-many-arguments] + dataset_id: str, + *, + local_registry: str | Path | LocalRegistry | None = None, + sources: Sequence[ReleaseSource | str | Path] | None = None, + force: bool = False, + open_access_only: bool = False, + keep_cache: bool = False, + progress: ProgressCallback | None = None, +) -> Path: + """Build a dataset release into a local registry with its installed connector. + + Missing parents and payload sources come from the first source in ``sources`` that provides them. + The build uses a separate environment with the connector's dependencies unless ``TIMENET_ISOLATION`` is ``off``. + + Args: + dataset_id: The dataset ID (``org/name``) or exact release (``org/name@1.2.0``). + local_registry: The writable local registry to build into: a ``LocalRegistry``, a local path, or a + ``file://`` URI, or ``None`` for ``TIMENET_LOCAL_REGISTRY``. + sources: Sources, registry URLs, or paths for missing releases, or ``None`` for + :func:`~timenet.client.default_sources` with ``local_registry`` first. + force: Build the release again, even if the local registry holds it. + open_access_only: Require open access for every release before running the plan. + keep_cache: Keep the raw downloads in ``TIMENET_CACHE`` after the build. + progress: A callback for copy and build progress events. + + Returns: + The release directory in the local registry. + + Raises: + TimeFValidationError: If ``local_registry`` is not a writable local registry, or ``sources`` is invalid. + TimeNetRegistryError: If a registry in ``sources`` fails to open. + TimeNetDatasetNotFoundError: If no installed connector builds the dataset. + TimeNetVersionUnavailableError: If the requested version differs from the connector's version. + TimeNetMissingParentError: If no source provides a required parent or payload source. + TimeNetAccessError: If no source grants access to a parent or a release violates ``open_access_only``. + TimeNetVersionConflictError: If a release to copy has other files than the same version in the local registry. + TimeNetFileError: If a stored release is damaged and no registry in ``sources`` provides it. + TimeNetBuildError: If a build fails. + """ # ruff: ignore[docstring-extraneous-exception] - raised by the coordinator and open_registry + target = _build_target(local_registry) + # The planner leaves the local registry out of the sources of its copies and builds. + parents = ( + default_sources(local_registry=target.root) + if sources is None + else as_release_sources(sources, keyword="sources") + ) + bare_id, version = split_ref(dataset_id) + # A connector builds only the version of its card, so "latest" asks for that version. + builder = declared(bare_id, None if is_latest(version) else version) + release, steps = plan_steps( + builder, target, parents_from=parents, force_build=force, open_access_only=open_access_only + ) + run_plan(Plan(tuple(steps)), target, options=BuildOptions(keep_cache=keep_cache, progress=progress)) + return release_dir(target.root, release) + + +def publish( + dataset_id: str | DatasetRef, + *, + dest: str | Path | WritableRegistry, + source: str | Path | BaseRegistry | None = None, + progress: ProgressCallback | None = None, +) -> DatasetRef: + """Copy a release and its required payload sources to another registry. + + The copy skips identical releases and rejects conflicting files before writing to ``dest``. + It replaces missing or damaged files at a local destination. + + Args: + dataset_id: ``org/name`` for the latest version in ``source``, ``org/name@1.2.0`` for an exact release, or + a ``DatasetRef``, which needs no lookup in ``source``. + dest: The registry to copy to, or its URL or path. + source: The source registry, URL, or path, or ``None`` for ``TIMENET_LOCAL_REGISTRY``. + progress: A callback for copy progress events. + + Returns: + The ID and version of the release in ``dest``. + + Raises: + TimeNetRegistryError: If ``dest`` or ``source`` fails to open as a registry, or ``dest`` is read-only. + TimeNetDatasetNotFoundError: If ``source`` lacks the release. + TimeNetAccessError: If a registry withholds files or requires open access that a copied release lacks. + TimeNetVersionConflictError: If ``dest`` holds different files for the release or a required payload source. + TimeNetMissingParentError: If a required payload source is missing from ``source`` and incomplete in ``dest``. + """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry and copy_release + target = open_registry(dest, label="dest") if isinstance(dest, (str, Path)) else dest + if source is None: + origin = default_local_registry() + else: + origin = open_registry(source, label="source") if isinstance(source, (str, Path)) else source + if isinstance(dataset_id, DatasetRef): + # The caller names the exact release, so the copy starts at once. + copy_release(dataset_id, origin, target, progress=progress) + return dataset_id + bare_id, version = split_ref(dataset_id) + answer = resolve_release([origin], bare_id, version) + if isinstance(answer, NotServed): + raise answer.access_error(bare_id) + if isinstance(answer, NotFound): + wanted = bare_id if version is None else f"{bare_id}@{version}" + raise TimeNetDatasetNotFoundError(f"{wanted} is not in {origin}.") + release = DatasetRef(dataset_id=bare_id, version=answer.version) + copy_release(release, origin, target, progress=progress) + return release + + +def _build_target(local_registry: str | Path | LocalRegistry | None) -> LocalRegistry: + """Open the local registry that a build writes into. + + Args: + local_registry: A local registry, path, or ``file://`` URI, or ``None`` for the configured one. + + Returns: + The writable local registry. + + Raises: + TimeFValidationError: If ``local_registry`` is remote, read-only, or not a registry. + TimeNetRegistryError: If ``local_registry`` is blank or has an unsupported scheme. + """ # ruff: ignore[docstring-extraneous-exception] - raised by registry_location + if local_registry is None: + return default_local_registry() + if isinstance(local_registry, (str, Path)): + location = registry_location(local_registry, label="local_registry") + if isinstance(location, Path): + return LocalRegistry(location) + elif isinstance(local_registry, LocalRegistry) and not local_registry.read_only: + return local_registry + raise TimeFValidationError( + "local_registry takes a writable local registry, because a build writes into it. Publish the result to " + "another registry with timenet.publish()." + ) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index 2b32bbfed..a8920d3d4 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -1,4 +1,5 @@ import importlib +import shutil import subprocess import sys @@ -10,9 +11,15 @@ from timenet.cache import human_bytes, stored_releases from timenet.cli.app import _enum_list, app +from timenet.client import TimeNet from timenet.config import make_cache_folder from timenet.dataset import TimeFDataset -from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError +from timenet.errors import ( + TimeFValidationError, + TimeNetDatasetNotFoundError, + TimeNetMissingParentError, + TimeNetRegistryError, +) from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version from timenet.testing import make_dataset @@ -23,6 +30,7 @@ runner = CliRunner() +HELLO = DatasetRef.parse("timenet/hello-world@1.0.0") # Import the app submodule directly so we can patch its module-global ``app`` when testing main()'s # error handling. _cli_module = importlib.import_module("timenet.cli.app") @@ -248,6 +256,90 @@ def test_each_command_builds_the_list_from_its_flags(registry_root, tmp_path): assert result.stdout == f"{store / 'timenet/hello-world/1.0.0'}\n" +def test_publish_copies_a_release_from_the_store(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + dest = tmp_path / "dest" + result = runner.invoke( + app, ["publish", "timenet/hello-world", "--dest", str(dest), "--local-registry", str(store.root)] + ) + assert result.exit_code == 0, result.output + assert result.stdout.strip() == "timenet/hello-world@1.0.0" + assert LocalRegistry(dest).exists("timenet/hello-world", "1.0.0") + assert "📥 Copying 'timenet/hello-world'…" in result.stderr + assert "✅ Published timenet/hello-world@1.0.0 to" in result.stderr + + +def test_publish_rejects_a_source_that_cannot_be_opened(tmp_path): + result = runner.invoke(app, ["publish", "x/y", "--dest", str(tmp_path / "dest"), "--source", "foo://x"]) + assert result.exit_code == 1 + assert isinstance(result.exception, TimeNetRegistryError) + assert str(result.exception).startswith("--source 'foo://x' has an unsupported registry scheme.") + + +def test_publish_to_an_empty_dest_fails_and_writes_nothing_into_the_working_directory(tmp_path, monkeypatch): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + cwd = tmp_path / "cwd" + cwd.mkdir() + monkeypatch.chdir(cwd) + argv = ["publish", "timenet/hello-world", "--dest", "", "--local-registry", str(store.root)] + result = runner.invoke(app, argv) + assert isinstance(result.exception, TimeNetRegistryError) + assert str(result.exception).startswith("dest is blank") + assert list(cwd.iterdir()) == [] + + +def test_publish_needs_dest(tmp_path): + result = runner.invoke(app, ["publish", "timenet/hello-world"]) + assert result.exit_code == 2 + # Rich colours the usage error on a CI runner, which splits the option name with escape codes. + assert "--dest" in click.unstyle(result.output) + + +def test_publish_of_a_release_that_the_store_lacks_names_the_fix(tmp_path): + store, other, dest = tmp_path / "store", tmp_path / "other", str(tmp_path / "dest") + result = runner.invoke(app, ["publish", "timenet/hello-world", "--dest", dest, "--local-registry", str(store)]) + assert type(result.exception) is TimeNetDatasetNotFoundError + assert str(result.exception) == ( + f"timenet/hello-world is not in local registry {store}. " + f"Build it into the local registry with `timenet-build build timenet/hello-world --out {store}`, " + f"or copy it there with `timenet download timenet/hello-world --local-registry {store}`." + ) + # The hint keeps the version, and names no store without --local-registry. + result = runner.invoke(app, ["publish", "timenet/hello-world@1.0.0", "--dest", dest]) + assert str(result.exception).endswith( + "Build it into the local registry with `timenet-build build timenet/hello-world@1.0.0`, " + "or copy it there with `timenet download timenet/hello-world@1.0.0`." + ) + # A registry that --source names is not the store, so the error names only that registry. + result = runner.invoke(app, ["publish", "timenet/hello-world", "--dest", dest, "--source", str(other)]) + assert str(result.exception) == f"timenet/hello-world is not in local registry {other}." + + +def test_publish_adds_no_hint_to_a_payload_source_that_the_store_lacks(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + child = TimeFDataset( + metadata=make_dataset().metadata.model_copy( + update={"dataset_id": "test/child", "parents": {ParentAlias("hello"): HELLO}} + ) + ) + # The child reads the values of one Record of the parent, so the parent is its payload source. + with TimeNet(sources=[store]).open(str(HELLO)) as reader: + child.import_from(reader, alias=ParentAlias("hello"), records=[make_dataset().records[0].id]) + store.put_dataset(child) + shutil.rmtree(store.root / "timenet") + dest = tmp_path / "dest" + + result = runner.invoke(app, ["publish", "test/child", "--dest", str(dest), "--local-registry", str(store.root)]) + + # The error of the payload source names its own registries, and the hint for the release does not apply. + assert type(result.exception) is TimeNetMissingParentError + assert str(result.exception).startswith(f"payload source {HELLO} of test/child@1.0.0 is not in ") + assert "timenet-build" not in str(result.exception) + + @pytest.mark.parametrize(("flag", "variable"), [("--auto-build", "false"), ("--no-auto-build", "true")]) def test_the_auto_build_flag_beats_its_variable(monkeypatch, flag, variable): monkeypatch.setenv("TIMENET_AUTO_BUILD", variable) diff --git a/packages/timenet/tests/test_producer.py b/packages/timenet/tests/test_producer.py new file mode 100644 index 000000000..e8a804b83 --- /dev/null +++ b/packages/timenet/tests/test_producer.py @@ -0,0 +1,302 @@ +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import httpx +import pytest + +import timenet +from timenet.client import TimeNet +from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset +from timenet.errors import ( + TimeFValidationError, + TimeNetAccessError, + TimeNetDatasetNotFoundError, + TimeNetMissingParentError, + TimeNetRegistryError, +) +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version +from timenet.sources import InstalledConnectors +from timenet.testing import install_backends, make_dataset +from timenet.types import Access +from timenet.types.reference_keys import ParentAlias +from timenet.types.references import DatasetRef + + +PARENT = DatasetRef.parse("test/parent@1.0.0") +CHILD = DatasetRef.parse("test/child@1.0.0") +HELLO = DatasetRef.parse("timenet/hello-world@1.0.0") + + +def _card(reference, **update): + """Return the card of the demo dataset under another release.""" + return make_dataset().metadata.model_copy( + update={"dataset_id": reference.dataset_id, "dataset_version": reference.version, **update} + ) + + +def _demo(card): + """Return the demo dataset under ``card``.""" + dataset = make_dataset() + dataset._metadata = card + return dataset + + +class _Backend: + """A builder backend of another package. It builds the demo dataset under each card that it holds.""" + + def __init__(self, cards, calls): + self.cards = {card.dataset_id: card for card in cards} + self.calls = calls + + def dataset_ids(self): + return list(self.cards) + + def card(self, dataset_id): + return self.cards.get(dataset_id) + + def build(self, release, store, *, force, pinned_by, options): + self.calls.append((release, options)) + store.put_dataset(_demo(self.cards[release.dataset_id]), force=force, pinned_by=pinned_by or None) + + +@pytest.fixture +def cards(monkeypatch): + """Install a card for CHILD and a card for PARENT. CHILD declares PARENT. + + Returns: + The list of the calls of the backend: ``(release, options)`` for each build. + """ + calls = [] + backend = _Backend([_card(CHILD, parents={ParentAlias("parent"): PARENT}), _card(PARENT)], calls) + install_backends(monkeypatch, backend) + return calls + + +def _store_parent(registry): + """Put a PARENT release into a registry.""" + registry.put_dataset(_demo(_card(PARENT))) + + +def test_build_with_local_registry_and_sources_takes_both_only_from_them(tmp_path, monkeypatch, cards): + env_store, origin = tmp_path / "env-store", LocalRegistry(tmp_path / "origin") + _store_parent(origin) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(env_store)) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(origin.root)) + store = tmp_path / "store" + # The remote registry of the environment holds the parent, but the list of sources is empty. + with pytest.raises(TimeNetMissingParentError): + timenet.build(CHILD.dataset_id, local_registry=store, sources=[]) + assert not env_store.exists() + assert cards == [] + + +def test_build_without_sources_copies_a_parent_from_the_remote_registry(tmp_path, monkeypatch, cards): + origin = LocalRegistry(tmp_path / "origin") + _store_parent(origin) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(origin.root)) + store = LocalRegistry(tmp_path / "store") + assert timenet.build(CHILD.dataset_id, local_registry=store.root) == store.root / "test/child/1.0.0" + assert store.exists(PARENT.dataset_id, "1.0.0") + # The parent came as a copy, so only the child was built. + assert [release for release, _ in cards] == [CHILD] + + +def test_build_lands_in_the_store_that_the_environment_names(tmp_path, monkeypatch, cards): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "env-store")) + assert timenet.build(PARENT.dataset_id) == tmp_path / "env-store/test/parent/1.0.0" + assert LocalRegistry(tmp_path / "env-store").exists(PARENT.dataset_id, "1.0.0") + + +def test_auto_build_false_leaves_the_connectors_out_of_the_default_parents(tmp_path, monkeypatch, cards): + # The remote registry of the environment is empty, so only the installed connector builds the parent. + store = tmp_path / "store" + monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") + with pytest.raises(TimeNetMissingParentError): + timenet.build(CHILD.dataset_id, local_registry=store) + assert cards == [] + + monkeypatch.setenv("TIMENET_AUTO_BUILD", "true") + timenet.build(CHILD.dataset_id, local_registry=store) + assert [release for release, _ in cards] == [PARENT, CHILD] + + +def test_open_access_only_refuses_a_gated_parent_before_any_step(tmp_path, cards): + origin, store = LocalRegistry(tmp_path / "origin"), tmp_path / "store" + gated = _card(PARENT, access=Access.CREDENTIALED, access_url="https://physionet.example/dua") + origin.put_dataset(_demo(gated)) + + with pytest.raises(TimeNetAccessError, match="credentialed"): + timenet.build(CHILD.dataset_id, local_registry=store, sources=[origin], open_access_only=True) + assert cards == [] + assert not store.exists() + + +def test_timenet_exports_the_client_and_its_sources(): + assert (timenet.TimeNet, timenet.LocalRegistry, timenet.InstalledConnectors) == ( + TimeNet, + LocalRegistry, + InstalledConnectors, + ) + + +@pytest.mark.parametrize("kind", ["remote", "url", "read-only"]) +def test_build_rejects_a_local_registry_that_it_cannot_write(tmp_path, cards, kind): + target = { + "remote": RemoteRegistry("https://registry.example.test"), + "url": "s3://bucket/registry", + "read-only": LocalRegistry(tmp_path / "store", read_only=True), + }[kind] + with pytest.raises(TimeFValidationError) as caught: + timenet.build(CHILD.dataset_id, local_registry=target) # ty: ignore[invalid-argument-type] + assert str(caught.value) == ( + "local_registry takes a writable local registry, because a build writes into it. Publish the result to " + "another registry with timenet.publish()." + ) + assert cards == [] + + +def test_build_rejects_sources_given_as_a_string(tmp_path, cards): + with pytest.raises(TimeFValidationError) as caught: + timenet.build(CHILD.dataset_id, local_registry=tmp_path / "store", sources="timenet://") + assert str(caught.value) == "sources takes a list of sources. Pass sources=['timenet://']." + + +def test_sources_of_a_build_name_the_item_that_cannot_be_opened(tmp_path, cards): + with pytest.raises(TimeNetRegistryError) as caught: + timenet.build(CHILD.dataset_id, local_registry=tmp_path / "store", sources=["timenet://", "foo://x"]) + assert str(caught.value) == ( + "sources[1] 'foo://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + ) + + +@pytest.fixture +def store(tmp_path, monkeypatch): + """Name the store with the environment, and return it empty.""" + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) + return LocalRegistry(tmp_path / "store") + + +def test_publish_copies_the_latest_release_of_the_store(tmp_path, store): + store.put_dataset(_demo(_card(DatasetRef.parse("timenet/hello-world@0.9.0")))) + store.put_dataset(make_dataset()) + dest = tmp_path / "dest" + + assert timenet.publish("timenet/hello-world", dest=dest) == DatasetRef.parse("timenet/hello-world@1.0.0") + assert LocalRegistry(dest).exists("timenet/hello-world", "1.0.0") + assert not LocalRegistry(dest).exists("timenet/hello-world", "0.9.0") + + +def test_publish_takes_a_source(tmp_path, store): + origin = LocalRegistry(tmp_path / "origin") + origin.put_dataset(make_dataset()) + dest = LocalRegistry(tmp_path / "dest") + events = [] + + assert ( + timenet.publish("timenet/hello-world@1.0.0", dest=dest, source=str(origin.root), progress=events.append) + == HELLO + ) + assert dest.exists(HELLO.dataset_id, "1.0.0") + assert not store.root.exists() + assert [type(event).__name__ for event in events if type(event).__name__ != "BytesMoved"] == [ + "StepStarted", + "StepFinished", + ] + + +def test_publish_of_an_exact_release_asks_the_source_nothing(tmp_path, monkeypatch, store): + store.put_dataset(make_dataset()) + monkeypatch.setattr(LocalRegistry, "resolve", lambda self, dataset_id, version: pytest.fail("resolved")) + assert timenet.publish(HELLO, dest=tmp_path / "dest") == HELLO + assert LocalRegistry(tmp_path / "dest").exists(HELLO.dataset_id, "1.0.0") + + +def _store_reading(registry, release, source, record): + """Store ``release``, which keeps one Record of ``source`` and adds one Record of its own values.""" + dataset = TimeFDataset(metadata=_card(release, parents={ParentAlias("source"): source})) + with TimeFReader(open_local_version(registry.root / source.dataset_id / str(source.version))) as reader: + dataset.import_from(reader, alias=ParentAlias("source"), records=[record]) + own = make_dataset().records[0].signals[0] + signal = Signal.from_values(own.to_numpy(), spec=own.spec, name="own", time_axis=RegularAxis.from_rate_hz(16)) + dataset.add_record(record=Record(sources=(Source(name="Monitor", signals=(signal,)),))) + registry.put_dataset(dataset) + + +def test_publish_copies_only_the_payload_sources_that_the_release_lists(tmp_path, store): + # READER reads the own Record of SOURCE, which reads HELLO. Origin lacks HELLO, which READER does not need. + origin = LocalRegistry(tmp_path / "origin") + origin.put_dataset(make_dataset()) + source, reader = DatasetRef.parse("test/source@1.0.0"), DatasetRef.parse("test/reader@1.0.0") + _store_reading(origin, source, HELLO, make_dataset().records[0].id) + _store_reading(origin, reader, source, 2) + shutil.rmtree(origin.root / HELLO.dataset_id) + dest = LocalRegistry(tmp_path / "dest") + + assert timenet.publish(str(reader), dest=dest, source=origin) == reader + assert dest.exists(source.dataset_id, "1.0.0") + assert not dest.exists(HELLO.dataset_id, "1.0.0") + + +def test_publish_of_a_release_that_the_source_lacks_names_the_source(tmp_path): + source = LocalRegistry(tmp_path / "source") + for wanted in ("timenet/hello-world", "timenet/hello-world@1.0.0"): + with pytest.raises(TimeNetDatasetNotFoundError) as caught: + timenet.publish(wanted, dest=tmp_path / "dest", source=source) + assert type(caught.value) is TimeNetDatasetNotFoundError + assert str(caught.value) == f"{wanted} is not in local registry {source.root}." + assert not (tmp_path / "dest").exists() + + +def test_publish_of_a_listed_release_that_the_source_does_not_serve_raises_the_access_error(tmp_path): + card = _card( + DatasetRef.parse("org/gated@1.0.0"), access=Access.CREDENTIALED, access_url="https://physionet.example/dua" + ) + + class _Hosted(RemoteRegistry): + def get_manifest(self, dataset_id, version=None): + return SimpleNamespace(metadata=card, lineage=()) + + hosted = _Hosted("http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404))) + with pytest.raises(TimeNetAccessError) as caught: + timenet.publish("org/gated", dest=tmp_path / "dest", source=hosted) + assert str(caught.value) == ( + "org/gated@1.0.0 is credentialed. Get access at https://physionet.example/dua for org/gated@1.0.0." + ) + assert not (tmp_path / "dest").exists() + + +@pytest.mark.parametrize("keyword", ["dest", "source"]) +def test_publish_names_a_registry_that_cannot_be_opened(tmp_path, keyword): + with pytest.raises(TimeNetRegistryError) as caught: + if keyword == "dest": + timenet.publish("timenet/hello-world", dest="foo://x") + else: + timenet.publish("timenet/hello-world", dest=tmp_path / "dest", source="foo://x") + assert str(caught.value) == ( + f"{keyword} 'foo://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + ) + + +def test_import_timenet_loads_the_producer_on_first_use(): + code = "\n".join( + [ + "import sys, timenet", + "assert {'TimeNet', 'build', 'publish'} <= set(dir(timenet))", + "assert 'timenet.client' not in sys.modules", + "build = timenet.build", + "assert 'timenet.client' in sys.modules", + "assert build is sys.modules['timenet.producer'].build", + "try:", + " timenet.nothing", + "except AttributeError as error:", + " assert str(error) == \"module 'timenet' has no attribute 'nothing'\", error", + "else:", + " raise AssertionError('timenet.nothing exists')", + ] + ) + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/zensical.toml b/zensical.toml index 5e0b129e0..6342476d2 100644 --- a/zensical.toml +++ b/zensical.toml @@ -22,7 +22,7 @@ nav = [ { "Datasets" = [{ "Use datasets" = [{ "The client (SDK)" = "client.md" }, { "Registries" = "registry.md" }, { "Load into your stack" = "usage.md" }] }, { "Build datasets" = [{ "Connectors" = "connectors.md" }, { "Build & publish" = "build.md" }] }, { "Catalog" = [{ "Datasets" = "catalog/datasets.md" }, { "Benchmarks" = "catalog/benchmarks.md" }, { "Licensing" = "catalog/licensing.md" }] }] }, { "CLI" = [{ "Overview" = "cli/index.md" }, { "timenet (consumer)" = "cli/timenet.md" }, { "timenet-build (producer)" = "cli/build.md" }] }, # >>> api-nav: generated by scripts/gen_api_docs.py — do not edit >>> - { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.sources" = "api/timenet.sources.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/timenet_connectors.sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, + { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.producer" = "api/producer.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.sources" = "api/timenet.sources.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/timenet_connectors.sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, # <<< api-nav <<< ]