From 4bb2f6614ce7150a1beb4d237b579e2a3ad6637a Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Wed, 4 Nov 2026 20:04:04 +0100 Subject: [PATCH 1/3] feat!: add timenet.build() and publish() A build from Python went through timenet_connectors.build(). It took fetch_from and build_parents instead of a list of sources, and it wrote to the local registry only through glue. Publishing a release had no function of its own. Only the build CLI copied a release to a remote --out. The shell had no command to copy a release out of the local registry. The new module timenet.producer adds build() and publish(). The timenet package exports them, together with TimeNet, LocalRegistry and InstalledConnectors, and loads each on first use. The plan and its steps stay internal. build() builds a release into the local registry and returns its directory, like TimeNet.download(). Missing parents and payload sources come from sources, the same name as in TimeNet. Without sources, the build takes the whole default list, and the planner leaves the local registry out of the sources of its copies and builds. With open_access_only=True, build() refuses a plan that needs a release without open access, before any step runs. publish() takes a dataset ID or an exact DatasetRef, which needs no lookup in the source. It copies a complete release and its payload sources to dest. A published version cannot change, so other files in dest raise TimeNetVersionConflictError. The defaults of build() come from the environment at call time. A local_registry that is remote or read-only raises TimeFValidationError, which points to publish(). timenet publish DATASET --dest URL|PATH copies a release with publish(). The release comes from the local registry or from the registry of --source. An empty --dest is an error, not the current directory. timenet_connectors.build() and load() are gone, with their glue. The tests, the example and the API reference use the new names. BREAKING CHANGE: timenet_connectors.build() and timenet_connectors.load() are gone. Build with timenet.build() and read with TimeNet().load(). Part of TNET-157. --- examples/load_tsqa.py | 2 +- examples/test_mean_classifier.py | 6 +- .../src/timenet_connectors/__init__.py | 6 +- .../src/timenet_connectors/api.py | 120 ------- .../tests/test_build_plan.py | 76 ++--- .../tests/test_end_to_end.py | 17 +- ...t_package_api.py => test_timenet_build.py} | 24 +- packages/timenet/src/timenet/__init__.py | 57 +++- packages/timenet/src/timenet/cli/app.py | 51 +++ packages/timenet/src/timenet/client.py | 52 +-- .../src/timenet/coordinator/__init__.py | 3 +- packages/timenet/src/timenet/producer.py | 162 ++++++++++ packages/timenet/tests/test_cli.py | 94 +++++- packages/timenet/tests/test_producer.py | 302 ++++++++++++++++++ zensical.toml | 2 +- 15 files changed, 750 insertions(+), 224 deletions(-) delete mode 100644 packages/timenet-connectors/src/timenet_connectors/api.py rename packages/timenet-connectors/tests/{test_package_api.py => test_timenet_build.py} (73%) create mode 100644 packages/timenet/src/timenet/producer.py create mode 100644 packages/timenet/tests/test_producer.py diff --git a/examples/load_tsqa.py b/examples/load_tsqa.py index 9256ee49c..31e9a5763 100644 --- a/examples/load_tsqa.py +++ b/examples/load_tsqa.py @@ -11,7 +11,7 @@ from pathlib import Path import sys -from timenet.client import TimeNet +from timenet import TimeNet def main() -> None: diff --git a/examples/test_mean_classifier.py b/examples/test_mean_classifier.py index e99f382d8..b19e027ac 100644 --- a/examples/test_mean_classifier.py +++ b/examples/test_mean_classifier.py @@ -17,13 +17,13 @@ from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split -from timenet.client import TimeNet -import timenet_connectors +import timenet +from timenet import TimeNet # Build and load through the same explicit local registry. registry = Path(".timenet-registry") -timenet_connectors.build("timenet/test-mean", out=registry) +timenet.build("timenet/test-mean", local_registry=registry) dataset = TimeNet(sources=[registry]).load("timenet/test-mean") # A quick tour of what we loaded: identity, counts, specs, and a record preview. diff --git a/packages/timenet-connectors/src/timenet_connectors/__init__.py b/packages/timenet-connectors/src/timenet_connectors/__init__.py index b650362a8..42d33ab39 100644 --- a/packages/timenet-connectors/src/timenet_connectors/__init__.py +++ b/packages/timenet-connectors/src/timenet_connectors/__init__.py @@ -5,8 +5,6 @@ (see :mod:`timenet_connectors.discovery`). Reusable bases, for example for the HuggingFace Hub, live under ``timenet_connectors.bases``. The connector contract is :class:`timenet.connectors.BaseConnector`. -The module-level helpers :func:`build` and :func:`load` (from :mod:`timenet_connectors.api`) are the -producer-side shortcuts. Use them to build and consume a dataset from local code. +To build a dataset from Python, call ``timenet.build("org/name")``, and read it with +``TimeNet().load("org/name")``. """ - -from timenet_connectors.api import build as build, load as load diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py deleted file mode 100644 index dd295cbae..000000000 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Build and load datasets with installed connectors.""" - -from pathlib import Path -from typing import TYPE_CHECKING - - -if TYPE_CHECKING: - from timenet.dataset import TimeFDataset - from timenet.registry import BaseRegistry - - -def build( # ruff: ignore[too-many-arguments] - dataset_id: str, - *, - version: str | None = None, - out: str | Path | None = None, - force: bool = False, - keep_cache: bool = False, - fetch_from: "str | BaseRegistry | None" = None, - build_parents: bool = False, -) -> Path: - # ruff: ignore[docstring-extraneous-exception] (raised by the plan and its run, not directly here) - """Build a dataset release in a local registry. - - The connector runs in an environment built from its ``requirements.txt``. - Set ``TIMENET_ISOLATION=off`` to use the current interpreter. - - Args: - dataset_id: The ``org/name`` ID declared in the connector's card. - version: The required card version, or ``None`` to accept any version. - out: The output directory, or ``None`` to use ``TIMENET_LOCAL_REGISTRY``. - force: Rebuild an existing release instead of reusing it. - keep_cache: Keep raw downloads after a successful build. - fetch_from: A registry or registry URI to copy missing parent releases from. - build_parents: Build missing parent releases from their installed connectors. - - Returns: - The release's version directory. - - Raises: - TimeNetDatasetNotFoundError: If no installed connector builds ``dataset_id``. - TimeNetVersionUnavailableError: If the card version differs from ``version``. - TimeNetMissingParentError: If a required parent release is missing. - """ - from timenet.builders import BuildOptions, declared # ruff: ignore[import-outside-top-level] - from timenet.config import settings # ruff: ignore[import-outside-top-level] - from timenet.coordinator.build import plan_release_build, run_plan # ruff: ignore[import-outside-top-level] - from timenet.errors import TimeNetMissingParentError # ruff: ignore[import-outside-top-level] - from timenet.registry import LocalRegistry, open_registry # ruff: ignore[import-outside-top-level] - from timenet.registry._paths import release_dir # ruff: ignore[import-outside-top-level, import-private-name] - from timenet.sources import InstalledConnectors # ruff: ignore[import-outside-top-level] - - release = declared(dataset_id).release - # TODO(adrian): Remove temporary glue which maps a missing out onto the local registry of the default - # list until timenet.build() replaces this function. - root = Path(out).expanduser() if out is not None else settings().local_registry_dir - source = open_registry(fetch_from) if isinstance(fetch_from, str) else fetch_from - store = LocalRegistry(root) - try: - plan = plan_release_build( - f"{dataset_id}@{version}" if version else dataset_id, - store=store, - # TODO(adrian): Remove temporary glue which maps fetch_from and build_parents onto a - # list of sources until the producer API replaces this function. - parents_from=[ - *([source] if source is not None else []), - *([InstalledConnectors()] if build_parents else []), - ], - force=force, - ) - run_plan(plan, store, options=BuildOptions(keep_cache=keep_cache)) - except TimeNetMissingParentError as error: - error.hint = _missing_hint(fetch_from=fetch_from, build_parents=build_parents) - raise - return release_dir(root, release) - - -def _missing_hint(*, fetch_from: "str | BaseRegistry | None", build_parents: bool) -> str | None: - """Suggest options to obtain a missing release. - - Args: - fetch_from: The selected registry or URI, or ``None``. - build_parents: Whether parent builds are enabled. - - Returns: - A sentence naming unused options, or ``None`` if both are enabled. - """ - if fetch_from is None and not build_parents: - return "Copy it with fetch_from= or build it with build_parents=True." - if fetch_from is None: - return "Copy it with fetch_from=." - if not build_parents: - return "Build it with build_parents=True." - return None - - -def load( - dataset_id: str, - version: str | None = None, - *, - fetch_from: "str | BaseRegistry | None" = None, - build_parents: bool = False, -) -> "TimeFDataset": - """Build a dataset if needed and load its values into memory. - - The build writes to the default local registry. - - Args: - dataset_id: The dataset ID (``org/name``). - version: The version declared in the connector's card, or ``None`` for the latest. - fetch_from: A registry or URI to copy missing dependencies from. - build_parents: Build missing parent releases from their installed connectors. - - Returns: - The dataset with all stored values in memory. - """ - from timenet.client import TimeNet # ruff: ignore[import-outside-top-level] - - build(dataset_id, version=version, fetch_from=fetch_from, build_parents=build_parents) - return TimeNet().load(dataset_id, version) diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index f69c385aa..f9126b9d9 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -6,6 +6,7 @@ import pytest from typer.testing import CliRunner +import timenet from timenet import builders from timenet.connectors import BaseConnector from timenet.coordinator.build import plan_release_build, run_plan @@ -30,7 +31,6 @@ from timenet.types.reference_keys import ParentAlias, SourceKey from timenet.types.references import DatasetRef from timenet.values_backends import ValuesBackend -import timenet_connectors from timenet_connectors.builder import backend as backend_module, cli as cli_module, env as env_module from timenet_connectors.builder.backend import ConnectorBuilder from timenet_connectors.builder.cli import app @@ -535,45 +535,32 @@ def test_build_prepares_parents_before_the_isolated_child(tmp_path, monkeypatch) assert result.stdout.strip() == str(out / CHILD.dataset_id / str(CHILD.version)) -@pytest.mark.parametrize("obtain", ["fetch_from", "build_parents"]) -def test_load_obtains_a_missing_parent_before_it_builds_and_reads(tmp_path, monkeypatch, obtain): +@pytest.mark.parametrize("entry", ["producer", "coordinator"]) +def test_every_build_entry_point_prepares_parents(tmp_path, monkeypatch, child_builds, entry): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "out")) - calls = [] - monkeypatch.setattr(backend_module, "run_isolated_build", _prepared_builds(calls)) - origin = LocalRegistry(tmp_path / "origin") - parent = make_dataset() - parent._metadata = cards[PARENT.dataset_id] - origin.put_dataset(parent) - - with pytest.raises(TimeNetMissingParentError, match=r"^parent release test/parent@1\.0\.0 of test/child") as caught: - timenet_connectors.load(CHILD.dataset_id) - assert str(caught.value).endswith(". Copy it with fetch_from= or build it with build_parents=True.") - if obtain == "fetch_from": - dataset = timenet_connectors.load(CHILD.dataset_id, fetch_from=origin) - else: - dataset = timenet_connectors.load(CHILD.dataset_id, build_parents=True) + out = tmp_path / "out" - assert dataset.metadata.dataset_id == CHILD.dataset_id - # The child holds the Record it imported from the parent. - assert [record.id for record in dataset.records] == [make_dataset().records[0].id] - builds = [(CHILD.dataset_id, False)] - assert calls == (builds if obtain == "fetch_from" else [(PARENT.dataset_id, False), *builds]) + def build(origin=None): + parents_from = [] if origin is None else [origin] + if entry == "producer": + timenet.build(CHILD.dataset_id, local_registry=out, parents_from=parents_from) + else: + # The client builds through the coordinator. + store = LocalRegistry(out) + run_plan(plan_release_build(CHILD.dataset_id, store=store, parents_from=parents_from), store) + with pytest.raises(TimeNetMissingParentError, match=r"parent release test/parent@1\.0\.0 of test/child@1\.0\.0"): + build() + assert child_builds == [] -def test_load_copies_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch): - _install_cards(monkeypatch, parent={"parent": GRAND}) - out = LocalRegistry(tmp_path / "out") - _parent_without_its_source(out) + # A registry instance to fetch from, as the SDK layers accept one. origin = LocalRegistry(tmp_path / "origin") - origin.put_dataset(make_dataset()) - monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(out.root)) - - dataset = timenet_connectors.load(PARENT.dataset_id, fetch_from=origin) - - assert len(dataset.records) == 2 - # The build copies the payload source into the store, where the read finds it. - assert out.exists(GRAND.dataset_id, str(GRAND.version)) + parent = make_dataset() + parent._metadata = cards[PARENT.dataset_id] + origin.put_dataset(parent) + build(origin) + assert child_builds == [(CHILD.dataset_id, False)] + assert LocalRegistry(out).exists(PARENT.dataset_id, str(PARENT.version)) def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch): @@ -813,26 +800,17 @@ def test_the_requested_build_waits_for_a_payload_source_of_a_held_parent(tmp_pat ) -@pytest.mark.parametrize( - ("entry", "hint"), [("cli", "Build it with --build-parents."), ("api", "Copy it with fetch_from=.")] -) -def test_the_hint_names_only_the_option_that_the_build_did_not_use(tmp_path, monkeypatch, entry, hint): - # No registry holds the parent, and no installed connector builds it. The CLI build uses - # --fetch-from, and the API build uses build_parents=True. +def test_the_hint_names_only_the_option_that_the_build_did_not_use(tmp_path, monkeypatch): + # No registry holds the parent, and no installed connector builds it. The build uses --fetch-from. cards = _install_cards(monkeypatch, child={"parent": PARENT}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) out = tmp_path / "out" - if entry == "cli": - options = ["--fetch-from", str(tmp_path / "origin")] - error = runner.invoke(app, ["build", CHILD.dataset_id, "--out", str(out), *options]).exception - else: - with pytest.raises(TimeNetMissingParentError) as caught: - timenet_connectors.build(CHILD.dataset_id, out=out, build_parents=True) - error = caught.value + options = ["--fetch-from", str(tmp_path / "origin")] + error = runner.invoke(app, ["build", CHILD.dataset_id, "--out", str(out), *options]).exception assert isinstance(error, TimeNetMissingParentError) - assert error.hint == hint + assert error.hint == "Build it with --build-parents." def _count_reads(monkeypatch, registry) -> dict[str, int]: diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 853617a91..1febeb2d4 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -4,6 +4,7 @@ import pytest from typer.testing import CliRunner +import timenet from timenet.client import TimeNet from timenet.config import settings from timenet.errors import TimeFValidationError, TimeNetRegistryError @@ -11,7 +12,6 @@ from timenet.registry import LocalRegistry, RemoteRegistry from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal -from timenet_connectors import build, load from timenet_connectors.builder.cli import _resolve_target, app as build_app from timenet_connectors.datasets.timenet.hello_world import HelloWorldConnector from timenet_connectors.discovery import available, resolve @@ -63,15 +63,10 @@ def test_build_unknown_id_fails(tmp_path): assert result.exit_code != 0 -def test_build_rejects_version_mismatch(): - # The guard fires before the pipeline runs, so no registry is written. +def test_load_rejects_version_mismatch(tmp_path): + client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), InstalledConnectors()]) with pytest.raises(TimeFValidationError, match="builds version"): - build("timenet/hello-world", version="9.9.9") - - -def test_load_rejects_version_mismatch(): - with pytest.raises(TimeFValidationError, match="builds version"): - load("timenet/hello-world", "9.9.9") + client.load("timenet/hello-world", "9.9.9") # ---- where a bare `build` writes ----------------------------------------------------------------- @@ -115,10 +110,10 @@ def test_sdk_build_cleans_cache_but_keep_cache_retains(tmp_path, monkeypatch): monkeypatch.setenv("TIMENET_ISOLATION", "off") # the in-process path, the one that kept the cache cache = settings().downloads_dir / "timenet" / "hello-world" - build("timenet/hello-world", out=tmp_path / "r1") + timenet.build("timenet/hello-world", local_registry=tmp_path / "r1") assert not cache.exists() - build("timenet/hello-world", out=tmp_path / "r2", keep_cache=True) + timenet.build("timenet/hello-world", local_registry=tmp_path / "r2", keep_cache=True) assert cache.exists() diff --git a/packages/timenet-connectors/tests/test_package_api.py b/packages/timenet-connectors/tests/test_timenet_build.py similarity index 73% rename from packages/timenet-connectors/tests/test_package_api.py rename to packages/timenet-connectors/tests/test_timenet_build.py index 1e189abc2..318467fc2 100644 --- a/packages/timenet-connectors/tests/test_package_api.py +++ b/packages/timenet-connectors/tests/test_timenet_build.py @@ -1,14 +1,18 @@ import pytest +import timenet +from timenet.client import TimeNet from timenet.errors import TimeNetVersionUnavailableError from timenet.registry import LocalRegistry from timenet.testing import make_dataset from timenet.types import ClassificationTask from timenet.types.references import DatasetRef -import timenet_connectors from timenet_connectors.builder import backend as backend_module +HELLO = DatasetRef.parse("timenet/hello-world@1.0.0") + + def _unexpected_isolation(*args, **kwargs): raise AssertionError(f"run_isolated_build should not have run: {args} {kwargs}") @@ -17,16 +21,15 @@ def _unexpected_resolve(*args, **kwargs): raise AssertionError(f"resolve should not have run: {args} {kwargs}") -def test_build_then_load_round_trip(monkeypatch): - # timenet_connectors.build/load anchor on the store of the default list, so a build here loads back. +def test_build_then_load_round_trip(monkeypatch, tmp_path): # In-process, so the round trip stays in this interpreter instead of shelling out to uv. monkeypatch.setenv("TIMENET_ISOLATION", "off") monkeypatch.setattr(backend_module, "run_isolated_build", _unexpected_isolation) + out = tmp_path / "out" - version_dir = timenet_connectors.build("timenet/test-mean") - assert (version_dir / "manifest.json").exists() + assert timenet.build("timenet/test-mean", local_registry=out) == out / "timenet/test-mean/1.0.0" - dataset = timenet_connectors.load("timenet/test-mean") + dataset = TimeNet(sources=[str(out)]).load("timenet/test-mean") assert len(dataset.records) == 1000 x, y = dataset.to_features_and_targets(task=ClassificationTask) @@ -46,11 +49,10 @@ def fake_isolated(release, store_root, *, force=False, keep_cache=False, quiet=F monkeypatch.setattr(backend_module, "run_isolated_build", fake_isolated) - version_dir = timenet_connectors.build("timenet/hello-world", out=str(tmp_path), force=True) + timenet.build("timenet/hello-world", local_registry=tmp_path / "out", force=True) # The child is not quiet: it shows its downloads and shards. - assert calls == [(DatasetRef.parse("timenet/hello-world@1.0.0"), tmp_path, True, False, False)] - assert version_dir == tmp_path / "timenet" / "hello-world" / "1.0.0" + assert calls == [(HELLO, tmp_path / "out", True, False, False)] def test_build_rejects_a_version_the_connector_does_not_declare(monkeypatch, tmp_path): @@ -60,7 +62,7 @@ def test_build_rejects_a_version_the_connector_does_not_declare(monkeypatch, tmp with pytest.raises( TimeNetVersionUnavailableError, match=r"builds version 1\.0\.0, not the requested 9\.9\.9" ) as caught: - timenet_connectors.build("timenet/hello-world", version="9.9.9", out=tmp_path) + timenet.build("timenet/hello-world@9.9.9", local_registry=tmp_path / "out") assert str(caught.value.declared) == "1.0.0" - assert not tmp_path.joinpath("timenet").exists() + assert not (tmp_path / "out").exists() diff --git a/packages/timenet/src/timenet/__init__.py b/packages/timenet/src/timenet/__init__.py index 04a914252..7bcbfb566 100644 --- a/packages/timenet/src/timenet/__init__.py +++ b/packages/timenet/src/timenet/__init__.py @@ -1 +1,56 @@ -"""TimeNet: register, query, download, convert, and explore time series datasets.""" +"""Build, publish, and read time series datasets.""" + +import importlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from timenet.client import TimeNet + from timenet.producer import build, publish + from timenet.registry import LocalRegistry + from timenet.sources import InstalledConnectors + + +__all__ = [ + "InstalledConnectors", + "LocalRegistry", + "TimeNet", + "build", + "publish", +] + + +def __getattr__(name: str) -> object: + """Load an exported name on first access, so ``import timenet`` stays cheap. + + Args: + name: The attribute name. + + Returns: + The exported object. + + Raises: + AttributeError: If the package does not export the name. + """ + module = { + "InstalledConnectors": "timenet.sources", + "LocalRegistry": "timenet.registry", + "TimeNet": "timenet.client", + "build": "timenet.producer", + "publish": "timenet.producer", + }.get(name) + if module is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(importlib.import_module(module), name) + # Later lookups find the name without this function. + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """List package attributes, including exports that have not loaded yet. + + Returns: + The attribute names in sorted order. + """ + return sorted({*globals(), *__all__}) diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 8115391de..544181067 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -3,6 +3,7 @@ from collections.abc import Callable, Iterable from functools import partial from pathlib import Path +import shlex from typing import TypeVar from rich.console import Console @@ -15,7 +16,10 @@ from timenet.cli.ui import console as ui, progress_renderer from timenet.client import TimeNet, default_local_registry, default_sources from timenet.config import settings +from timenet.errors import TimeNetDatasetNotFoundError +from timenet.producer import publish as publish_release from timenet.progress import ProgressCallback, ProgressEvent, StepStarted +from timenet.registry import open_registry from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access from timenet.types.references import split_ref @@ -232,6 +236,53 @@ def _keep_steps(started: list[StepStarted], show: ProgressCallback | None, event show(event) +@app.command() +def publish( + dataset_id: str, + dest: str = typer.Option(..., "--dest", metavar="URL|PATH", help="The registry to copy to: a URL or a local path."), + source: str | None = typer.Option( + None, + "--source", + metavar="URL|PATH", + help="The registry to copy from: a URL or a local path (default: the local registry).", + ), + local_registry: str | None = LOCAL_REGISTRY, +) -> None: + """Copy a release and its payload sources to another registry. Prints the release to stdout. + + Without ``--source``, the release comes from the store. A published version cannot change, so + the command copies nothing if the destination holds the same files, and fails if it holds other + files. + \f + + Raises: + TimeNetDatasetNotFoundError: If the source lacks the release. Without ``--source``, the + message says how to put the release into the store. + """ # ruff: ignore[escape-sequence-in-docstring] - the form feed ends the help text of the command + store = default_local_registry(local_registry=local_registry) + origin = store if source is None else open_registry(source, label="--source") + try: + with progress_renderer() as progress: + release = publish_release(dataset_id, dest=dest, source=origin, progress=progress) + except TimeNetDatasetNotFoundError as error: + # The hint fits only a release that the local registry lacks. A subclass, such as a missing payload + # source, names its own registries. + if source is not None or type(error) is not TimeNetDatasetNotFoundError: + raise + bare_id, version = split_ref(dataset_id) + wanted = bare_id if version is None else f"{bare_id}@{version}" + # Repeat the path of --local-registry, so that each command of the hint uses it too. + path = shlex.quote(str(store.root)) + out, flag = ("", "") if local_registry is None else (f" --out {path}", f" --local-registry {path}") + hint = ( + f"Build it into the local registry with `timenet-build build {wanted}{out}`, " + f"or copy it there with `timenet download {wanted}{flag}`." + ) + raise TimeNetDatasetNotFoundError(f"{str(error).rstrip('.')}. {hint}") from error + ui.success(f"Published {release} to {dest}") + typer.echo(str(release)) + + local_app = typer.Typer(help="Inspect and clear the local files of TimeNet.", no_args_is_help=True) app.add_typer(local_app, name="local") cache_app = typer.Typer( diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 2d5a5a834..a54b4e788 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -124,38 +124,36 @@ def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None] return ref_id, ref_version if ref_version is not None else (version or None) -def _open_sources(sources: Sequence[ReleaseSource | str | Path] | None) -> tuple[ReleaseSource, ...]: - """Validate sources and open registry URLs and paths. +def as_release_sources(items: Sequence[ReleaseSource | str | Path], *, keyword: str) -> list[ReleaseSource]: + """Convert registry URLs and paths to release sources. + + Existing source objects keep their identity and position. A path opens a writable local registry. Args: - sources: A nonempty sequence of sources, registry URLs, or paths. + items: A sequence of sources, registry URLs, or local paths. + keyword: The argument name to show in errors. Returns: - A tuple of sources in the given order. + The sources, in order. An empty list stays empty. Raises: - TimeFValidationError: If sources are not a nonempty sequence of valid sources, registry URLs, or paths. + TimeFValidationError: If ``items`` is not a valid list. TimeNetRegistryError: If a registry URL or path is blank or cannot be opened. """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry - if isinstance(sources, (str, os.PathLike)): - shown = repr(str(sources)) - raise TimeFValidationError( - f"sources takes a list of sources. Pass sources=[{shown}], or " - f"sources=[{shown}, InstalledConnectors()] to build what the path lacks" - ) - if not isinstance(sources, Sequence): - raise TimeFValidationError(f"sources takes a list of sources. Pass sources=[{type(sources).__name__}(...)]") - if not sources: - raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") + if isinstance(items, (str, os.PathLike)): + raise TimeFValidationError(f"{keyword} takes a list of sources. Pass {keyword}=[{str(items)!r}].") + if not isinstance(items, Sequence): + raise TimeFValidationError(f"{keyword} takes a list of sources. Pass {keyword}=[{type(items).__name__}(...)].") opened: list[ReleaseSource] = [] - for index, item in enumerate(sources): + for index, item in enumerate(items): if isinstance(item, (str, os.PathLike)): - opened.append(open_registry(item if isinstance(item, str) else Path(item), label=f"sources[{index}]")) + path = item if isinstance(item, str) else Path(item) + opened.append(open_registry(path, label=f"{keyword}[{index}]")) elif isinstance(item, (BaseRegistry, InstalledConnectors)): opened.append(item) else: - raise TimeFValidationError(f"sources[{index}] is not a source, a registry URL or a path: {item!r}") - return tuple(opened) + raise TimeFValidationError(f"{keyword}[{index}] is not a source, a registry URL or a path: {item!r}") + return opened class TimeNet: @@ -172,9 +170,21 @@ def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) sources: Ordered registries, URLs, paths or connectors, or ``None`` for :func:`default_sources`. Raises: - TimeFValidationError: If sources are invalid, or have connectors but no writable local registry. + TimeFValidationError: If ``sources`` is not a valid list, or has connectors but no writable + local registry. """ - self._sources = tuple(default_sources()) if sources is None else _open_sources(sources) + if sources is None: + self._sources = tuple(default_sources()) + else: + if isinstance(sources, (str, os.PathLike)): + shown = repr(str(sources)) + raise TimeFValidationError( + f"sources takes a list of sources. Pass sources=[{shown}], or " + f"sources=[{shown}, InstalledConnectors()] to build what the path lacks" + ) + self._sources = tuple(as_release_sources(sources, keyword="sources")) + if not self._sources: + raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") self._store = next( (source for source in self._sources if isinstance(source, LocalRegistry) and not source.read_only), None ) diff --git a/packages/timenet/src/timenet/coordinator/__init__.py b/packages/timenet/src/timenet/coordinator/__init__.py index fce0d199d..8382eceef 100644 --- a/packages/timenet/src/timenet/coordinator/__init__.py +++ b/packages/timenet/src/timenet/coordinator/__init__.py @@ -1,4 +1,5 @@ """Plan and run dataset builds and reads. -Not part of the user API: use ``timenet-build`` to build datasets and ``TimeNet`` to read them. +Not part of the user API: to build a dataset, use ``timenet.build()`` or the ``timenet-build`` +command. To read one, use ``TimeNet``. """ diff --git a/packages/timenet/src/timenet/producer.py b/packages/timenet/src/timenet/producer.py new file mode 100644 index 000000000..0f3566692 --- /dev/null +++ b/packages/timenet/src/timenet/producer.py @@ -0,0 +1,162 @@ +"""Build releases into a local registry, and publish them to other registries. + +Each argument of :func:`build`, :func:`plan_build` and :func:`publish` that you leave out comes from +the environment at the time of the call, as in the default list of sources of ``TimeNet()``. So a +default build lands where a default client reads. +""" + +from collections.abc import Sequence +from pathlib import Path + +from timenet._registry_uri import registry_location +from timenet.builders import BuildOptions, declared +from timenet.client import as_release_sources, default_local_registry, default_sources +from timenet.coordinator.build import run_plan +from timenet.coordinator.plan import Plan, plan_steps +from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError +from timenet.progress import ProgressCallback +from timenet.registry import BaseRegistry, LocalRegistry, WritableRegistry, copy_release, open_registry +from timenet.registry._paths import release_dir +from timenet.sources import NotFound, NotServed, ReleaseSource, resolve_release +from timenet.types.references import DatasetRef, is_latest, split_ref + + +def build( # ruff: ignore[too-many-arguments] + dataset_id: str, + *, + local_registry: str | Path | LocalRegistry | None = None, + sources: Sequence[ReleaseSource | str | Path] | None = None, + force: bool = False, + open_access_only: bool = False, + keep_cache: bool = False, + progress: ProgressCallback | None = None, +) -> Path: + """Build a dataset release into a local registry with its installed connector. + + Missing parents and payload sources come from the first source in ``sources`` that provides them. + The build uses a separate environment with the connector's dependencies unless ``TIMENET_ISOLATION`` is ``off``. + + Args: + dataset_id: The dataset ID (``org/name``) or exact release (``org/name@1.2.0``). + local_registry: The writable local registry to build into: a ``LocalRegistry``, a local path, or a + ``file://`` URI, or ``None`` for ``TIMENET_LOCAL_REGISTRY``. + sources: Sources, registry URLs, or paths for missing releases, or ``None`` for + :func:`~timenet.client.default_sources` with ``local_registry`` first. + force: Build the release again, even if the local registry holds it. + open_access_only: Refuse a plan that needs a release without open access, before any step runs. + keep_cache: Keep the raw downloads in ``TIMENET_CACHE`` after the build. + progress: Receives the progress events of each copy and each build. + + Returns: + The release directory in the local registry. + + Raises: + TimeFValidationError: If ``local_registry`` is not a writable local registry, or ``sources`` is invalid. + TimeNetRegistryError: If a registry in ``sources`` fails to open. + TimeNetDatasetNotFoundError: If no installed connector builds the dataset. + TimeNetVersionUnavailableError: If the requested version differs from the connector's version. Nothing + runs then. + TimeNetMissingParentError: If no source provides a required parent or payload source. Nothing runs then. + TimeNetAccessError: If a source denies access to a parent and no later source provides it, or + ``open_access_only`` is set and the plan needs a release without open access. Nothing runs then. + TimeNetVersionConflictError: If a release to copy has other files than the same version in the local registry. + TimeNetFileError: If a stored release is damaged and no registry in ``sources`` provides it. + TimeNetBuildError: If a build fails. + """ # ruff: ignore[docstring-extraneous-exception] - raised by the coordinator and open_registry + target = _build_target(local_registry) + # The planner leaves the local registry out of the sources of its copies and builds. + parents = ( + default_sources(local_registry=target.root) + if sources is None + else as_release_sources(sources, keyword="sources") + ) + bare_id, version = split_ref(dataset_id) + # A connector builds only the version of its card, so "latest" asks for that version. + builder = declared(bare_id, None if is_latest(version) else version) + release, steps = plan_steps( + builder, target, parents_from=parents, force_build=force, open_access_only=open_access_only + ) + run_plan(Plan(tuple(steps)), target, options=BuildOptions(keep_cache=keep_cache, progress=progress)) + return release_dir(target.root, release) + + +def publish( + dataset_id: str | DatasetRef, + *, + dest: str | Path | WritableRegistry, + source: str | Path | BaseRegistry | None = None, + progress: ProgressCallback | None = None, +) -> DatasetRef: + """Copy a complete release and the payload sources that it needs to another registry. + + A published version cannot change, so the call copies nothing if ``dest`` holds the same files, + and raises an error if it holds other files. A local ``dest`` with a missing or damaged file gets + the release again. + + Args: + dataset_id: ``org/name`` for the latest version in ``source``, ``org/name@1.2.0`` for an exact release, or + a ``DatasetRef``, which needs no lookup in ``source``. + dest: The registry to copy to, or its URL or path. + source: The registry to copy from, or its URL or path. ``None`` means the local registry, + ``TIMENET_LOCAL_REGISTRY``. + progress: Receives the progress events of the copy. + + Returns: + The release that ``dest`` now holds. + + Raises: + TimeNetRegistryError: If ``dest`` or ``source`` cannot be opened as a registry, or ``dest`` is read-only. + TimeNetDatasetNotFoundError: If ``source`` does not hold the release. + TimeNetAccessError: If ``source`` lists the release but does not serve its files. Also if a + release of the copy is not open, and ``dest`` or ``source`` requires open access. + TimeNetVersionConflictError: If ``dest`` holds the release, or a payload source that it + needs, with other files. Nothing is copied then. + TimeNetMissingParentError: If a payload source is neither in ``source`` nor complete in + ``dest``. + """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry and copy_release + target = open_registry(dest, label="dest") if isinstance(dest, (str, Path)) else dest + if source is None: + origin = default_local_registry() + else: + origin = open_registry(source, label="source") if isinstance(source, (str, Path)) else source + if isinstance(dataset_id, DatasetRef): + # The caller names the exact release, so the copy starts at once. + copy_release(dataset_id, origin, target, progress=progress) + return dataset_id + bare_id, version = split_ref(dataset_id) + answer = resolve_release([origin], bare_id, version) + if isinstance(answer, NotServed): + raise answer.access_error(bare_id) + if isinstance(answer, NotFound): + wanted = bare_id if version is None else f"{bare_id}@{version}" + raise TimeNetDatasetNotFoundError(f"{wanted} is not in {origin}.") + release = DatasetRef(dataset_id=bare_id, version=answer.version) + copy_release(release, origin, target, progress=progress) + return release + + +def _build_target(local_registry: str | Path | LocalRegistry | None) -> LocalRegistry: + """Open the local registry that a build writes into. + + Args: + local_registry: A local registry, path, or ``file://`` URI, or ``None`` for the configured one. + + Returns: + The writable local registry. + + Raises: + TimeFValidationError: If ``local_registry`` is remote, read-only, or not a registry. + TimeNetRegistryError: If ``local_registry`` is blank or has an unsupported scheme. + """ # ruff: ignore[docstring-extraneous-exception] - raised by registry_location + if local_registry is None: + return default_local_registry() + if isinstance(local_registry, (str, Path)): + location = registry_location(local_registry, label="local_registry") + if isinstance(location, Path): + return LocalRegistry(location) + elif isinstance(local_registry, LocalRegistry) and not local_registry.read_only: + return local_registry + raise TimeFValidationError( + "local_registry takes a writable local registry, because a build writes into it. Publish the result to " + "another registry with timenet.publish()." + ) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index 2b32bbfed..a8920d3d4 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -1,4 +1,5 @@ import importlib +import shutil import subprocess import sys @@ -10,9 +11,15 @@ from timenet.cache import human_bytes, stored_releases from timenet.cli.app import _enum_list, app +from timenet.client import TimeNet from timenet.config import make_cache_folder from timenet.dataset import TimeFDataset -from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError +from timenet.errors import ( + TimeFValidationError, + TimeNetDatasetNotFoundError, + TimeNetMissingParentError, + TimeNetRegistryError, +) from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version from timenet.testing import make_dataset @@ -23,6 +30,7 @@ runner = CliRunner() +HELLO = DatasetRef.parse("timenet/hello-world@1.0.0") # Import the app submodule directly so we can patch its module-global ``app`` when testing main()'s # error handling. _cli_module = importlib.import_module("timenet.cli.app") @@ -248,6 +256,90 @@ def test_each_command_builds_the_list_from_its_flags(registry_root, tmp_path): assert result.stdout == f"{store / 'timenet/hello-world/1.0.0'}\n" +def test_publish_copies_a_release_from_the_store(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + dest = tmp_path / "dest" + result = runner.invoke( + app, ["publish", "timenet/hello-world", "--dest", str(dest), "--local-registry", str(store.root)] + ) + assert result.exit_code == 0, result.output + assert result.stdout.strip() == "timenet/hello-world@1.0.0" + assert LocalRegistry(dest).exists("timenet/hello-world", "1.0.0") + assert "📥 Copying 'timenet/hello-world'…" in result.stderr + assert "✅ Published timenet/hello-world@1.0.0 to" in result.stderr + + +def test_publish_rejects_a_source_that_cannot_be_opened(tmp_path): + result = runner.invoke(app, ["publish", "x/y", "--dest", str(tmp_path / "dest"), "--source", "foo://x"]) + assert result.exit_code == 1 + assert isinstance(result.exception, TimeNetRegistryError) + assert str(result.exception).startswith("--source 'foo://x' has an unsupported registry scheme.") + + +def test_publish_to_an_empty_dest_fails_and_writes_nothing_into_the_working_directory(tmp_path, monkeypatch): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + cwd = tmp_path / "cwd" + cwd.mkdir() + monkeypatch.chdir(cwd) + argv = ["publish", "timenet/hello-world", "--dest", "", "--local-registry", str(store.root)] + result = runner.invoke(app, argv) + assert isinstance(result.exception, TimeNetRegistryError) + assert str(result.exception).startswith("dest is blank") + assert list(cwd.iterdir()) == [] + + +def test_publish_needs_dest(tmp_path): + result = runner.invoke(app, ["publish", "timenet/hello-world"]) + assert result.exit_code == 2 + # Rich colours the usage error on a CI runner, which splits the option name with escape codes. + assert "--dest" in click.unstyle(result.output) + + +def test_publish_of_a_release_that_the_store_lacks_names_the_fix(tmp_path): + store, other, dest = tmp_path / "store", tmp_path / "other", str(tmp_path / "dest") + result = runner.invoke(app, ["publish", "timenet/hello-world", "--dest", dest, "--local-registry", str(store)]) + assert type(result.exception) is TimeNetDatasetNotFoundError + assert str(result.exception) == ( + f"timenet/hello-world is not in local registry {store}. " + f"Build it into the local registry with `timenet-build build timenet/hello-world --out {store}`, " + f"or copy it there with `timenet download timenet/hello-world --local-registry {store}`." + ) + # The hint keeps the version, and names no store without --local-registry. + result = runner.invoke(app, ["publish", "timenet/hello-world@1.0.0", "--dest", dest]) + assert str(result.exception).endswith( + "Build it into the local registry with `timenet-build build timenet/hello-world@1.0.0`, " + "or copy it there with `timenet download timenet/hello-world@1.0.0`." + ) + # A registry that --source names is not the store, so the error names only that registry. + result = runner.invoke(app, ["publish", "timenet/hello-world", "--dest", dest, "--source", str(other)]) + assert str(result.exception) == f"timenet/hello-world is not in local registry {other}." + + +def test_publish_adds_no_hint_to_a_payload_source_that_the_store_lacks(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + child = TimeFDataset( + metadata=make_dataset().metadata.model_copy( + update={"dataset_id": "test/child", "parents": {ParentAlias("hello"): HELLO}} + ) + ) + # The child reads the values of one Record of the parent, so the parent is its payload source. + with TimeNet(sources=[store]).open(str(HELLO)) as reader: + child.import_from(reader, alias=ParentAlias("hello"), records=[make_dataset().records[0].id]) + store.put_dataset(child) + shutil.rmtree(store.root / "timenet") + dest = tmp_path / "dest" + + result = runner.invoke(app, ["publish", "test/child", "--dest", str(dest), "--local-registry", str(store.root)]) + + # The error of the payload source names its own registries, and the hint for the release does not apply. + assert type(result.exception) is TimeNetMissingParentError + assert str(result.exception).startswith(f"payload source {HELLO} of test/child@1.0.0 is not in ") + assert "timenet-build" not in str(result.exception) + + @pytest.mark.parametrize(("flag", "variable"), [("--auto-build", "false"), ("--no-auto-build", "true")]) def test_the_auto_build_flag_beats_its_variable(monkeypatch, flag, variable): monkeypatch.setenv("TIMENET_AUTO_BUILD", variable) diff --git a/packages/timenet/tests/test_producer.py b/packages/timenet/tests/test_producer.py new file mode 100644 index 000000000..e8a804b83 --- /dev/null +++ b/packages/timenet/tests/test_producer.py @@ -0,0 +1,302 @@ +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import httpx +import pytest + +import timenet +from timenet.client import TimeNet +from timenet.dataset import Record, RegularAxis, Signal, Source, TimeFDataset +from timenet.errors import ( + TimeFValidationError, + TimeNetAccessError, + TimeNetDatasetNotFoundError, + TimeNetMissingParentError, + TimeNetRegistryError, +) +from timenet.reader import TimeFReader +from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version +from timenet.sources import InstalledConnectors +from timenet.testing import install_backends, make_dataset +from timenet.types import Access +from timenet.types.reference_keys import ParentAlias +from timenet.types.references import DatasetRef + + +PARENT = DatasetRef.parse("test/parent@1.0.0") +CHILD = DatasetRef.parse("test/child@1.0.0") +HELLO = DatasetRef.parse("timenet/hello-world@1.0.0") + + +def _card(reference, **update): + """Return the card of the demo dataset under another release.""" + return make_dataset().metadata.model_copy( + update={"dataset_id": reference.dataset_id, "dataset_version": reference.version, **update} + ) + + +def _demo(card): + """Return the demo dataset under ``card``.""" + dataset = make_dataset() + dataset._metadata = card + return dataset + + +class _Backend: + """A builder backend of another package. It builds the demo dataset under each card that it holds.""" + + def __init__(self, cards, calls): + self.cards = {card.dataset_id: card for card in cards} + self.calls = calls + + def dataset_ids(self): + return list(self.cards) + + def card(self, dataset_id): + return self.cards.get(dataset_id) + + def build(self, release, store, *, force, pinned_by, options): + self.calls.append((release, options)) + store.put_dataset(_demo(self.cards[release.dataset_id]), force=force, pinned_by=pinned_by or None) + + +@pytest.fixture +def cards(monkeypatch): + """Install a card for CHILD and a card for PARENT. CHILD declares PARENT. + + Returns: + The list of the calls of the backend: ``(release, options)`` for each build. + """ + calls = [] + backend = _Backend([_card(CHILD, parents={ParentAlias("parent"): PARENT}), _card(PARENT)], calls) + install_backends(monkeypatch, backend) + return calls + + +def _store_parent(registry): + """Put a PARENT release into a registry.""" + registry.put_dataset(_demo(_card(PARENT))) + + +def test_build_with_local_registry_and_sources_takes_both_only_from_them(tmp_path, monkeypatch, cards): + env_store, origin = tmp_path / "env-store", LocalRegistry(tmp_path / "origin") + _store_parent(origin) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(env_store)) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(origin.root)) + store = tmp_path / "store" + # The remote registry of the environment holds the parent, but the list of sources is empty. + with pytest.raises(TimeNetMissingParentError): + timenet.build(CHILD.dataset_id, local_registry=store, sources=[]) + assert not env_store.exists() + assert cards == [] + + +def test_build_without_sources_copies_a_parent_from_the_remote_registry(tmp_path, monkeypatch, cards): + origin = LocalRegistry(tmp_path / "origin") + _store_parent(origin) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(origin.root)) + store = LocalRegistry(tmp_path / "store") + assert timenet.build(CHILD.dataset_id, local_registry=store.root) == store.root / "test/child/1.0.0" + assert store.exists(PARENT.dataset_id, "1.0.0") + # The parent came as a copy, so only the child was built. + assert [release for release, _ in cards] == [CHILD] + + +def test_build_lands_in_the_store_that_the_environment_names(tmp_path, monkeypatch, cards): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "env-store")) + assert timenet.build(PARENT.dataset_id) == tmp_path / "env-store/test/parent/1.0.0" + assert LocalRegistry(tmp_path / "env-store").exists(PARENT.dataset_id, "1.0.0") + + +def test_auto_build_false_leaves_the_connectors_out_of_the_default_parents(tmp_path, monkeypatch, cards): + # The remote registry of the environment is empty, so only the installed connector builds the parent. + store = tmp_path / "store" + monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") + with pytest.raises(TimeNetMissingParentError): + timenet.build(CHILD.dataset_id, local_registry=store) + assert cards == [] + + monkeypatch.setenv("TIMENET_AUTO_BUILD", "true") + timenet.build(CHILD.dataset_id, local_registry=store) + assert [release for release, _ in cards] == [PARENT, CHILD] + + +def test_open_access_only_refuses_a_gated_parent_before_any_step(tmp_path, cards): + origin, store = LocalRegistry(tmp_path / "origin"), tmp_path / "store" + gated = _card(PARENT, access=Access.CREDENTIALED, access_url="https://physionet.example/dua") + origin.put_dataset(_demo(gated)) + + with pytest.raises(TimeNetAccessError, match="credentialed"): + timenet.build(CHILD.dataset_id, local_registry=store, sources=[origin], open_access_only=True) + assert cards == [] + assert not store.exists() + + +def test_timenet_exports_the_client_and_its_sources(): + assert (timenet.TimeNet, timenet.LocalRegistry, timenet.InstalledConnectors) == ( + TimeNet, + LocalRegistry, + InstalledConnectors, + ) + + +@pytest.mark.parametrize("kind", ["remote", "url", "read-only"]) +def test_build_rejects_a_local_registry_that_it_cannot_write(tmp_path, cards, kind): + target = { + "remote": RemoteRegistry("https://registry.example.test"), + "url": "s3://bucket/registry", + "read-only": LocalRegistry(tmp_path / "store", read_only=True), + }[kind] + with pytest.raises(TimeFValidationError) as caught: + timenet.build(CHILD.dataset_id, local_registry=target) # ty: ignore[invalid-argument-type] + assert str(caught.value) == ( + "local_registry takes a writable local registry, because a build writes into it. Publish the result to " + "another registry with timenet.publish()." + ) + assert cards == [] + + +def test_build_rejects_sources_given_as_a_string(tmp_path, cards): + with pytest.raises(TimeFValidationError) as caught: + timenet.build(CHILD.dataset_id, local_registry=tmp_path / "store", sources="timenet://") + assert str(caught.value) == "sources takes a list of sources. Pass sources=['timenet://']." + + +def test_sources_of_a_build_name_the_item_that_cannot_be_opened(tmp_path, cards): + with pytest.raises(TimeNetRegistryError) as caught: + timenet.build(CHILD.dataset_id, local_registry=tmp_path / "store", sources=["timenet://", "foo://x"]) + assert str(caught.value) == ( + "sources[1] 'foo://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + ) + + +@pytest.fixture +def store(tmp_path, monkeypatch): + """Name the store with the environment, and return it empty.""" + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) + return LocalRegistry(tmp_path / "store") + + +def test_publish_copies_the_latest_release_of_the_store(tmp_path, store): + store.put_dataset(_demo(_card(DatasetRef.parse("timenet/hello-world@0.9.0")))) + store.put_dataset(make_dataset()) + dest = tmp_path / "dest" + + assert timenet.publish("timenet/hello-world", dest=dest) == DatasetRef.parse("timenet/hello-world@1.0.0") + assert LocalRegistry(dest).exists("timenet/hello-world", "1.0.0") + assert not LocalRegistry(dest).exists("timenet/hello-world", "0.9.0") + + +def test_publish_takes_a_source(tmp_path, store): + origin = LocalRegistry(tmp_path / "origin") + origin.put_dataset(make_dataset()) + dest = LocalRegistry(tmp_path / "dest") + events = [] + + assert ( + timenet.publish("timenet/hello-world@1.0.0", dest=dest, source=str(origin.root), progress=events.append) + == HELLO + ) + assert dest.exists(HELLO.dataset_id, "1.0.0") + assert not store.root.exists() + assert [type(event).__name__ for event in events if type(event).__name__ != "BytesMoved"] == [ + "StepStarted", + "StepFinished", + ] + + +def test_publish_of_an_exact_release_asks_the_source_nothing(tmp_path, monkeypatch, store): + store.put_dataset(make_dataset()) + monkeypatch.setattr(LocalRegistry, "resolve", lambda self, dataset_id, version: pytest.fail("resolved")) + assert timenet.publish(HELLO, dest=tmp_path / "dest") == HELLO + assert LocalRegistry(tmp_path / "dest").exists(HELLO.dataset_id, "1.0.0") + + +def _store_reading(registry, release, source, record): + """Store ``release``, which keeps one Record of ``source`` and adds one Record of its own values.""" + dataset = TimeFDataset(metadata=_card(release, parents={ParentAlias("source"): source})) + with TimeFReader(open_local_version(registry.root / source.dataset_id / str(source.version))) as reader: + dataset.import_from(reader, alias=ParentAlias("source"), records=[record]) + own = make_dataset().records[0].signals[0] + signal = Signal.from_values(own.to_numpy(), spec=own.spec, name="own", time_axis=RegularAxis.from_rate_hz(16)) + dataset.add_record(record=Record(sources=(Source(name="Monitor", signals=(signal,)),))) + registry.put_dataset(dataset) + + +def test_publish_copies_only_the_payload_sources_that_the_release_lists(tmp_path, store): + # READER reads the own Record of SOURCE, which reads HELLO. Origin lacks HELLO, which READER does not need. + origin = LocalRegistry(tmp_path / "origin") + origin.put_dataset(make_dataset()) + source, reader = DatasetRef.parse("test/source@1.0.0"), DatasetRef.parse("test/reader@1.0.0") + _store_reading(origin, source, HELLO, make_dataset().records[0].id) + _store_reading(origin, reader, source, 2) + shutil.rmtree(origin.root / HELLO.dataset_id) + dest = LocalRegistry(tmp_path / "dest") + + assert timenet.publish(str(reader), dest=dest, source=origin) == reader + assert dest.exists(source.dataset_id, "1.0.0") + assert not dest.exists(HELLO.dataset_id, "1.0.0") + + +def test_publish_of_a_release_that_the_source_lacks_names_the_source(tmp_path): + source = LocalRegistry(tmp_path / "source") + for wanted in ("timenet/hello-world", "timenet/hello-world@1.0.0"): + with pytest.raises(TimeNetDatasetNotFoundError) as caught: + timenet.publish(wanted, dest=tmp_path / "dest", source=source) + assert type(caught.value) is TimeNetDatasetNotFoundError + assert str(caught.value) == f"{wanted} is not in local registry {source.root}." + assert not (tmp_path / "dest").exists() + + +def test_publish_of_a_listed_release_that_the_source_does_not_serve_raises_the_access_error(tmp_path): + card = _card( + DatasetRef.parse("org/gated@1.0.0"), access=Access.CREDENTIALED, access_url="https://physionet.example/dua" + ) + + class _Hosted(RemoteRegistry): + def get_manifest(self, dataset_id, version=None): + return SimpleNamespace(metadata=card, lineage=()) + + hosted = _Hosted("http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404))) + with pytest.raises(TimeNetAccessError) as caught: + timenet.publish("org/gated", dest=tmp_path / "dest", source=hosted) + assert str(caught.value) == ( + "org/gated@1.0.0 is credentialed. Get access at https://physionet.example/dua for org/gated@1.0.0." + ) + assert not (tmp_path / "dest").exists() + + +@pytest.mark.parametrize("keyword", ["dest", "source"]) +def test_publish_names_a_registry_that_cannot_be_opened(tmp_path, keyword): + with pytest.raises(TimeNetRegistryError) as caught: + if keyword == "dest": + timenet.publish("timenet/hello-world", dest="foo://x") + else: + timenet.publish("timenet/hello-world", dest=tmp_path / "dest", source="foo://x") + assert str(caught.value) == ( + f"{keyword} 'foo://x' has an unsupported registry scheme. " + "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + ) + + +def test_import_timenet_loads_the_producer_on_first_use(): + code = "\n".join( + [ + "import sys, timenet", + "assert {'TimeNet', 'build', 'publish'} <= set(dir(timenet))", + "assert 'timenet.client' not in sys.modules", + "build = timenet.build", + "assert 'timenet.client' in sys.modules", + "assert build is sys.modules['timenet.producer'].build", + "try:", + " timenet.nothing", + "except AttributeError as error:", + " assert str(error) == \"module 'timenet' has no attribute 'nothing'\", error", + "else:", + " raise AssertionError('timenet.nothing exists')", + ] + ) + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/zensical.toml b/zensical.toml index 5e0b129e0..6342476d2 100644 --- a/zensical.toml +++ b/zensical.toml @@ -22,7 +22,7 @@ nav = [ { "Datasets" = [{ "Use datasets" = [{ "The client (SDK)" = "client.md" }, { "Registries" = "registry.md" }, { "Load into your stack" = "usage.md" }] }, { "Build datasets" = [{ "Connectors" = "connectors.md" }, { "Build & publish" = "build.md" }] }, { "Catalog" = [{ "Datasets" = "catalog/datasets.md" }, { "Benchmarks" = "catalog/benchmarks.md" }, { "Licensing" = "catalog/licensing.md" }] }] }, { "CLI" = [{ "Overview" = "cli/index.md" }, { "timenet (consumer)" = "cli/timenet.md" }, { "timenet-build (producer)" = "cli/build.md" }] }, # >>> api-nav: generated by scripts/gen_api_docs.py — do not edit >>> - { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.sources" = "api/timenet.sources.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.api" = "api/api.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/timenet_connectors.sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, + { "API Reference" = [{ "Overview" = "api/index.md" }, { "timenet.builders" = "api/builders.md" }, { "timenet.client" = "api/client.md" }, { "timenet.config" = "api/config.md" }, { "timenet.connectors" = "api/connectors.md" }, { "timenet.dataset" = "api/dataset.md" }, { "timenet.engine" = "api/engine.md" }, { "timenet.errors" = "api/errors.md" }, { "timenet.format" = "api/format.md" }, { "timenet.json" = "api/json.md" }, { "timenet.manifest" = "api/manifest.md" }, { "timenet.producer" = "api/producer.md" }, { "timenet.progress" = "api/progress.md" }, { "timenet.provenance" = "api/provenance.md" }, { "timenet.reader" = "api/reader.md" }, { "timenet.registry" = "api/registry.md" }, { "timenet.sources" = "api/timenet.sources.md" }, { "timenet.storage" = "api/storage.md" }, { "timenet.types" = "api/types.md" }, { "timenet.values_backends" = "api/values_backends.md" }, { "timenet.writer" = "api/writer.md" }, { "timenet_connectors.bases" = "api/bases.md" }, { "timenet_connectors.discovery" = "api/discovery.md" }, { "timenet_connectors.download" = "api/download.md" }, { "timenet_connectors.sources" = "api/timenet_connectors.sources.md" }, { "timenet_connectors.splitting" = "api/splitting.md" }, { "timenet_connectors.time_axes" = "api/time_axes.md" }] }, # <<< api-nav <<< ] From 1e55e2af54d59706b82775226d88dadea6dce8a4 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 00:31:21 +0100 Subject: [PATCH 2/3] docs: shorten the docstrings of the producer API Shorter docstrings for the code that this change adds or changes. Part of TNET-157. --- .../src/timenet_connectors/builder/cli.py | 6 +-- packages/timenet/src/timenet/cli/app.py | 10 ++-- packages/timenet/src/timenet/client.py | 10 ++-- .../src/timenet/coordinator/__init__.py | 4 +- packages/timenet/src/timenet/producer.py | 46 ++++++++----------- 5 files changed, 33 insertions(+), 43 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index dd79fad25..dc39a036a 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -206,13 +206,13 @@ def _build_to( # ruff: ignore[too-many-arguments] ) -> str: """Build a release for a local or remote output. - Remote outputs receive a copy of a build in the local registry. If the output already holds the release and - ``force`` is not set, nothing runs and a status line says so. + Remote registries receive a copy from the local registry. + The build skips an existing release unless ``force`` is set. Args: dataset_id: The dataset ID. target: The local output directory or remote registry. - fetch_from: The registry to copy missing parents from, or ``None`` to use the remote output if present. + fetch_from: The registry for missing parents, or ``None`` to use the remote output if present. build_parents: Build missing parents with installed connectors. force: Rebuild an existing local release. options: Options for each build. diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 544181067..1267d8feb 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -248,16 +248,14 @@ def publish( ), local_registry: str | None = LOCAL_REGISTRY, ) -> None: - """Copy a release and its payload sources to another registry. Prints the release to stdout. + """Copy a release and its payload sources to a registry, and print the release to stdout. - Without ``--source``, the release comes from the store. A published version cannot change, so - the command copies nothing if the destination holds the same files, and fails if it holds other - files. + The source defaults to the local registry. + The copy skips identical releases and rejects releases with conflicting files. \f Raises: - TimeNetDatasetNotFoundError: If the source lacks the release. Without ``--source``, the - message says how to put the release into the store. + TimeNetDatasetNotFoundError: If the source lacks the release. """ # ruff: ignore[escape-sequence-in-docstring] - the form feed ends the help text of the command store = default_local_registry(local_registry=local_registry) origin = store if source is None else open_registry(source, label="--source") diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index a54b4e788..4b2ead3c7 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -134,7 +134,7 @@ def as_release_sources(items: Sequence[ReleaseSource | str | Path], *, keyword: keyword: The argument name to show in errors. Returns: - The sources, in order. An empty list stays empty. + The sources in their original order, or an empty list for empty input. Raises: TimeFValidationError: If ``items`` is not a valid list. @@ -160,18 +160,18 @@ class TimeNet: """Browse, download, and read TimeNet datasets.""" def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) -> None: - """Create a client with an ordered list of dataset sources. + """Create a client that searches dataset sources in order. Sources accept registries, registry URLs or paths, and ``InstalledConnectors()``. A path opens a writable ``LocalRegistry``. Builds land in the first ``LocalRegistry`` without ``read_only=True``, which ``InstalledConnectors()`` requires. Args: - sources: Ordered registries, URLs, paths or connectors, or ``None`` for :func:`default_sources`. + sources: Ordered sources, registry URLs or paths, or ``None`` for :func:`default_sources`. Raises: - TimeFValidationError: If ``sources`` is not a valid list, or has connectors but no writable - local registry. + TimeFValidationError: If ``sources`` is not a non-empty list of valid sources, or has installed + connectors but no writable local registry. """ if sources is None: self._sources = tuple(default_sources()) diff --git a/packages/timenet/src/timenet/coordinator/__init__.py b/packages/timenet/src/timenet/coordinator/__init__.py index 8382eceef..5dd106f24 100644 --- a/packages/timenet/src/timenet/coordinator/__init__.py +++ b/packages/timenet/src/timenet/coordinator/__init__.py @@ -1,5 +1,5 @@ """Plan and run dataset builds and reads. -Not part of the user API: to build a dataset, use ``timenet.build()`` or the ``timenet-build`` -command. To read one, use ``TimeNet``. +Not part of the user API: use ``timenet.build()`` or ``timenet-build`` to build datasets and +``TimeNet`` to read them. """ diff --git a/packages/timenet/src/timenet/producer.py b/packages/timenet/src/timenet/producer.py index 0f3566692..d32caf49d 100644 --- a/packages/timenet/src/timenet/producer.py +++ b/packages/timenet/src/timenet/producer.py @@ -1,8 +1,7 @@ -"""Build releases into a local registry, and publish them to other registries. +"""Build dataset releases in a local registry and publish them to other registries. -Each argument of :func:`build`, :func:`plan_build` and :func:`publish` that you leave out comes from -the environment at the time of the call, as in the default list of sources of ``TimeNet()``. So a -default build lands where a default client reads. +The default local and remote registries come from ``TIMENET_LOCAL_REGISTRY`` and ``TIMENET_REMOTE_REGISTRY``. +``TIMENET_AUTO_BUILD`` controls whether installed connectors supply missing parents by default. """ from collections.abc import Sequence @@ -43,9 +42,9 @@ def build( # ruff: ignore[too-many-arguments] sources: Sources, registry URLs, or paths for missing releases, or ``None`` for :func:`~timenet.client.default_sources` with ``local_registry`` first. force: Build the release again, even if the local registry holds it. - open_access_only: Refuse a plan that needs a release without open access, before any step runs. + open_access_only: Require open access for every release before running the plan. keep_cache: Keep the raw downloads in ``TIMENET_CACHE`` after the build. - progress: Receives the progress events of each copy and each build. + progress: A callback for copy and build progress events. Returns: The release directory in the local registry. @@ -54,11 +53,9 @@ def build( # ruff: ignore[too-many-arguments] TimeFValidationError: If ``local_registry`` is not a writable local registry, or ``sources`` is invalid. TimeNetRegistryError: If a registry in ``sources`` fails to open. TimeNetDatasetNotFoundError: If no installed connector builds the dataset. - TimeNetVersionUnavailableError: If the requested version differs from the connector's version. Nothing - runs then. - TimeNetMissingParentError: If no source provides a required parent or payload source. Nothing runs then. - TimeNetAccessError: If a source denies access to a parent and no later source provides it, or - ``open_access_only`` is set and the plan needs a release without open access. Nothing runs then. + TimeNetVersionUnavailableError: If the requested version differs from the connector's version. + TimeNetMissingParentError: If no source provides a required parent or payload source. + TimeNetAccessError: If no source grants access to a parent or a release violates ``open_access_only``. TimeNetVersionConflictError: If a release to copy has other files than the same version in the local registry. TimeNetFileError: If a stored release is damaged and no registry in ``sources`` provides it. TimeNetBuildError: If a build fails. @@ -87,32 +84,27 @@ def publish( source: str | Path | BaseRegistry | None = None, progress: ProgressCallback | None = None, ) -> DatasetRef: - """Copy a complete release and the payload sources that it needs to another registry. + """Copy a release and its required payload sources to another registry. - A published version cannot change, so the call copies nothing if ``dest`` holds the same files, - and raises an error if it holds other files. A local ``dest`` with a missing or damaged file gets - the release again. + The copy skips identical releases and rejects conflicting files before writing to ``dest``. + It replaces missing or damaged files at a local destination. Args: dataset_id: ``org/name`` for the latest version in ``source``, ``org/name@1.2.0`` for an exact release, or a ``DatasetRef``, which needs no lookup in ``source``. dest: The registry to copy to, or its URL or path. - source: The registry to copy from, or its URL or path. ``None`` means the local registry, - ``TIMENET_LOCAL_REGISTRY``. - progress: Receives the progress events of the copy. + source: The source registry, URL, or path, or ``None`` for ``TIMENET_LOCAL_REGISTRY``. + progress: A callback for copy progress events. Returns: - The release that ``dest`` now holds. + The ID and version of the release in ``dest``. Raises: - TimeNetRegistryError: If ``dest`` or ``source`` cannot be opened as a registry, or ``dest`` is read-only. - TimeNetDatasetNotFoundError: If ``source`` does not hold the release. - TimeNetAccessError: If ``source`` lists the release but does not serve its files. Also if a - release of the copy is not open, and ``dest`` or ``source`` requires open access. - TimeNetVersionConflictError: If ``dest`` holds the release, or a payload source that it - needs, with other files. Nothing is copied then. - TimeNetMissingParentError: If a payload source is neither in ``source`` nor complete in - ``dest``. + TimeNetRegistryError: If ``dest`` or ``source`` fails to open as a registry, or ``dest`` is read-only. + TimeNetDatasetNotFoundError: If ``source`` lacks the release. + TimeNetAccessError: If a registry withholds files or requires open access that a copied release lacks. + TimeNetVersionConflictError: If ``dest`` holds different files for the release or a required payload source. + TimeNetMissingParentError: If a required payload source is missing from ``source`` and incomplete in ``dest``. """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry and copy_release target = open_registry(dest, label="dest") if isinstance(dest, (str, Path)) else dest if source is None: From f2b6dcf40f1e4e16bc80ff59d51834d74826d0b9 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 10:56:25 +0100 Subject: [PATCH 3/3] test: trim the tests of the producer API Drop tests that cannot fail or repeat other tests, and sharpen the rest. Part of TNET-157. --- .../tests/test_build_plan.py | 16 ++++++--------- .../tests/test_end_to_end.py | 20 +------------------ 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index f9126b9d9..9e7f8918b 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -6,7 +6,6 @@ import pytest from typer.testing import CliRunner -import timenet from timenet import builders from timenet.connectors import BaseConnector from timenet.coordinator.build import plan_release_build, run_plan @@ -535,19 +534,16 @@ def test_build_prepares_parents_before_the_isolated_child(tmp_path, monkeypatch) assert result.stdout.strip() == str(out / CHILD.dataset_id / str(CHILD.version)) -@pytest.mark.parametrize("entry", ["producer", "coordinator"]) -def test_every_build_entry_point_prepares_parents(tmp_path, monkeypatch, child_builds, entry): +def test_every_build_entry_point_prepares_parents(tmp_path, monkeypatch, child_builds): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) out = tmp_path / "out" def build(origin=None): - parents_from = [] if origin is None else [origin] - if entry == "producer": - timenet.build(CHILD.dataset_id, local_registry=out, parents_from=parents_from) - else: - # The client builds through the coordinator. - store = LocalRegistry(out) - run_plan(plan_release_build(CHILD.dataset_id, store=store, parents_from=parents_from), store) + # The client and timenet.build() build through the coordinator. + store = LocalRegistry(out) + run_plan( + plan_release_build(CHILD.dataset_id, store=store, parents_from=[] if origin is None else [origin]), store + ) with pytest.raises(TimeNetMissingParentError, match=r"parent release test/parent@1\.0\.0 of test/child@1\.0\.0"): build() diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 1febeb2d4..51c8e38d5 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -4,10 +4,9 @@ import pytest from typer.testing import CliRunner -import timenet from timenet.client import TimeNet from timenet.config import settings -from timenet.errors import TimeFValidationError, TimeNetRegistryError +from timenet.errors import TimeNetRegistryError from timenet.provenance import build_env from timenet.registry import LocalRegistry, RemoteRegistry from timenet.sources import InstalledConnectors @@ -63,12 +62,6 @@ def test_build_unknown_id_fails(tmp_path): assert result.exit_code != 0 -def test_load_rejects_version_mismatch(tmp_path): - client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), InstalledConnectors()]) - with pytest.raises(TimeFValidationError, match="builds version"): - client.load("timenet/hello-world", "9.9.9") - - # ---- where a bare `build` writes ----------------------------------------------------------------- @@ -106,17 +99,6 @@ def test_resolve_target_rejects_an_empty_out_instead_of_the_current_directory(): _resolve_target("") -def test_sdk_build_cleans_cache_but_keep_cache_retains(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_ISOLATION", "off") # the in-process path, the one that kept the cache - cache = settings().downloads_dir / "timenet" / "hello-world" - - timenet.build("timenet/hello-world", local_registry=tmp_path / "r1") - assert not cache.exists() - - timenet.build("timenet/hello-world", local_registry=tmp_path / "r2", keep_cache=True) - assert cache.exists() - - def test_build_cleans_cache_but_keep_flag_retains(tmp_path): cache = settings().downloads_dir / "timenet" / "hello-world"