diff --git a/conftest.py b/conftest.py index 434f8cf2f..6802ea210 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,4 @@ -"""Give each test an empty TimeNet home and no other ``TIMENET_*`` setting.""" +"""Give each test an empty TimeNet store, registry and cache.""" import os @@ -9,11 +9,14 @@ @pytest.fixture(autouse=True) def _timenet_state(tmp_path_factory, monkeypatch): - """Set an empty temporary TimeNet home and clear other ``TIMENET_*`` variables. + """Set up an empty store, registry and cache for each test. - Ignore settings from ``.env`` files. + Clear existing ``TIMENET_*`` variables and ignore the ``.env`` file. """ for name in [name for name in os.environ if name.startswith("TIMENET_")]: monkeypatch.delenv(name) monkeypatch.setitem(TimeNetSettings.model_config, "env_file", None) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path_factory.mktemp("timenet-home"))) + root = tmp_path_factory.mktemp("timenet-state") + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(root / "store")) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(root / "remote")) + monkeypatch.setenv("TIMENET_CACHE", str(root / "cache")) diff --git a/packages/timenet-connectors/src/timenet_connectors/api.py b/packages/timenet-connectors/src/timenet_connectors/api.py index b45c92bc6..dd295cbae 100644 --- a/packages/timenet-connectors/src/timenet_connectors/api.py +++ b/packages/timenet-connectors/src/timenet_connectors/api.py @@ -1,9 +1,4 @@ -"""Producer-side shortcuts to build and load a dataset from local code. - -:func:`build` runs a connector through the engine into the shared local registry. :func:`load` runs -build and reads the result back. The writer and reader stacks are heavy, so they load lazily. This -keeps the import of :mod:`timenet_connectors` cheap. -""" +"""Build and load datasets with installed connectors.""" from pathlib import Path from typing import TYPE_CHECKING @@ -27,39 +22,38 @@ def build( # ruff: ignore[too-many-arguments] # ruff: ignore[docstring-extraneous-exception] (raised by the plan and its run, not directly here) """Build a dataset release in a local registry. - The build reuses an existing release unless ``force`` is set. The connector runs in an environment - from its ``requirements.txt``, or in this interpreter if ``TIMENET_ISOLATION=off``. + The connector runs in an environment built from its ``requirements.txt``. + Set ``TIMENET_ISOLATION=off`` to use the current interpreter. Args: dataset_id: The ``org/name`` ID declared in the connector's card. version: The required card version, or ``None`` to accept any version. - out: The local registry directory, or ``None`` to use the default local registry. - force: Rebuild an existing release. - keep_cache: Keep the raw download cache after a successful build. - fetch_from: The registry or URI to copy missing parents from before trying builds. - build_parents: Build missing parents with installed connectors. + out: The output directory, or ``None`` to use ``TIMENET_LOCAL_REGISTRY``. + force: Rebuild an existing release instead of reusing it. + keep_cache: Keep raw downloads after a successful build. + fetch_from: A registry or registry URI to copy missing parent releases from. + build_parents: Build missing parent releases from their installed connectors. Returns: - The version directory of the release. + The release's version directory. Raises: TimeNetDatasetNotFoundError: If no installed connector builds ``dataset_id``. - TimeNetVersionUnavailableError: If the card's version differs from ``version``. - TimeNetMissingParentError: If a required release cannot be copied or built. + TimeNetVersionUnavailableError: If the card version differs from ``version``. + TimeNetMissingParentError: If a required parent release is missing. """ from timenet.builders import BuildOptions, declared # ruff: ignore[import-outside-top-level] + from timenet.config import settings # ruff: ignore[import-outside-top-level] from timenet.coordinator.build import plan_release_build, run_plan # ruff: ignore[import-outside-top-level] from timenet.errors import TimeNetMissingParentError # ruff: ignore[import-outside-top-level] - from timenet.registry import ( # ruff: ignore[import-outside-top-level] - LocalRegistry, - default_registry_path, - open_registry, - ) + from timenet.registry import LocalRegistry, open_registry # ruff: ignore[import-outside-top-level] from timenet.registry._paths import release_dir # ruff: ignore[import-outside-top-level, import-private-name] from timenet.sources import InstalledConnectors # ruff: ignore[import-outside-top-level] release = declared(dataset_id).release - root = Path(out).expanduser() if out is not None else default_registry_path() + # TODO(adrian): Remove temporary glue which maps a missing out onto the local registry of the default + # list until timenet.build() replaces this function. + root = Path(out).expanduser() if out is not None else settings().local_registry_dir source = open_registry(fetch_from) if isinstance(fetch_from, str) else fetch_from store = LocalRegistry(root) try: diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index 3df7a9670..dd79fad25 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -48,23 +48,25 @@ def _root(quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress stat def _resolve_target(out: str | None) -> WritableRegistry | Path: - """Resolve the registry or local directory for build output. - - It tries ``out``, then ``$TIMENET_REGISTRY``, then the default local registry. + """Resolve a build output to a local directory or writable remote registry. Args: - out: The output URL or path, or ``None`` to use the configured default. + out: The output URL or path, or ``None`` to use ``TIMENET_LOCAL_REGISTRY``. Returns: - A registry for a remote URL, or a ``Path`` for a local directory. - """ - target = out if out is not None else settings().registry - if target is None: - return settings().registry_path + A local directory ``Path`` or a remote ``WritableRegistry``. + + Raises: + TimeNetRegistryError: If ``out`` is blank or cannot be opened as a registry. + """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry + # TODO(adrian): Remove temporary glue which maps a missing --out onto the local registry of the + # default list until the build CLI takes the flags of the defaults. + if out is None: + return settings().local_registry_dir try: - return local_registry_path(target) + return local_registry_path(out, label="--out") except TimeNetRegistryError: - return open_registry(target) + return open_registry(out, label="--out") @app.command() @@ -75,7 +77,7 @@ def build( None, "--out", help="Output registry: a local directory or a timenet:// / http(s):// / s3:// URL " - "(default: $TIMENET_REGISTRY, else the local registry).", + "(default: the local registry, $TIMENET_LOCAL_REGISTRY).", ), force: bool = typer.Option( False, "--force", "-f", help="Build the release again, even if the local registry holds it." @@ -202,20 +204,21 @@ def _build_to( # ruff: ignore[too-many-arguments] force: bool, options: BuildOptions, ) -> str: - """Build a release locally and copy it to a remote output if needed. + """Build a release for a local or remote output. - Reuse an existing release unless ``force`` is true. + Remote outputs receive a copy of a build in the local registry. If the output already holds the release and + ``force`` is not set, nothing runs and a status line says so. Args: dataset_id: The dataset ID. - target: The local directory or remote registry to write to. - fetch_from: The registry to copy missing parents from. ``None`` uses a remote output, if any. + target: The local output directory or remote registry. + fetch_from: The registry to copy missing parents from, or ``None`` to use the remote output if present. build_parents: Build missing parents with installed connectors. force: Rebuild an existing local release. - options: Settings for each build. + options: Options for each build. Returns: - The local version directory or the remote version string. + The local version directory or remote version string. Raises: BadParameter: If ``force`` is set and the remote output already holds the version. @@ -233,7 +236,9 @@ def _build_to( # ruff: ignore[too-many-arguments] console.success(f"Nothing to do: '{release.dataset_id}' {release.version} is already in the remote --out") return str(release.version) else: - store, remote = LocalRegistry(settings().registry_path), target + # TODO(adrian): Remove temporary glue which maps a remote --out onto a build into the + # local registry of the default list until --publish-to replaces a remote --out. + store, remote = LocalRegistry(settings().local_registry_dir), target # A build into the remote registry found the parents there. So without --fetch-from, the build # copies a missing parent from the remote registry into the local one. fetch_from = fetch_from if fetch_from is not None else remote diff --git a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py index 998bce222..a70877ca8 100644 --- a/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py +++ b/packages/timenet-connectors/src/timenet_connectors/datasets/bash_lab/sensorcaps/tests/test_connector.py @@ -215,7 +215,7 @@ def test_pipeline_preserves_source_semantics_after_source_removal(monkeypatch, t # Ten rows cover multiline features with/without location and a partial task batch. monkeypatch.setattr(control_writer, "_TASK_BATCH_SIZE", 3) monkeypatch.setenv("TIMENET_CACHE", str(tmp_path / "cache")) - cache = settings().cache_dir / "bash-lab/sensorcaps" + cache = settings().downloads_dir / "bash-lab/sensorcaps" async def download_async(self, cache_dir): assert cache_dir == cache diff --git a/packages/timenet-connectors/tests/test_build_backend.py b/packages/timenet-connectors/tests/test_build_backend.py index e0e6dd95f..8f7e51fac 100644 --- a/packages/timenet-connectors/tests/test_build_backend.py +++ b/packages/timenet-connectors/tests/test_build_backend.py @@ -113,7 +113,6 @@ def test_build_in_process_round_trips_the_connector_default_zarr_backend(tmp_pat connector_cls = backend_module.resolve("timenet/hello-world") monkeypatch.setattr(connector_cls, "values_backend", ValuesBackend.ZARR) monkeypatch.setenv("TIMENET_ISOLATION", "off") - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) store = LocalRegistry(tmp_path / "registry") ConnectorBuilder().build(HELLO, store, force=False, pinned_by={}, options=BuildOptions()) diff --git a/packages/timenet-connectors/tests/test_build_cli.py b/packages/timenet-connectors/tests/test_build_cli.py index 0b81d818e..808b0b55e 100644 --- a/packages/timenet-connectors/tests/test_build_cli.py +++ b/packages/timenet-connectors/tests/test_build_cli.py @@ -88,10 +88,9 @@ def test_build_isolates_when_the_flag_overrides_the_setting(monkeypatch, tmp_pat assert len(calls) == 1 -def test_build_without_isolation_shows_each_build_line_once(monkeypatch, tmp_path): +def test_build_without_isolation_shows_each_build_line_once(tmp_path): # In this process, the coordinator sends the build step, and the renderer shows it. Nothing else # prints a build line. - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) result = runner.invoke(app, ["build", "timenet/hello-world", "--out", str(tmp_path / "out"), "--no-isolation"]) @@ -177,7 +176,7 @@ def test_isolated_build_prints_the_version_dir_and_narrates_once(tmp_path): # the child's inherited stderr is captured here too, where a double-narration regression would # surface: the parent and the child both see the build step. registry = tmp_path / "registry" - env = {**os.environ, "TIMENET_HOME": str(tmp_path / "home")} + env = dict(os.environ) # the store, the remote registry and the cache of this test env.pop("TIMENET_ISOLATION", None) # default is "on"; make sure nothing forces it off run_cli = [sys.executable, "-c", "from timenet_connectors.builder.cli import main; main()"] result = subprocess.run( @@ -246,7 +245,6 @@ def test_the_build_prepared_release_command_does_not_plan(monkeypatch, tmp_path) monkeypatch.setattr(ConnectorBuilder, "card", lambda self, dataset_id: connectors[dataset_id]().metadata()) monkeypatch.setattr(backend_module, "resolve", connectors.__getitem__) monkeypatch.setenv("TIMENET_ISOLATION", "off") - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) monkeypatch.setattr(cli_module.console, "quiet", False) # The store lacks the parent. A planner would build it first. The command must fail on the @@ -261,7 +259,6 @@ def test_the_build_prepared_release_command_does_not_plan(monkeypatch, tmp_path) def test_the_build_prepared_release_command_builds_a_release_without_parents(monkeypatch, tmp_path): monkeypatch.setenv("TIMENET_ISOLATION", "off") - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) monkeypatch.setattr(cli_module.console, "quiet", False) result = runner.invoke( diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index 57fef1214..f69c385aa 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -538,8 +538,7 @@ def test_build_prepares_parents_before_the_isolated_child(tmp_path, monkeypatch) @pytest.mark.parametrize("obtain", ["fetch_from", "build_parents"]) def test_load_obtains_a_missing_parent_before_it_builds_and_reads(tmp_path, monkeypatch, obtain): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "out")) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "out")) calls = [] monkeypatch.setattr(backend_module, "run_isolated_build", _prepared_builds(calls)) origin = LocalRegistry(tmp_path / "origin") @@ -568,8 +567,7 @@ def test_load_copies_a_missing_payload_source_through_fetch_from(tmp_path, monke _parent_without_its_source(out) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.setenv("TIMENET_REGISTRY", str(out.root)) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(out.root)) dataset = timenet_connectors.load(PARENT.dataset_id, fetch_from=origin) @@ -616,7 +614,6 @@ def child(command, env): def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) out = LocalRegistry(tmp_path / "out") if case == "damaged": out.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) @@ -655,15 +652,15 @@ def _remote_out(tmp_path, monkeypatch, cards) -> tuple[_Hosted, LocalRegistry, l """Make the ``--out`` of the CLI a registry that is not local, and record each isolated build. Returns: - The registry that ``--out`` names, the default local registry, and the list of the builds. + The registry that ``--out`` names, the store, and the list of the builds. """ remote = _Hosted(tmp_path / "hosted") monkeypatch.setattr(cli_module, "_resolve_target", lambda target: remote) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) calls = [] monkeypatch.setattr(backend_module, "run_isolated_build", _prepared_builds(calls)) - return remote, LocalRegistry(tmp_path / "home/registry"), calls + return remote, LocalRegistry(tmp_path / "store"), calls def test_build_to_a_remote_out_refuses_a_credentialed_release_before_any_download(tmp_path, monkeypatch): @@ -824,7 +821,6 @@ def test_the_hint_names_only_the_option_that_the_build_did_not_use(tmp_path, mon # --fetch-from, and the API build uses build_parents=True. cards = _install_cards(monkeypatch, child={"parent": PARENT}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) out = tmp_path / "out" if entry == "cli": diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 44fef0388..853617a91 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -6,8 +6,7 @@ from timenet.client import TimeNet from timenet.config import settings -from timenet.engine import run_pipeline -from timenet.errors import TimeFValidationError +from timenet.errors import TimeFValidationError, TimeNetRegistryError from timenet.provenance import build_env from timenet.registry import LocalRegistry, RemoteRegistry from timenet.sources import InstalledConnectors @@ -78,31 +77,19 @@ def test_load_rejects_version_mismatch(): # ---- where a bare `build` writes ----------------------------------------------------------------- -@pytest.fixture -def clean_env(monkeypatch, tmp_path): - """Isolate the TIMENET_* vars so a bare build's output root is fully determined by the test.""" - for var in ("TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): - monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - - -def test_build_defaults_to_home_registry(clean_env, tmp_path): - assert runner.invoke(build_app, ["build", "timenet/hello-world", "--no-isolation"]).exit_code == 0 - assert (tmp_path / "home" / "registry" / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists() - - -def test_build_honors_timenet_registry(clean_env, monkeypatch, tmp_path): +def test_build_honors_timenet_local_registry(monkeypatch, tmp_path): registry = tmp_path / "elsewhere" - monkeypatch.setenv("TIMENET_REGISTRY", str(registry)) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(registry)) assert runner.invoke(build_app, ["build", "timenet/hello-world", "--no-isolation"]).exit_code == 0 assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists() - # The SDK resolves $TIMENET_REGISTRY the same way, so it reads back what the build just wrote. - assert TimeNet().list()[0].dataset_id == "timenet/hello-world" + # The store of the SDK is the same directory, so it reads back what the build just wrote. An exact + # version resolves at the store, without asking the remote registry. + assert TimeNet().load("timenet/hello-world@1.0.0").metadata.dataset_id == "timenet/hello-world" -def test_build_out_overrides_timenet_registry(clean_env, monkeypatch, tmp_path): - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "elsewhere")) +def test_build_out_overrides_timenet_local_registry(monkeypatch, tmp_path): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "elsewhere")) out = tmp_path / "out" result = runner.invoke(build_app, ["build", "timenet/hello-world", "--out", str(out), "--no-isolation"]) @@ -111,7 +98,7 @@ def test_build_out_overrides_timenet_registry(clean_env, monkeypatch, tmp_path): assert not (tmp_path / "elsewhere").exists() -def test_resolve_target_routes_remote_url_to_registry(clean_env, tmp_path): +def test_resolve_target_routes_remote_url_to_registry(tmp_path): # A remote --out publishes through a writable registry (constructed offline, no network). assert isinstance(_resolve_target("timenet://"), RemoteRegistry) assert isinstance(_resolve_target("https://registry.timenet.ai"), RemoteRegistry) @@ -119,29 +106,14 @@ def test_resolve_target_routes_remote_url_to_registry(clean_env, tmp_path): assert _resolve_target(str(tmp_path / "reg")) == tmp_path / "reg" -def test_resolve_target_publishes_to_remote_timenet_registry(clean_env, monkeypatch): - # A bare build with a remote $TIMENET_REGISTRY now publishes to it instead of failing. - monkeypatch.setenv("TIMENET_REGISTRY", "timenet://") - assert isinstance(_resolve_target(None), RemoteRegistry) - - -def test_run_pipeline_cleans_cache_when_requested(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - version_dir = run_pipeline(HelloWorldConnector(), tmp_path / "registry", keep_cache=False) - assert (version_dir / "manifest.json").exists() # build succeeded - assert not (settings().cache_dir / "timenet" / "hello-world").exists() # raw cache removed - - -def test_run_pipeline_keeps_cache_when_asked(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - run_pipeline(HelloWorldConnector(), tmp_path / "registry", keep_cache=True) - assert (settings().cache_dir / "timenet" / "hello-world").exists() +def test_resolve_target_rejects_an_empty_out_instead_of_the_current_directory(): + with pytest.raises(TimeNetRegistryError, match=r"^--out is blank"): + _resolve_target("") def test_sdk_build_cleans_cache_but_keep_cache_retains(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) monkeypatch.setenv("TIMENET_ISOLATION", "off") # the in-process path, the one that kept the cache - cache = settings().cache_dir / "timenet" / "hello-world" + cache = settings().downloads_dir / "timenet" / "hello-world" build("timenet/hello-world", out=tmp_path / "r1") assert not cache.exists() @@ -150,9 +122,8 @@ def test_sdk_build_cleans_cache_but_keep_cache_retains(tmp_path, monkeypatch): assert cache.exists() -def test_build_cleans_cache_but_keep_flag_retains(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - cache = settings().cache_dir / "timenet" / "hello-world" +def test_build_cleans_cache_but_keep_flag_retains(tmp_path): + cache = settings().downloads_dir / "timenet" / "hello-world" built = runner.invoke(build_app, ["build", "timenet/hello-world", "--out", str(tmp_path / "r1"), "--no-isolation"]) assert built.exit_code == 0 @@ -166,8 +137,7 @@ def test_build_cleans_cache_but_keep_flag_retains(tmp_path, monkeypatch): @pytest.mark.slow -def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) +def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path): client = TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]) dataset = client.load("timenet/hello-world") @@ -177,8 +147,7 @@ def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monk @pytest.mark.slow -def test_an_isolated_build_records_its_environment(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) +def test_an_isolated_build_records_its_environment(tmp_path): TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]).load("timenet/hello-world") manifest = json.loads((tmp_path / "registry/timenet/hello-world/1.0.0/manifest.json").read_text()) diff --git a/packages/timenet-connectors/tests/test_package_api.py b/packages/timenet-connectors/tests/test_package_api.py index 74166be0e..1e189abc2 100644 --- a/packages/timenet-connectors/tests/test_package_api.py +++ b/packages/timenet-connectors/tests/test_package_api.py @@ -17,11 +17,8 @@ def _unexpected_resolve(*args, **kwargs): raise AssertionError(f"resolve should not have run: {args} {kwargs}") -def test_build_then_load_round_trip(monkeypatch, tmp_path): - # timenet_connectors.build/load anchor on the same registry, so a build here loads back. The - # consumer default is the hosted service, so point both at a local registry for this round trip. - monkeypatch.setenv("TIMENET_HOME", str(tmp_path)) - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "registry")) +def test_build_then_load_round_trip(monkeypatch): + # timenet_connectors.build/load anchor on the store of the default list, so a build here loads back. # In-process, so the round trip stays in this interpreter instead of shelling out to uv. monkeypatch.setenv("TIMENET_ISOLATION", "off") monkeypatch.setattr(backend_module, "run_isolated_build", _unexpected_isolation) diff --git a/packages/timenet/src/timenet/_fs.py b/packages/timenet/src/timenet/_fs.py index 27856a475..0c03b0951 100644 --- a/packages/timenet/src/timenet/_fs.py +++ b/packages/timenet/src/timenet/_fs.py @@ -1,7 +1,8 @@ -"""Compare directory paths and replace directories from staging.""" +"""Compare directory paths, delete directory trees, and replace directories from staging.""" from collections.abc import Generator from contextlib import contextmanager +import os from pathlib import Path import shutil import uuid @@ -10,8 +11,8 @@ def same_directory(a: Path, b: Path) -> bool: """Check whether two paths name the same directory, as the file system sees them. - Existing paths match when they identify the same directory on disk, and never match a missing path. - Missing paths match when their nearest existing parents match and the names below them are equal as written. + The paths compare by :func:`path_identity`, so a symlink or a case variant on a volume that ignores case + matches, and names that do not exist compare as written. Args: a: The first path, whether or not it exists. @@ -20,30 +21,76 @@ def same_directory(a: Path, b: Path) -> bool: Returns: ``True`` if the paths match. """ - if a.exists() or b.exists(): - # On a volume that ignores case, a case variant of an existing path exists too. - return a.exists() and b.exists() and a.samefile(b) - return _below_existing_parent(a.resolve()) == _below_existing_parent(b.resolve()) + return path_identity(a.resolve()) == path_identity(b.resolve()) -def _below_existing_parent(path: Path) -> tuple[int, int, tuple[str, ...]]: - """Identify a missing path by its nearest existing parent and the names below that parent. +def path_identity(path: Path) -> tuple[int, int, tuple[str, ...]]: + """Identify a path using its nearest existing ancestor. Args: - path: A missing absolute path without symlinks. + path: An absolute path without symlinks. Returns: - The parent's device and inode numbers, followed by the names below it as written. + The ancestor's device and inode numbers, followed by the remaining names as written. """ - for depth, parent in enumerate(path.parents, start=1): + for depth, candidate in enumerate((path, *path.parents)): try: - info = parent.stat() + info = candidate.stat() except OSError: continue - return info.st_dev, info.st_ino, path.parts[len(path.parts) - depth :] + missing = path.parts[len(path.parts) - depth :] + return info.st_dev, info.st_ino, missing return 0, 0, path.parts # nothing exists, not even the root +def lies_in(path: Path, folder: Path) -> bool: + """Check whether ``path`` is ``folder`` or lies inside it. + + The check follows symlinks and respects the filesystem's case rules for existing paths. + It compares path parts that do not exist as written. + + Args: + path: The path to check, whether or not it exists. + folder: The folder to check against, whether or not it exists. + + Returns: + ``True`` if ``path`` is ``folder`` or lies inside it. + """ + target = path_identity(folder.expanduser().resolve()) + path = path.expanduser().resolve() + return any(path_identity(candidate) == target for candidate in (path, *path.parents)) + + +def delete_and_measure(path: Path) -> int: + """Delete a directory tree and return the bytes it held. It stats each file only once. + + One file pass both measures and unlinks each file. Then :func:`shutil.rmtree` removes the empty + directory skeleton. This avoids a second walk over every file. + + An unreadable directory or a failed removal raises an ``OSError``. + + Args: + path: The directory to delete. + + Returns: + The total size in bytes of the files removed. + """ + + def fail(error: OSError) -> None: + raise error # os.walk otherwise skips a directory it cannot read, silently leaving it behind + + freed = 0 + for parent, _dirs, files in os.walk(path, topdown=False, onerror=fail): # os.walk: Path.walk is 3.12+ + parent_dir = Path(parent) + for name in files: + entry = parent_dir / name + # lstat: a symlink counts as itself, not as its (possibly huge, possibly external) target. + freed += entry.lstat().st_size + entry.unlink(missing_ok=True) # a concurrent build may have removed it between walk and unlink + shutil.rmtree(path) + return freed + + def publish_directory(staging: Path, final: Path) -> None: """Rename ``staging`` to ``final``, replacing any existing directory. diff --git a/packages/timenet/src/timenet/_registry_uri.py b/packages/timenet/src/timenet/_registry_uri.py new file mode 100644 index 000000000..3be0c4bc8 --- /dev/null +++ b/packages/timenet/src/timenet/_registry_uri.py @@ -0,0 +1,86 @@ +"""Parse registry URLs and local directory paths.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal +from urllib.parse import urlparse +from urllib.request import url2pathname + +from timenet.errors import TimeNetRegistryError + + +TIMENET_REGISTRY_URL = "https://registry.timenet.ai" +"""The hosted registry URL represented by ``timenet://``.""" + +_REMOTE_SCHEMES = ("timenet://", "http://", "https://", "s3://") + +_SCHEMES = "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." + + +@dataclass(frozen=True) +class RemoteLocation: + """The URL of a remote registry and the kind of service behind it.""" + + url: str + """The URL, with ``timenet://`` resolved to the hosted registry.""" + scheme: Literal["http", "s3"] + """``"s3"`` for an S3 bucket, ``"http"`` for a TimeNet HTTP service.""" + + +def registry_location(uri: str | Path, *, label: str = "registry") -> RemoteLocation | Path: + """Parse a registry value into the location of a remote registry or the directory of a local one. + + Args: + uri: A URL, ``timenet://`` / ``s3://`` / ``file://`` URI, or local path. + label: The name of the value in an error message, such as ``sources[0]``. + + Returns: + A remote location with ``timenet://`` resolved, or a local path with ``~`` expanded. + + Raises: + TimeNetRegistryError: If the value is blank or uses an unsupported scheme or invalid registry URI. + """ + text = str(uri) + if text.startswith("timenet://"): + # timenet:// is a bare alias for the hosted service root; it takes no path (dataset ids are + # passed to the client methods, not folded into the registry URI). Reject a stray remainder + # rather than drop it silently, so a mistyped URI fails loudly instead of hitting the default host. + if text[len("timenet://") :].strip("/"): + raise TimeNetRegistryError(f"{label} {text!r}: timenet:// takes no path") + return RemoteLocation(TIMENET_REGISTRY_URL, "http") + if text.startswith("s3://"): + return RemoteLocation(text, "s3") + if text.startswith(_REMOTE_SCHEMES): + return RemoteLocation(text, "http") + return local_registry_path(text, label=label) + + +def local_registry_path(uri: str | Path, *, label: str = "registry") -> Path: + """Resolve a local registry address to a directory path. + + Args: + uri: A ``file://`` URI or local path. + label: The name of the value in an error message, such as ``sources[0]``. + + Returns: + The local directory path with ``~`` expanded. + + Raises: + TimeNetRegistryError: If the value is blank, remote or an invalid local registry URI. + """ + text = str(uri) + if not text.strip(): + # Path("") is the current directory, which a blank value never means. + raise TimeNetRegistryError(f"{label} is blank; give a registry URL or a local path") + if text.startswith(_REMOTE_SCHEMES): + raise TimeNetRegistryError(f"{label} {text!r} is remote; the local registry needs a local directory") + if text.startswith("file://"): + parsed = urlparse(text) + if parsed.netloc: + raise TimeNetRegistryError(f"{label} {text!r} must be an absolute file:// URI (three slashes). {_SCHEMES}") + if not parsed.path: + raise TimeNetRegistryError(f"{label} {text!r} must name an absolute path") + return Path(url2pathname(parsed.path)).expanduser() + if "://" in text: + raise TimeNetRegistryError(f"{label} {text!r} has an unsupported registry scheme. {_SCHEMES}") + return Path(text).expanduser() diff --git a/packages/timenet/src/timenet/cache.py b/packages/timenet/src/timenet/cache.py index 590697da3..71fcacd7e 100644 --- a/packages/timenet/src/timenet/cache.py +++ b/packages/timenet/src/timenet/cache.py @@ -1,34 +1,51 @@ -"""Inspect and clear the local TimeNet cache under ``~/.cache/timenet/`` (see :mod:`timenet.config`). +"""Inspect and clear the cache and the stored releases. -Datasets live in the local registry (built locally) and the download storage. Raw sources land in the -download cache. All three sit under the home directory. +Cache removal requires TimeNet's ``CACHEDIR.TAG`` marker. """ -# TODO(adrian): Remove temporary glue which maps the "download storage" of this module onto the -# storage directory, settings().storage_dir, until a later change replaces this module and its -# view of the storage directory. download() copies into the local registry now, so the storage directory -# holds only the copies that remote registries cache. - +from collections.abc import Iterable from dataclasses import dataclass -import os from pathlib import Path -import shutil -from timenet.config import settings -from timenet.format.constants import MANIFEST_FILE +from timenet._fs import delete_and_measure, lies_in +from timenet.config import TimeNetSettings, is_cache_folder +from timenet.errors import TimeFValidationError +from timenet.registry import LocalRegistry +from timenet.registry._paths import release_dir +from timenet.types.references import DatasetRef @dataclass(frozen=True) -class CachedDataset: - """One cached dataset version on disk.""" +class StoredRelease: + """A stored release with its path and size in bytes.""" - location: str # "registry" or "storage" - dataset_id: str - version: str + release: DatasetRef path: Path size_bytes: int +@dataclass(frozen=True) +class ClearedCache: + """The bytes freed and paths removed or kept by a cache clear.""" + + freed: int + """The number of bytes freed.""" + removed: list[Path] + """The paths successfully removed.""" + kept: list[Path] + """The cache paths kept because they lack TimeNet's cache marker.""" + + +@dataclass(frozen=True) +class ClearedReleases: + """The bytes freed and release folders removed by a release clear.""" + + freed: int + """The number of bytes freed.""" + removed: list[Path] + """The release folders successfully removed.""" + + def human_bytes(n: int) -> str: """Format a byte count as a human-readable string. @@ -47,105 +64,147 @@ def human_bytes(n: int) -> str: return f"{size:.1f} TB" -def cached_datasets() -> list[CachedDataset]: - """List dataset versions and their sizes in the local registry and storage directory. +def stored_releases(store: LocalRegistry) -> list[StoredRelease]: + """List the releases of a local registry and their sizes in bytes. + + Args: + store: The local registry to inspect. Returns: - The cached versions sorted by location, dataset ID, and version. + The releases, sorted by dataset ID and version. """ - config = settings() - # TODO(adrian): Remove temporary glue which maps the download storage onto the storage - # directory until a later change replaces this module, as the note at the top says. - found = _scan(config.registry_path, "registry") + _scan(config.storage_dir, "storage") - return sorted(found, key=lambda dataset: (dataset.location, dataset.dataset_id, dataset.version)) + found: list[StoredRelease] = [] + for release in store.releases(): + path = release_dir(store.root, release) + found.append(StoredRelease(release, path, _dir_size(path))) + return found + +def cache_size(cache_dir: Path) -> int: + """Measure cached downloads and control databases marked as TimeNet's. -def raw_cache_size() -> int: - """Return the size in bytes of the raw download cache directory (0 if absent). + Only ``downloads`` and ``control`` folders with TimeNet's ``CACHEDIR.TAG`` file count. + A symlink contributes its own size rather than its target's size. + + Args: + cache_dir: The cache directory. Returns: - The total size of ````. + The total size in bytes, or ``0`` if no marked cache folders exist. """ - cache_dir = settings().cache_dir - return _dir_size(cache_dir) if cache_dir.is_dir() else 0 + total = 0 + for folder in _cache_folders(cache_dir): + if is_cache_folder(folder): + total += folder.lstat().st_size if folder.is_symlink() else _dir_size(folder) + return total -def clear_cache(*, include_registry: bool) -> tuple[int, list[Path]]: - """Delete the storage directory and raw download cache. +def clear_cache(cache_dir: Path, *, local_registry: LocalRegistry) -> ClearedCache: + """Clear cached raw source files and control databases. - Read or removal failures raise ``OSError``. + Only folders with TimeNet's ``CACHEDIR.TAG`` marker are removed. + Removing a symlink keeps its target. Args: - include_registry: Remove the local registry too. + cache_dir: The cache directory. + local_registry: The local registry, which must not overlap a cache folder selected for deletion. Returns: - A ``(bytes_freed, removed_dirs)`` pair with the removed directory paths. - """ - config = settings() - # TODO(adrian): Remove temporary glue which maps the download storage onto the storage - # directory until a later change replaces this module, as the note at the top says. - targets = [config.storage_dir, config.cache_dir] - if include_registry: - targets.append(config.registry_path) - + The bytes freed, paths successfully removed and cache paths kept. + + Raises: + TimeFValidationError: Before any deletion, if the cache is the home folder or a folder above it, or the local + registry overlaps a cache folder selected for deletion. + OSError: If a folder cannot be removed completely. + """ # ruff: ignore[docstring-extraneous-exception] (raised by _check_cache_deletion and delete_and_measure) + folders = _cache_folders(cache_dir) + marked = [folder for folder in folders if is_cache_folder(folder)] + if marked: + # A symlink goes and its target stays, so only the other folders can hold the local registry. + deleted = [folder for folder in marked if not folder.is_symlink()] + _check_cache_deletion(cache_dir, deleted, local_registry=local_registry.root) freed = 0 removed: list[Path] = [] - for target in targets: - if target.is_dir(): - freed += _delete_and_measure(target) - removed.append(target) - return freed, removed - + for folder in marked: + if folder.is_symlink(): + freed += folder.lstat().st_size + folder.unlink() # a symlink goes, and its target stays + else: + freed += delete_and_measure(folder) + removed.append(folder) + return ClearedCache(freed, removed, [folder for folder in folders if folder not in marked]) -def _delete_and_measure(path: Path) -> int: - """Delete a directory tree and return the bytes it held. It stats each file only once. - One file pass both measures and unlinks each file. Then :func:`shutil.rmtree` removes the empty - directory skeleton. This avoids a second walk over every file. +def clear_releases(store: LocalRegistry) -> ClearedReleases: + """Delete stored releases and the dataset directories left empty. - An unreadable directory or a failed removal raises an ``OSError``. + Other files in the local registry stay. Args: - path: The directory to delete. + store: The local registry. Returns: - The total size in bytes of the files removed. - """ - - def fail(error: OSError) -> None: - raise error # os.walk otherwise skips a directory it cannot read, silently leaving it behind + The bytes freed and the release folders removed. + Raises: + TimeNetRegistryError: If the local registry is read-only. + OSError: If a folder cannot be removed completely. + """ # ruff: ignore[docstring-extraneous-exception] (raised by remove_release) freed = 0 - for parent, _dirs, files in os.walk(path, topdown=False, onerror=fail): # os.walk: Path.walk is 3.12+ - parent_dir = Path(parent) - for name in files: - entry = parent_dir / name - # lstat: a symlink counts as itself, not as its (possibly huge, possibly external) target. - freed += entry.lstat().st_size - entry.unlink(missing_ok=True) # a concurrent build may have removed it between walk and unlink - shutil.rmtree(path) - return freed + removed: list[Path] = [] + for release in store.releases(): + freed += store.remove_release(release) + removed.append(release_dir(store.root, release)) + return ClearedReleases(freed, removed) + + +def _check_cache_deletion(cache_dir: Path, folders: Iterable[Path], *, local_registry: Path) -> None: + """Check that TimeNet may delete folders of the cache. + + Args: + cache_dir: The cache directory. + folders: The folders of the cache to delete. + local_registry: The local registry directory, which must stay. + + Raises: + TimeFValidationError: If the cache is the home folder or a folder above it, or a folder and the local registry + overlap. + """ + home = _home() + if home is not None and lies_in(home, cache_dir): + raise TimeFValidationError( + f"The cache {cache_dir} is the home folder or a folder above it. TimeNet deletes the downloads and control " + "folders of the cache, so give TIMENET_CACHE a folder of its own, for example " + f"{TimeNetSettings.model_fields['cache'].default}." + ) + for folder in folders: + if lies_in(local_registry, folder) or lies_in(folder, local_registry): + raise TimeFValidationError( + f"the local registry {local_registry} overlaps {folder}, which TimeNet deletes. Move the local " + "registry out of the cache" + ) + + +def _home() -> Path | None: + """Find the user's home directory. + + Returns: + The home directory, or ``None`` if the system cannot find it. + """ + try: + return Path.home() + except RuntimeError: + return None -def _scan(root: Path, location: str) -> list[CachedDataset]: - """Find every ``<...>//manifest.json`` under ``root`` and measure its version directory. +def _cache_folders(cache_dir: Path) -> list[Path]: + """Find existing download and control cache paths. Returns: - The cached datasets found under ``root``. + The ``downloads`` and ``control`` paths that exist, including symlinks. """ - if not root.is_dir(): - return [] - datasets: list[CachedDataset] = [] - for manifest_path in root.rglob(MANIFEST_FILE): - version_dir = manifest_path.parent - parts = version_dir.relative_to(root).parts - if any(part.startswith(".") or ".tmp-" in part for part in parts): - continue - dataset_id = "/".join(parts[:-1]) - if not dataset_id: - continue - datasets.append(CachedDataset(location, dataset_id, parts[-1], version_dir, _dir_size(version_dir))) - return datasets + folders = (cache_dir / "downloads", cache_dir / "control") + return [folder for folder in folders if folder.is_symlink() or folder.exists()] def _dir_size(path: Path) -> int: diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 0134b4b01..4900abd56 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -2,16 +2,18 @@ from collections.abc import Callable, Iterable from functools import partial +from pathlib import Path from typing import TypeVar from rich.console import Console from rich.table import Table import typer -from timenet.cache import CachedDataset, cached_datasets, clear_cache, human_bytes, raw_cache_size +from timenet.cache import cache_size, clear_cache, clear_releases, human_bytes, stored_releases from timenet.cli.runner import run_cli from timenet.cli.ui import console as ui, progress_renderer -from timenet.client import TimeNet, default_sources +from timenet.client import TimeNet, default_local_registry, default_sources +from timenet.config import settings from timenet.progress import ProgressCallback, ProgressEvent, StepStarted from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access @@ -26,7 +28,7 @@ None, "--registry", "-r", - help="Registry URL or path that this command reads (else $TIMENET_REGISTRY, then timenet://).", + help="Registry URL or path that this command reads (else $TIMENET_REMOTE_REGISTRY, then timenet://).", ) # TODO(adrian): Remove temporary glue which maps --registry onto the list after the local registry, until the CLI # takes the flags of the defaults. @@ -230,61 +232,84 @@ def _keep_steps(started: list[StepStarted], show: ProgressCallback | None, event show(event) -cache_app = typer.Typer(help="Inspect and clear the local cache.", no_args_is_help=True) -app.add_typer(cache_app, name="cache") +local_app = typer.Typer(help="Inspect and clear the local files of TimeNet.", no_args_is_help=True) +app.add_typer(local_app, name="local") +cache_app = typer.Typer( + help="The cache in $TIMENET_CACHE: raw source files of connectors and control databases of remote registries.", + no_args_is_help=True, +) +local_app.add_typer(cache_app, name="cache") + +_YES = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt.") + + +@local_app.command("info") +def local_info() -> None: + """Show the path and size of the cache and the local registry.""" + store, cache_dir = default_local_registry(), settings().cache_dir + releases = stored_releases(store) + size = human_bytes(sum(item.size_bytes for item in releases)) + # Plain lines, so that a long path does not wrap. + typer.echo(_cache_line(cache_dir)) + typer.echo(f"Local registry {store.root}: {_count(len(releases), 'release')}, {size}") + + +@local_app.command("clear") +def local_clear(yes: bool = _YES) -> None: + """Delete the cache and every release in the local registry.""" + store, cache_dir = default_local_registry(), settings().cache_dir + if not yes: + typer.confirm(f"Delete the cache in {cache_dir} and every release in {store.root}?", abort=True) + # The cache clear checks the folders before it deletes anything, so it runs first. + cache = clear_cache(cache_dir, local_registry=store) + releases = clear_releases(store) + _warn_unmarked(cache.kept) + _freed(releases.freed + cache.freed, len(releases.removed) + len(cache.removed)) @cache_app.command("info") def cache_info() -> None: - """Show the datasets in the local cache with their sizes.""" - datasets = cached_datasets() - if datasets: - console.print(_cache_table(datasets)) - else: - console.print("No cached datasets.") - raw = raw_cache_size() - if raw: - console.print(f"Raw download cache: {human_bytes(raw)}") + """Show the path and size of the cache.""" + typer.echo(_cache_line(settings().cache_dir)) -def _cache_table(datasets: list[CachedDataset]) -> Table: - """Build a table of cached dataset versions with a total-size footer. +@cache_app.command("clear") +def cache_clear(yes: bool = _YES) -> None: + """Delete the raw source files and the control databases in the cache.""" + cache_dir = settings().cache_dir + if not yes: + typer.confirm(f"Delete the raw source files and control databases in {cache_dir}?", abort=True) + cleared = clear_cache(cache_dir, local_registry=default_local_registry()) + _warn_unmarked(cleared.kept) + _freed(cleared.freed, len(cleared.removed)) - Args: - datasets: The cached dataset versions to render. - Returns: - A populated :class:`rich.table.Table`. - """ - table = Table(title="TimeNet cache", title_justify="left", title_style="bold") - table.add_column("Location") - table.add_column("Dataset", style="cyan") - table.add_column("Version") - table.add_column("Size", justify="right") - for dataset in datasets: - table.add_row(dataset.location, dataset.dataset_id, dataset.version, human_bytes(dataset.size_bytes)) - table.add_section() - total = sum(dataset.size_bytes for dataset in datasets) - table.add_row("total", "", "", human_bytes(total), style="bold") - return table +def _cache_line(cache_dir: Path) -> str: + """Return the path and the size of the cache.""" + return f"Cache {cache_dir}: {human_bytes(cache_size(cache_dir))}" -@cache_app.command("clear") -def cache_clear( - all_: bool = typer.Option(False, "--all", help="Also remove the default local registry (builds and downloads)."), - yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt."), -) -> None: - """Delete the storage directory and raw cache. +def _warn_unmarked(folders: list[Path]) -> None: + """Warn about cache folders kept because they lack TimeNet's marker.""" + for folder in folders: + ui.warn( + f"Kept {folder}: it has no CACHEDIR.TAG from TimeNet, so TimeNet did not create it. Delete it " + "yourself if you do not need it." + ) - Use ``--all`` to also remove the default local registry. + +def _freed(freed: int, locations: int) -> None: + """Print the bytes that a clear freed, and from how many locations.""" + typer.echo(f"Freed {human_bytes(freed)} from {locations} location(s).") + + +def _count(number: int, noun: str) -> str: + """Format a count with a noun, adding ``s`` unless the count is 1. + + Returns: + The count and noun, such as ``"1 build"`` or ``"2 builds"``. """ - if not yes: - what = "the storage directory and the raw cache" - if all_: - what += " and the local registry (builds and downloads)" - typer.confirm(f"Remove {what}?", abort=True) - freed, removed = clear_cache(include_registry=all_) - typer.echo(f"Freed {human_bytes(freed)} from {len(removed)} location(s).") + return f"{number} {noun}" if number == 1 else f"{number} {noun}s" def main() -> None: diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index c67edadd1..2d5a5a834 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -61,34 +61,48 @@ _OrList: TypeAlias = T | list[T] | None -def default_sources(*, remote_registry: str | Path | None = None) -> list[ReleaseSource]: - """Return the local registry, an optional remote registry, and the installed connectors in that order. +def default_local_registry(*, local_registry: str | Path | None = None) -> LocalRegistry: + """Open the default local registry for downloaded and built releases. + + Args: + local_registry: The local registry directory (``TIMENET_LOCAL_REGISTRY``, default ``~/.timenet/registry``). + + Returns: + A writable local registry. + + Raises: + TimeFValidationError: If the local registry path or another setting is invalid. + TimeNetRegistryError: If ``local_registry`` names a remote registry. + """ # ruff: ignore[docstring-extraneous-exception] (raised by settings) + return LocalRegistry(settings(local_registry=local_registry).local_registry_dir) - The local registry uses a local ``$TIMENET_REGISTRY`` or ``/registry``. - The optional registry uses ``remote_registry``, a remote ``$TIMENET_REGISTRY``, or ``timenet://`` - if the variable is unset. + +def default_sources( + *, + local_registry: str | Path | None = None, + remote_registry: str | Path | None = None, + auto_build: bool | None = None, +) -> list[ReleaseSource]: + """Return the local registry, the remote registry and optional installed connectors in lookup order. + + A ``None`` argument uses the current environment setting or its default. Args: - remote_registry: A registry URL or path, or ``None`` to use the environment settings. + local_registry: The local registry directory (``TIMENET_LOCAL_REGISTRY``, default ``~/.timenet/registry``). + remote_registry: A registry URL or path (``TIMENET_REMOTE_REGISTRY``, default ``timenet://``). + auto_build: Whether to end with ``InstalledConnectors()`` (``TIMENET_AUTO_BUILD``, default true). Returns: - A new source list based on the current environment. - """ - # TODO(adrian): Remove temporary glue which maps TIMENET_REGISTRY and TIMENET_HOME onto the - # list until the variables of the defaults (TIMENET_LOCAL_REGISTRY, TIMENET_REMOTE_REGISTRY, - # TIMENET_AUTO_BUILD) replace them. - cfg = settings() - configured = None if cfg.registry is None else open_registry(cfg.registry) - local = configured if isinstance(configured, LocalRegistry) else None - store = LocalRegistry(cfg.registry_path if local is None else local.root) - if remote_registry is not None: - remote = open_registry(remote_registry) - elif local is not None: - # A client of a local $TIMENET_REGISTRY read no other registry, and neither does this list. - return [store, InstalledConnectors()] - else: - remote = configured if configured is not None else open_registry("timenet://") - return [store, remote, InstalledConnectors()] + A new list of sources in lookup order. + + Raises: + TimeFValidationError: If an argument or environment setting is invalid. + TimeNetRegistryError: If ``local_registry`` names a remote registry, or the remote registry cannot be opened. + """ # ruff: ignore[docstring-extraneous-exception] (raised by settings and open_registry) + cfg = settings(local_registry=local_registry, remote_registry=remote_registry, auto_build=auto_build) + store = LocalRegistry(cfg.local_registry_dir) + remote = open_registry(cfg.remote_registry, label="TIMENET_REMOTE_REGISTRY") + return [store, remote, *([InstalledConnectors()] if cfg.auto_build else [])] def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None]: @@ -155,7 +169,7 @@ def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) ``read_only=True``, which ``InstalledConnectors()`` requires. Args: - sources: Sources in read order, or ``None`` to use :func:`default_sources`. + sources: Ordered registries, URLs, paths or connectors, or ``None`` for :func:`default_sources`. Raises: TimeFValidationError: If sources are invalid, or have connectors but no writable local registry. diff --git a/packages/timenet/src/timenet/config.py b/packages/timenet/src/timenet/config.py index 470bc0de7..b32c68b76 100644 --- a/packages/timenet/src/timenet/config.py +++ b/packages/timenet/src/timenet/config.py @@ -1,27 +1,51 @@ -"""Runtime configuration, resolved with pydantic-settings. +"""Load TimeNet settings from arguments and ``TIMENET_*`` environment variables. -All local state lives under one home directory (default ``~/.cache/timenet``). This mirrors -HuggingFace's ``HF_HOME`` -> ``HF_DATASETS_CACHE`` / ``HF_HUB_CACHE`` hierarchy. Set ``TIMENET_HOME`` -to relocate everything. Each per-area env var overrides only its own path. For any value the precedence -is explicit argument (for example a CLI flag), then environment variable, then default. Add future -configuration to this model. +Explicit values override environment variables, which override defaults. """ +import os from pathlib import Path from typing import Any, Literal +from pydantic import ValidationError, ValidationInfo, field_validator +from pydantic_core import ErrorDetails from pydantic_settings import BaseSettings, SettingsConfigDict +from timenet._registry_uri import local_registry_path +from timenet.errors import TimeFValidationError + + +# Follow the Cache Directory Tagging Specification by Bryan Ford, from 2004. +# +# A folder is a cache if it holds a file CACHEDIR.TAG that starts with a fixed signature line. +# Many common tools write it, for example Cargo in target/, pytest in .pytest_cache, ruff, mypy +# and uv in their caches. TimeNet's own cleanup logic uses it as a safety check. This way, a +# TIMENET_CACHE that points at the wrong folder, such as your home or another tool's cache, is +# never wiped. Backup tools also skip TimeNet's caches as a side effect. +CACHE_TAG = "CACHEDIR.TAG" +"""The marker filename for cache folders created by TimeNet.""" + +_CACHE_TAG_LINES = ( + "Signature: 8a477f597d28d172789f06886806bc55", + "# This file is a cache directory tag created by TimeNet.", + "# For information about cache directory tags, see https://bford.info/cachedir/", +) +"""The cache marker text, starting with the standard cache tag signature.""" + class TimeNetSettings(BaseSettings): - """TimeNet's settings. Reads ``TIMENET_*`` environment variables (and an optional ``.env``).""" + """TimeNet settings from arguments, ``TIMENET_*`` variables and an optional ``.env`` file.""" model_config = SettingsConfigDict(env_prefix="TIMENET_", env_file=".env", extra="ignore") - home: Path = Path("~/.cache/timenet") - storage: Path | None = None - cache: Path | None = None - registry: str | None = None + local_registry: str = "~/.timenet/registry" + """The local registry path or ``file://`` URI (``TIMENET_LOCAL_REGISTRY``).""" + remote_registry: str = "timenet://" + """The registry after the local registry in the default sources (``TIMENET_REMOTE_REGISTRY``).""" + auto_build: bool = True + """Whether default sources include installed connectors (``TIMENET_AUTO_BUILD``).""" + cache: str = "~/.cache/timenet" + """The cache directory for raw downloads and control databases (``TIMENET_CACHE``).""" token: str | None = None """Bearer token for a remote registry (``TIMENET_TOKEN``); ``None`` is anonymous.""" isolation: Literal["on", "off"] = "on" @@ -31,42 +55,122 @@ class TimeNetSettings(BaseSettings): environment, which is what stops it re-execing forever. """ - @property - def home_dir(self) -> Path: - """The resolved home directory (analog of ``HF_HOME``).""" - return self.home.expanduser() + @field_validator("local_registry", "remote_registry", "cache", mode="before") + @classmethod + def _not_empty(cls, value: Any, info: ValidationInfo) -> Any: + """Convert paths to strings and reject blank settings. + + Args: + value: The value to validate. + info: Validation metadata that includes the field name. + + Returns: + The value with any path converted to a string. + + Raises: + TimeFValidationError: If the value is empty or contains only whitespace. + """ + if isinstance(value, os.PathLike): + value = os.fspath(value) + if isinstance(value, str) and not value.strip(): + name = str(info.field_name) + raise TimeFValidationError( + f"TIMENET_{name.upper()} must not be empty; give it a value, " + f"or leave it unset for {cls.model_fields[name].default}." + ) + return value @property - def storage_dir(self) -> Path: - """Where the client caches loaded or downloaded datasets (analog of ``HF_DATASETS_CACHE``).""" - return (self.storage if self.storage is not None else self.home_dir / "storage").expanduser() + def local_registry_dir(self) -> Path: + """The local registry directory with ``~`` expanded. + + Raises: + TimeNetRegistryError: If the local registry path is blank, remote or uses an invalid URI. + """ # ruff: ignore[docstring-extraneous-exception] (raised by local_registry_path) + return local_registry_path(self.local_registry) @property def cache_dir(self) -> Path: - """Where the client caches raw build sources and Hub downloads (analog of ``HF_HUB_CACHE``).""" - return (self.cache if self.cache is not None else self.home_dir / "cache").expanduser() + """The cache directory path with ``~`` expanded.""" + return Path(self.cache).expanduser() + + @property + def downloads_dir(self) -> Path: + """The raw download cache directory, with a subdirectory for each dataset ID.""" + return self.cache_dir / "downloads" @property def control_cache_dir(self) -> Path: """Where verified remote control databases are cached by checksum.""" return self.cache_dir / "control" - @property - def registry_path(self) -> Path: - """The default local registry directory (``$TIMENET_REGISTRY`` is the selector, not this).""" - return self.home_dir / "registry" - def settings(**overrides: Any) -> TimeNetSettings: - """Build settings so explicit non-``None`` overrides win over env vars and defaults. + """Load and validate the TimeNet settings. + + Overrides other than ``None`` take precedence over environment variables and defaults. + + Args: + **overrides: Values by field name, with ``None`` values ignored. + + Returns: + The validated settings. + + Raises: + TimeFValidationError: If a value is invalid. + """ + given = {key: value for key, value in overrides.items() if value is not None} + try: + return TimeNetSettings(**given) + except ValidationError as error: + raise TimeFValidationError(" ".join(_sentence(item) for item in error.errors())) from error + + +def _sentence(item: ErrorDetails) -> str: + """Describe a setting error using its environment variable name. + + Args: + item: One error from a pydantic ``ValidationError``. + + Returns: + A sentence describing the invalid setting. + """ + cause = item.get("ctx", {}).get("error") + if isinstance(cause, TimeFValidationError): + return str(cause) + name = "TIMENET_" + "_".join(str(part) for part in item["loc"]).upper() + return f"{name} is invalid: {item['msg']}, got {item['input']!r}." + + +def make_cache_folder(folder: Path) -> None: + """Create a cache folder with TimeNet's ``CACHEDIR.TAG`` marker. + + Existing folders stay unchanged. + + Args: + folder: The cache folder to create along with any missing parents. + """ + try: + folder.mkdir(parents=True) + except FileExistsError: + return + (folder / CACHE_TAG).write_text("\n".join(_CACHE_TAG_LINES) + "\n") + + +def is_cache_folder(folder: Path) -> bool: + """Check a folder for TimeNet's ``CACHEDIR.TAG`` marker. - This function drops a ``None`` override. A missing CLI flag then falls through to the environment, - then the default. + The check follows symlinks. Args: - **overrides: Field overrides (for example ``storage=...``, ``registry=...``). + folder: The folder to check. Returns: - The resolved :class:`TimeNetSettings`. + ``True`` if the marker's first two lines match TimeNet's cache tag. """ - return TimeNetSettings(**{key: value for key, value in overrides.items() if value is not None}) + try: + with (folder / CACHE_TAG).open(encoding="utf-8", errors="replace") as tag: + head = [tag.readline().rstrip("\n") for _ in range(2)] + except OSError: + return False + return head == list(_CACHE_TAG_LINES[:2]) diff --git a/packages/timenet/src/timenet/engine/engine.py b/packages/timenet/src/timenet/engine/engine.py index 22a0e4e99..cf9405a68 100644 --- a/packages/timenet/src/timenet/engine/engine.py +++ b/packages/timenet/src/timenet/engine/engine.py @@ -5,7 +5,8 @@ from pathlib import Path import shutil -from timenet.config import settings +from timenet.cache import _check_cache_deletion +from timenet.config import is_cache_folder, make_cache_folder, settings from timenet.connectors import BaseConnector, BuildContext from timenet.dataset import TimeFDataset from timenet.dataset.composition import _record_parent @@ -48,7 +49,7 @@ def run_pipeline( # ruff: ignore[too-many-arguments] Args: connector: The connector to build. root: Output root. This function writes the dataset to ``///``. - cache_dir: Directory for downloaded artifacts (defaults to ``/``). + cache_dir: Directory for downloaded artifacts (defaults to ``/downloads/``). keep_cache: Keep the cache directory instead of removing it once the dataset is stored. Conversion is the only stage that needs the raw sources, so removing them frees disk after a successful build. The sources re-download on the next run. @@ -63,8 +64,10 @@ def run_pipeline( # ruff: ignore[too-many-arguments] Raises: TimeFValidationError: If the card declares parents and the connector does not override ``compose``, or the dataset that ``compose`` returns has another card than the one that - the engine read. The build then stores no release. - """ # ruff: ignore[docstring-extraneous-exception] - raised by _check_composes and _compose + the engine read. The build then stores no release. Also before any download, if + ``root`` and the raw downloads that the build removes overlap, or the cache is the home + folder or a folder above it. + """ # ruff: ignore[docstring-extraneous-exception] - raised by _check_composes, _compose and the cache check values_backend = _check_values_backend(connector) # Read only the connector's dataset.yaml card, a tiny local file, not the dataset itself. This lets # us resolve the version directory and skip the expensive download and convert when it exists. @@ -75,7 +78,19 @@ def run_pipeline( # ruff: ignore[too-many-arguments] return version_dir _check_composes(connector, metadata) - cache = cache_dir if cache_dir is not None else settings().cache_dir / metadata.dataset_id + # Only clean a cache that we created. A caller-supplied cache_dir is user-owned. We must never + # delete it. + downloads = None + if cache_dir is None: + cfg = settings() + downloads = cfg.downloads_dir + make_cache_folder(downloads) + cache = downloads / metadata.dataset_id + if not keep_cache and is_cache_folder(downloads): + # The build removes these downloads at the end, so it checks them before the first download. + _check_cache_deletion(cfg.cache_dir, [cache], local_registry=root) + else: + cache = cache_dir cache.mkdir(parents=True, exist_ok=True) # Open the declared parents before downloading, so a missing parent fails before any raw I/O. @@ -93,13 +108,22 @@ def run_pipeline( # ruff: ignore[too-many-arguments] replace=force, pinned_by=pinned_by, ) - # Only clean a cache that we created. A caller-supplied cache_dir is user-owned. We must never - # delete it. - if not keep_cache and cache_dir is None and cache.is_dir(): - shutil.rmtree(cache) + if not keep_cache and downloads is not None: + _remove_downloads(downloads, cache) return version_dir +def _remove_downloads(downloads: Path, folder: Path) -> None: + """Remove dataset downloads when the download cache has TimeNet's marker. + + Args: + downloads: The download cache folder. + folder: The dataset directory in ``downloads``. + """ + if folder.is_dir() and is_cache_folder(downloads): + shutil.rmtree(folder) + + def open_parents(metadata: DatasetMetadata, registry: LocalRegistry) -> BuildContext: """Open readers for the dataset's declared parents in a local registry. diff --git a/packages/timenet/src/timenet/format/control_cache.py b/packages/timenet/src/timenet/format/control_cache.py index 19dff5d9e..9808f8330 100644 --- a/packages/timenet/src/timenet/format/control_cache.py +++ b/packages/timenet/src/timenet/format/control_cache.py @@ -10,7 +10,7 @@ import pyarrow.fs as pafs -from timenet.config import settings +from timenet.config import make_cache_folder, settings from timenet.errors import TimeFFormatError from timenet.format.checksums import file_checksum @@ -41,7 +41,7 @@ def materialize_control(version: DatasetVersion) -> Path: digest = part.checksum.removeprefix("sha256:") cache_dir = settings().control_cache_dir - cache_dir.mkdir(parents=True, exist_ok=True) + make_cache_folder(cache_dir) destination = cache_dir / f"{digest}.duckdb" if destination.exists() and destination.stat().st_size == part.size: if file_checksum(destination) == part.checksum: diff --git a/packages/timenet/src/timenet/registry/__init__.py b/packages/timenet/src/timenet/registry/__init__.py index 4ebfb5825..3b6368c48 100644 --- a/packages/timenet/src/timenet/registry/__init__.py +++ b/packages/timenet/src/timenet/registry/__init__.py @@ -1,14 +1,10 @@ """Dataset registries: serve compiled TimeF versions and search over metadata.""" +from timenet._registry_uri import TIMENET_REGISTRY_URL, local_registry_path from timenet.registry._copy import copy_release from timenet.registry.base import BaseRegistry from timenet.registry.dependencies import open_local_version -from timenet.registry.factory import ( - TIMENET_REGISTRY_URL, - default_registry_path, - local_registry_path, - open_registry, -) +from timenet.registry.factory import open_registry from timenet.registry.local import LocalRegistry from timenet.registry.remote import RemoteRegistry from timenet.registry.s3 import S3Registry @@ -25,7 +21,6 @@ "S3Registry", "WritableRegistry", "copy_release", - "default_registry_path", "local_registry_path", "open_local_version", "open_registry", diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index 15fc05b99..1d57b575d 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -102,7 +102,7 @@ def _copy_one( # ruff: ignore[too-many-arguments] # A local registry is a folder that the copy reads or writes directly. if isinstance(dest, LocalRegistry): report = None if progress is None else _bytes_of(reference, manifest, progress) - directory = release_dir(dest.local_root, reference) + directory = release_dir(dest.root, reference) source.download_release(reference, directory, force=force, manifest=manifest, progress_cb=report) return if isinstance(source, LocalRegistry): diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index d6bd2c131..e87fa33d7 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -65,11 +65,6 @@ def check_writable(self, write: str) -> None: f"{self} has read_only=True, so it refuses {write}. Open it without read_only=True to write to it." ) - @property - @abstractmethod - def local_root(self) -> Path: - """The local registry or cache directory that holds prepared releases.""" - @property def requires_open_access(self) -> bool: """Whether stored or served releases must declare open access.""" diff --git a/packages/timenet/src/timenet/registry/dependencies.py b/packages/timenet/src/timenet/registry/dependencies.py index 42a7fd599..50d743bfb 100644 --- a/packages/timenet/src/timenet/registry/dependencies.py +++ b/packages/timenet/src/timenet/registry/dependencies.py @@ -25,7 +25,6 @@ from timenet.format.constants import MANIFEST_FILE from timenet.manifest import FilePart, Manifest, PayloadSource, read_manifest from timenet.registry._paths import release_dir -from timenet.registry.shared import complete_copy from timenet.registry.version import DatasetVersion from timenet.types.access import Access, effective_access from timenet.types.reference_keys import SourceKey @@ -322,33 +321,6 @@ def open_complete(ref: DatasetRef, *, requested: bool) -> DatasetVersion: return attach_payload_sources(version, lambda ref: open_complete(ref, requested=False), searched=lambda: str(root)) -def open_cached_or_remote_release( - root: Path, reference: DatasetRef, open_remote_version: Callable[[DatasetRef], DatasetVersion] -) -> DatasetVersion: - """Open a complete cached release or fall back to remote storage. - - Payload sources remain unattached. - - Args: - root: The cache directory to check before opening remote files. - reference: The exact dataset ID and version to open. - open_remote_version: Open one remote release without its payload sources. - - Returns: - A handle to the complete cached copy, or a remote handle if the cache is incomplete. - - Raises: - TimeFFormatError: If the manifest declares a different release. - """ # ruff: ignore[docstring-extraneous-exception] - raised by validate_manifest_reference - directory = release_dir(root, reference) - cached = _load_complete_local_manifest(directory, reference) - if cached is not None: - return DatasetVersion(cached, pafs.LocalFileSystem(), str(directory)) - version = open_remote_version(reference) - validate_manifest_reference(version.manifest, reference) - return version - - def _ensure_dataset_version_available_locally( # ruff: ignore[too-many-arguments] directory: Path, reference: DatasetRef, @@ -391,26 +363,6 @@ def _ensure_dataset_version_available_locally( # ruff: ignore[too-many-argument raise TimeNetDatasetNotFoundError(f"complete release {reference} is missing at {directory}") -def _load_complete_local_manifest( - directory: Path, - reference: DatasetRef, - *, - manifest: Manifest | None = None, -) -> Manifest | None: - """Load and validate a local release, returning None if files are missing or incomplete. - - Returns: - The manifest if it exists and every listed file has the recorded size, otherwise ``None``. - """ - manifest_path = directory / MANIFEST_FILE - if not manifest_path.is_file(): - return None - if manifest is None: - manifest = read_manifest(manifest_path) - validate_manifest_reference(manifest, reference) - return manifest if complete_copy(directory, manifest) else None - - def _download_dataset_version( directory: Path, reference: DatasetRef, diff --git a/packages/timenet/src/timenet/registry/factory.py b/packages/timenet/src/timenet/registry/factory.py index 117f5483d..d4b0af228 100644 --- a/packages/timenet/src/timenet/registry/factory.py +++ b/packages/timenet/src/timenet/registry/factory.py @@ -1,27 +1,15 @@ """The :func:`open_registry` factory: dispatch a URI to the right registry backend.""" from pathlib import Path -from urllib.parse import urlparse -from urllib.request import url2pathname -from timenet.config import settings -from timenet.errors import TimeNetRegistryError +from timenet._registry_uri import registry_location from timenet.registry.local import LocalRegistry from timenet.registry.remote import RemoteRegistry from timenet.registry.s3 import S3Registry from timenet.registry.writable import WritableRegistry -TIMENET_REGISTRY_URL = "https://registry.timenet.ai" -"""The hosted TimeNet registry that the ``timenet://`` scheme is an alias for.""" - -_REMOTE_SCHEMES = ("timenet://", "http://", "https://", "s3://") - - -_SCHEMES = "Use an http(s)://, s3://, timenet:// or file:// URL, or a local path." - - -def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None, label: str = "registry") -> WritableRegistry: +def open_registry(uri: str | Path, *, label: str = "registry") -> WritableRegistry: """Open a registry from a URI or path. The scheme selects the backend. ``http(s)://`` and ``timenet://`` open a :class:`RemoteRegistry`, @@ -30,8 +18,6 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None, label Args: uri: A URL, ``timenet://`` / ``s3://`` / ``file://`` URI, or local path. - cache_dir: Where a remote backend caches downloads. The local backend ignores it. Defaults to - the configured storage directory when ``None``. label: The name of the value in an error message, such as ``sources[0]``. Returns: @@ -41,65 +27,10 @@ def open_registry(uri: str | Path, *, cache_dir: str | Path | None = None, label TimeNetRegistryError: If ``uri`` is blank or carries a scheme no backend handles, if a ``timenet://`` URI carries a path, or if ``uri`` is a ``file://`` URI with a host component (the host would be dropped without notice) or without a path. - """ - text = str(uri) - if text.startswith("timenet://"): - # timenet:// is a bare alias for the hosted service root; it takes no path (dataset ids are - # passed to the client methods, not folded into the registry URI). Reject a stray remainder - # rather than drop it silently, so a mistyped URI fails loudly instead of hitting the default host. - if text[len("timenet://") :].strip("/"): - raise TimeNetRegistryError(f"{label} {text!r}: timenet:// takes no path") - return RemoteRegistry(TIMENET_REGISTRY_URL, cache_dir=cache_dir) - if text.startswith(("http://", "https://")): - return RemoteRegistry(text, cache_dir=cache_dir) - if text.startswith("s3://"): - return S3Registry(text, cache_dir=cache_dir) - return LocalRegistry(local_registry_path(text, label=label)) - - -def local_registry_path(uri: str | Path, *, label: str = "registry") -> Path: - """Convert a local registry URL or path to a directory path. - - Args: - uri: A ``file://`` URI or local path. - label: The name of the value in an error message, such as ``sources[0]``. - - Returns: - The local directory path with ``~`` expanded. - - Raises: - TimeNetRegistryError: If ``uri`` is blank, names a remote registry, or is not a valid local URL. - """ - text = str(uri) - if not text.strip(): - # Path("") is the current directory, which a blank value never means. - raise TimeNetRegistryError(f"{label} is blank; give a registry URL or a local path") - if text.startswith(_REMOTE_SCHEMES): - raise TimeNetRegistryError(f"{label} {text!r} is remote; build writes to a local directory") - if text.startswith("file://"): - parsed = urlparse(text) - if parsed.netloc: - raise TimeNetRegistryError(f"{label} {text!r} must be an absolute file:// URI (three slashes). {_SCHEMES}") - if not parsed.path: - raise TimeNetRegistryError(f"{label} {text!r} must name an absolute path") - return Path(url2pathname(parsed.path)).expanduser() - if "://" in text: - raise TimeNetRegistryError(f"{label} {text!r} has an unsupported registry scheme. {_SCHEMES}") - return Path(text).expanduser() - - -def default_registry_path() -> Path: - """Resolve the local registry directory a build writes to (and the SDK reads from) by default. - - If ``$TIMENET_REGISTRY`` names a local directory, use it. Otherwise use the default - ``/registry``. The build CLI and the ``timenet_connectors`` build and load helpers - all call this function, so producer and consumer agree on where a dataset lands. If - ``$TIMENET_REGISTRY`` names a remote registry, this function propagates the - :class:`~timenet.errors.TimeNetRegistryError` from :func:`local_registry_path`, because a build cannot - write to a remote registry. - - Returns: - The local registry directory. - """ - cfg = settings() - return cfg.registry_path if cfg.registry is None else local_registry_path(cfg.registry) + """ # ruff: ignore[docstring-extraneous-exception] (raised by registry_location) + location = registry_location(uri, label=label) + if isinstance(location, Path): + return LocalRegistry(location) + if location.scheme == "s3": + return S3Registry(location.url) + return RemoteRegistry(location.url) diff --git a/packages/timenet/src/timenet/registry/lazy_read.py b/packages/timenet/src/timenet/registry/lazy_read.py index 32654c6f6..deadd053c 100644 --- a/packages/timenet/src/timenet/registry/lazy_read.py +++ b/packages/timenet/src/timenet/registry/lazy_read.py @@ -11,7 +11,7 @@ from timenet.errors import TimeFValidationError from timenet.manifest import Manifest, PayloadSource from timenet.registry._paths import release_dir -from timenet.registry.dependencies import open_cached_or_remote_release +from timenet.registry.dependencies import validate_manifest_reference from timenet.registry.version import DatasetVersion from timenet.registry.writable import WritableRegistry from timenet.types.references import DatasetRef @@ -19,7 +19,7 @@ class LazyReadRegistry(WritableRegistry, ABC): - """A registry that reads remote files on demand and reuses complete cached releases. + """A registry that reads remote files on demand. Backends differ only in how one remote release is bound to a filesystem, so :meth:`_open_remote_version` is the one method they implement for reading. @@ -84,18 +84,22 @@ def put_dataset( shutil.rmtree(staging, ignore_errors=True) def open_release(self, reference: DatasetRef, *, manifest: Manifest | None = None) -> DatasetVersion: - """Open a cached or remote release without attaching its payload sources. + """Open a remote release without attaching its payload sources. Args: reference: The dataset ID and exact version to open. manifest: A manifest already read from this registry, or ``None`` to fetch it. Returns: - A handle to a complete copy in :attr:`local_root`, or to remote files read on demand. - """ - return open_cached_or_remote_release( - self.local_root, reference, lambda ref: self._open_remote_version(ref, manifest=manifest) - ) + A handle to the remote files of the release. + + Raises: + TimeFFormatError: If the manifest declares a different release. + """ # ruff: ignore[docstring-extraneous-exception] - raised by validate_manifest_reference + if manifest is None: + manifest = self.get_manifest(reference.dataset_id, str(reference.version)) + validate_manifest_reference(manifest, reference) + return self._open_remote_version(reference, manifest=manifest) @abstractmethod def _open_remote_version(self, reference: DatasetRef, *, manifest: Manifest | None = None) -> DatasetVersion: diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index c28acec6d..b84d62616 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -7,7 +7,7 @@ import pyarrow.fs as pafs -from timenet._fs import same_directory, staged_directory +from timenet._fs import delete_and_measure, same_directory, staged_directory from timenet.dataset import TimeFDataset from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetDatasetNotFoundError, TimeNetFileError from timenet.format.constants import MANIFEST_FILE @@ -48,11 +48,6 @@ def root(self) -> Path: """The directory this registry reads from and writes to.""" return self._root - @property - def local_root(self) -> Path: - """The directory that holds complete local releases.""" - return self._root - @property def requires_open_access(self) -> bool: """Whether local releases must declare open access. @@ -65,30 +60,58 @@ def requires_open_access(self) -> bool: def list_datasets(self) -> list[DatasetMetadata]: """Return the latest-version metadata of every dataset, sorted by id. - Search at any depth. This finds both flat (``hello_world``) and namespaced (``org/name``) - layouts. A dataset id is the path from the root to a version directory's parent. - Returns: One :class:`~timenet.types.DatasetMetadata` per dataset. """ + latest: dict[str, Version] = {} + for release in self.releases(): + # The releases come sorted by version, so the last one of a dataset is its latest. + latest[release.dataset_id] = release.version + return [self.get_manifest(dataset_id, str(version)).metadata for dataset_id, version in latest.items()] + + def releases(self) -> list[DatasetRef]: + """List the committed releases at any depth below the root. + + A release is a ``//manifest.json`` outside hidden folders and unfinished writes. + + Returns: + The releases, sorted by dataset ID and version. + """ if not self._root.is_dir(): return [] - latest_versions: dict[str, str] = {} + found: list[DatasetRef] = [] for manifest_path in self._root.rglob(MANIFEST_FILE): - version_dir = manifest_path.parent - parts = version_dir.relative_to(self._root).parts - if any(part.startswith(".") or ".tmp-" in part for part in parts): + parts = manifest_path.parent.relative_to(self._root).parts + # A release needs a dataset ID before its version, so a manifest at the root or one level below is none. + if not parts[:-1] or any(part.startswith(".") or ".tmp-" in part for part in parts): continue - dataset_id = "/".join(parts[:-1]) - version = parts[-1] - if not dataset_id or not _is_version(version): - continue - current = latest_versions.get(dataset_id) - if current is None or Version.parse(version) > Version.parse(current): - latest_versions[dataset_id] = version - return [ - self.get_manifest(dataset_id, version).metadata for dataset_id, version in sorted(latest_versions.items()) - ] + try: + found.append(DatasetRef.parse(f"{'/'.join(parts[:-1])}@{parts[-1]}")) + except TimeFValidationError: + continue # for example a directory whose name is not a version + return sorted(found, key=lambda release: (release.dataset_id, release.version)) + + def remove_release(self, release: DatasetRef) -> int: + """Delete a release and the dataset folders that this leaves empty. + + Args: + release: The release to delete. + + Returns: + The bytes freed. + + Raises: + TimeNetRegistryError: If the registry is read-only. + OSError: If the release folder cannot be removed completely. + """ # ruff: ignore[docstring-extraneous-exception] (raised by check_writable and delete_and_measure) + self.check_writable(f"a removal of {release}") + directory = release_dir(self._root, release) + freed = delete_and_measure(directory) + parent = directory.parent + while self._root in parent.parents and not any(parent.iterdir()): + parent.rmdir() + parent = parent.parent + return freed def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest: """Return a dataset's manifest (latest version if unspecified). diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index 692e3d8a7..e439268e3 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -32,40 +32,32 @@ class RemoteRegistry(LazyReadRegistry): - """A dataset registry backed by the hosted TimeNet HTTP service.""" + """A TimeNet registry that reads and publishes datasets over HTTP(S).""" def __init__( self, base_url: str, *, token: str | None = None, - cache_dir: str | Path | None = None, transport: httpx.BaseTransport | None = None, read_only: bool = False, ) -> None: - """Open a remote registry. + """Configure a connection to a remote TimeNet registry. Args: - base_url: The service root, e.g. ``https://registry.timenet.ai``. - token: Bearer token. Defaults to ``$TIMENET_TOKEN`` (``None`` is anonymous). - cache_dir: Where downloads are cached; defaults to the configured storage directory. - transport: An httpx transport for testing; ``None`` uses the network. + base_url: The service root URL, such as ``https://registry.timenet.ai``. + token: Bearer token, or ``None`` for ``TIMENET_TOKEN`` with anonymous access when unset. + transport: Custom HTTP transport, or ``None`` to use the network. read_only: Refuse every write to this registry. """ cfg = settings() self._read_only = read_only self._base_url = base_url self._http = RegistryHttpClient(base_url, token=token if token is not None else cfg.token, transport=transport) - self._cache_dir = Path(cache_dir) if cache_dir is not None else cfg.storage_dir def __str__(self) -> str: return f"registry {self._base_url}" - @property - def local_root(self) -> Path: - """The directory that holds complete local releases.""" - return self._cache_dir - def list_datasets(self) -> list[DatasetMetadata]: """Return the latest-version metadata of every dataset, sorted by id. diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 1ea8a46fe..d2d0b338b 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -4,9 +4,9 @@ matches the hosted registry's bucket layout. The manifest's relative file paths do not change. Only the key prefix differs from a :class:`~timenet.registry.LocalRegistry`. Credentials, region, and an optional endpoint override come from boto3's default session (``AWS_*`` vars, ``AWS_PROFILE``, -``AWS_ENDPOINT_URL``). Opening a version reads remote payloads on demand. Complete cached releases -are reused. The backend has no catalog, so listing and search are unsupported. Use it to publish and -to fetch by an explicit ``org/name@version``. +``AWS_ENDPOINT_URL``). Opening a version reads remote payloads on demand. The backend has no +catalog, so listing and search are unsupported. Use it to publish and to fetch by an explicit +``org/name@version``. boto3 is optional. Install the ``timenet[s3]`` extra to use this backend. """ @@ -21,7 +21,6 @@ import pyarrow.fs as pafs -from timenet.config import settings from timenet.errors import TimeNetDatasetNotFoundError, TimeNetRegistryError from timenet.format.constants import MANIFEST_FILE from timenet.manifest import Manifest, parse_manifest @@ -38,37 +37,30 @@ class S3Registry(LazyReadRegistry): - """A registry that reads and writes datasets under an ``s3://bucket/prefix`` root.""" + """A registry that reads and publishes datasets under an S3 bucket or prefix.""" _staging_prefix = "timenet-s3-" - def __init__(self, uri: str, *, cache_dir: str | Path | None = None, read_only: bool = False) -> None: - """Open an S3-backed registry. + def __init__(self, uri: str, *, read_only: bool = False) -> None: + """Configure an S3 registry. Args: - uri: The bucket or prefix root, for example ``s3://my-bucket/registry``. - cache_dir: Where downloads are cached; defaults to the configured storage directory. + uri: The bucket or prefix URL, such as ``s3://my-bucket/registry``. read_only: Refuse every write to this registry. Raises: - TimeNetRegistryError: If ``uri`` is not an ``s3://bucket[/prefix]`` URL. + TimeNetRegistryError: If the URL lacks an ``s3`` scheme or bucket name. """ parsed = urlparse(uri) if parsed.scheme != "s3" or not parsed.netloc: raise TimeNetRegistryError(f"S3 registry URI must be s3://bucket[/prefix], got {uri!r}") self._bucket = parsed.netloc self._prefix = parsed.path.strip("/") - self._cache_dir = Path(cache_dir) if cache_dir is not None else settings().storage_dir self._read_only = read_only def __str__(self) -> str: return f"S3 registry {self._display_root()}" - @property - def local_root(self) -> Path: - """The directory that holds complete local releases.""" - return self._cache_dir - def list_datasets(self) -> list[DatasetMetadata]: """Unsupported: the S3 backend has no catalog. diff --git a/packages/timenet/tests/conftest.py b/packages/timenet/tests/conftest.py index b7f785f6f..42a982017 100644 --- a/packages/timenet/tests/conftest.py +++ b/packages/timenet/tests/conftest.py @@ -2,6 +2,7 @@ Registers Hypothesis profiles so the example count can be tuned per environment via the ``HYPOTHESIS_PROFILE`` environment variable (``dev`` for a fast edit loop, ``ci`` for the full run). +Also gives the tests a fake home folder, and a way to skip on a volume that tells case apart. """ import os @@ -29,6 +30,17 @@ settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "ci")) +@pytest.fixture +def home(monkeypatch, tmp_path): + """Point ``HOME`` and ``Path.home()`` at an empty folder, so that no test looks at the real home.""" + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setattr(Path, "home", staticmethod(lambda: home)) + assert Path("~").expanduser() == Path.home() == home + return home + + @pytest.fixture(autouse=True) def _no_installed_connectors(monkeypatch): """Install no builder backends, so that no test reads the connector packages of the environment.""" diff --git a/packages/timenet/tests/coordinator/test_build.py b/packages/timenet/tests/coordinator/test_build.py index fc866963a..4236a1a74 100644 --- a/packages/timenet/tests/coordinator/test_build.py +++ b/packages/timenet/tests/coordinator/test_build.py @@ -1,5 +1,6 @@ import shutil +from _fake_registry import build_service_fake import pytest from timenet import builders @@ -10,7 +11,7 @@ from timenet.errors import TimeNetAccessError, TimeNetRegistryError, TimeNetVersionUnavailableError from timenet.progress import BytesMoved, StepFinished, StepStarted from timenet.reader import TimeFReader -from timenet.registry import LocalRegistry, open_local_version +from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version from timenet.registry._paths import release_dir from timenet.sources import InstalledConnectors from timenet.testing import install_backends, make_dataset @@ -211,6 +212,22 @@ def test_a_forced_build_forces_only_the_requested_release(store, fake_parent, fa assert fake_child.calls == [(CHILD_REF, True, {})] +def test_a_build_takes_a_parent_that_the_store_holds_and_asks_a_later_registry_nothing( + tmp_path, store, fake_parent, fake_child +): + store.put_dataset(make_dataset()) + origin = LocalRegistry(tmp_path / "origin") + origin.put_dataset(make_dataset()) + transport, requests = build_service_fake(origin.root) + remote = RemoteRegistry("http://api.local", transport=transport) + + plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[store, remote]) + run_plan(plan, store) + + assert plan.steps == (_build(CHILD_REF),) + assert requests == [] + + def test_a_build_skips_a_release_the_store_holds(store, fake): store.put_dataset(_demo(_card(REF_AB))) assert plan_release_build("a/b", store=store).steps == () diff --git a/packages/timenet/tests/coordinator/test_read.py b/packages/timenet/tests/coordinator/test_read.py index 15324741e..fb3f8ef89 100644 --- a/packages/timenet/tests/coordinator/test_read.py +++ b/packages/timenet/tests/coordinator/test_read.py @@ -127,7 +127,7 @@ def _gated_remote(tmp_path) -> RemoteRegistry: origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(_demo(make_dataset().metadata.model_copy(update=_GATED))) transport, _ = build_fake(release_dir(origin.root, DEMO)) - return RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + return RemoteRegistry("http://api.local", transport=transport) def _offline(request): @@ -305,7 +305,7 @@ def test_a_read_without_a_version_stops_at_the_first_source_that_serves_it(tmp_p store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) # A later registry cannot be reached, and the installed connector builds a newer version. - down = RemoteRegistry("http://down.local", transport=httpx.MockTransport(_offline), cache_dir=tmp_path / "c") + down = RemoteRegistry("http://down.local", transport=httpx.MockTransport(_offline)) calls = _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) version = read_release(DEMO.dataset_id, None, sources=[store, down, InstalledConnectors()], store=store) @@ -451,7 +451,7 @@ def test_two_readers_that_pin_one_file_differently_fail_the_read_before_any_step def test_a_remote_read_fetches_each_manifest_once(tmp_path, monkeypatch): origin, _ = _child_of_the_demo(tmp_path / "origin") transport, _ = build_service_fake(origin.root) - remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + remote = RemoteRegistry("http://api.local", transport=transport) reads = _count_reads(monkeypatch, remote) version = read_release(CHILD.dataset_id, None, sources=[remote], store=None) @@ -556,7 +556,7 @@ def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatc for name, dataset in (("first", make_dataset()), ("second", _demo(_card(newer)))): LocalRegistry(tmp_path / name).put_dataset(dataset) transport, _ = build_service_fake(tmp_path / name) - registries.append(RemoteRegistry(f"http://{name}.local", transport=transport, cache_dir=tmp_path / f"{name}-c")) + registries.append(RemoteRegistry(f"http://{name}.local", transport=transport)) reads = _count_reads(monkeypatch, registries[0]) version = read_release(DEMO.dataset_id, "latest", sources=registries, store=None) diff --git a/packages/timenet/tests/format/test_control_cache.py b/packages/timenet/tests/format/test_control_cache.py index d9994e7a3..18746c4ac 100644 --- a/packages/timenet/tests/format/test_control_cache.py +++ b/packages/timenet/tests/format/test_control_cache.py @@ -1,5 +1,6 @@ import pyarrow.fs as pafs +from timenet.config import is_cache_folder, settings from timenet.format.checksums import file_checksum from timenet.format.control_cache import materialize_control from timenet.format.control_writer import DuckDBControlWriter @@ -30,3 +31,24 @@ def test_local_control_database_uses_its_existing_path(tmp_path): ) assert materialize_control(version) == path + + +def test_a_remote_control_database_lands_in_a_control_folder_with_the_marker(tmp_path): + path = tmp_path / "remote" / "control.duckdb" + path.parent.mkdir() + dataset = _dataset() + DuckDBControlWriter(path, external=PackedValueWriter(path.parent, StorageSources())).write_hierarchy(dataset) + manifest = Manifest( + dataset_id=dataset.metadata.dataset_id, + metadata=dataset.metadata, + files=ManifestFiles( + control=(FilePart(path=path.name, checksum=file_checksum(path), size=path.stat().st_size),), + ), + timef_format_version=1, + ) + # Any file system but the local one counts as remote, so the database is copied into the cache. + remote = pafs.SubTreeFileSystem(str(path.parent), pafs.LocalFileSystem()) + copy = materialize_control(DatasetVersion(manifest=manifest, filesystem=remote, root=".")) + assert copy.parent == settings().control_cache_dir + assert copy.read_bytes() == path.read_bytes() + assert is_cache_folder(copy.parent) diff --git a/packages/timenet/tests/reader/test_reader.py b/packages/timenet/tests/reader/test_reader.py index e7a0559e7..44beb7a20 100644 --- a/packages/timenet/tests/reader/test_reader.py +++ b/packages/timenet/tests/reader/test_reader.py @@ -16,6 +16,7 @@ import pytest from timenet.client import TimeNet +from timenet.config import settings from timenet.dataset import Record, Signal, Source, TimeFDataset from timenet.dataset.axis import RegularAxis from timenet.errors import TimeFFormatError, TimeFReaderError, TimeFValidationError, TimeNetFileError @@ -898,8 +899,6 @@ def failing(connection): def test_a_reader_verifies_a_remote_catalog_once_and_again_after_its_copy_is_gone(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.delenv("TIMENET_CACHE", raising=False) version_dir = _write(tmp_path) local = open_local_version(version_dir) # A filesystem other than the local one makes the reader fetch the catalog into its cache. @@ -923,6 +922,6 @@ def counting_checksum(path): assert len(checks) == 1 # The cached copy disappears while the reader is closed; the reopen fetches and checks it again. reader.close() - shutil.rmtree(tmp_path / "home" / "cache" / "control") + shutil.rmtree(settings().control_cache_dir) assert reader.record_ids() == expected assert len(checks) == 2 diff --git a/packages/timenet/tests/registry/test_conformance.py b/packages/timenet/tests/registry/test_conformance.py index a02a657a3..5354040b9 100644 --- a/packages/timenet/tests/registry/test_conformance.py +++ b/packages/timenet/tests/registry/test_conformance.py @@ -101,7 +101,7 @@ def _s3_registry(request, tmp_path, monkeypatch) -> S3Registry: # server instances, so a fixed name would leak these datasets into test_s3.py's own tn-test bucket. bucket = f"tn-conformance-{uuid.uuid4().hex[:12]}" boto3.client("s3", endpoint_url=endpoint).create_bucket(Bucket=bucket) - return S3Registry(f"s3://{bucket}/registry", cache_dir=tmp_path / "cache") + return S3Registry(f"s3://{bucket}/registry") @pytest.fixture(params=["local", "s3", "remote"]) @@ -119,7 +119,7 @@ def backend(request, tmp_path, monkeypatch) -> Backend: service_root = tmp_path / "service" service_root.mkdir() transport, _ = build_service_fake(service_root, token="tok_rw") - registry = RemoteRegistry("http://svc.local", token="tok_rw", cache_dir=tmp_path / "cache", transport=transport) + registry = RemoteRegistry("http://svc.local", token="tok_rw", transport=transport) supports_listing = True else: registry = _s3_registry(request, tmp_path, monkeypatch) diff --git a/packages/timenet/tests/registry/test_copy.py b/packages/timenet/tests/registry/test_copy.py index bb52830e4..37b1e2628 100644 --- a/packages/timenet/tests/registry/test_copy.py +++ b/packages/timenet/tests/registry/test_copy.py @@ -144,9 +144,7 @@ def _remote(tmp_path: Path, name: str, *, token: str | None = None) -> tuple[Rem root = tmp_path / name root.mkdir(exist_ok=True) transport, _ = build_service_fake(root, token=token) - registry = RemoteRegistry( - f"http://{name}.local", token=token, cache_dir=tmp_path / f"{name}-cache", transport=transport - ) + registry = RemoteRegistry(f"http://{name}.local", token=token, transport=transport) return registry, root diff --git a/packages/timenet/tests/registry/test_factory.py b/packages/timenet/tests/registry/test_factory.py index 07a6625b7..45cb3febb 100644 --- a/packages/timenet/tests/registry/test_factory.py +++ b/packages/timenet/tests/registry/test_factory.py @@ -1,24 +1,7 @@ import pytest -from timenet.errors import TimeNetDatasetNotFoundError, TimeNetRegistryError -from timenet.registry import LocalRegistry, default_registry_path - - -def test_default_registry_path_falls_back_to_home(monkeypatch, tmp_path): - monkeypatch.delenv("TIMENET_REGISTRY", raising=False) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path)) - assert default_registry_path() == tmp_path / "registry" - - -def test_default_registry_path_honors_local_registry(monkeypatch, tmp_path): - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "reg")) - assert default_registry_path() == tmp_path / "reg" - - -def test_default_registry_path_rejects_remote(monkeypatch): - monkeypatch.setenv("TIMENET_REGISTRY", "timenet://") - with pytest.raises(TimeNetRegistryError): - default_registry_path() +from timenet.errors import TimeNetDatasetNotFoundError +from timenet.registry import LocalRegistry def test_unknown_dataset_error_is_actionable(tmp_path): diff --git a/packages/timenet/tests/registry/test_registry.py b/packages/timenet/tests/registry/test_registry.py index d7eeda740..625ff104b 100644 --- a/packages/timenet/tests/registry/test_registry.py +++ b/packages/timenet/tests/registry/test_registry.py @@ -113,7 +113,7 @@ def test_local_registry_path_expands_user(monkeypatch, tmp_path): "uri", ["timenet://", "timenet://hello/world", "http://reg.example", "https://reg.example", "s3://bucket/reg"] ) def test_local_registry_path_rejects_remote(uri): - with pytest.raises(TimeNetRegistryError, match="remote"): + with pytest.raises(TimeNetRegistryError, match="is remote; the local registry needs a local directory"): local_registry_path(uri) @@ -351,3 +351,11 @@ def test_a_read_only_registry_refuses_put_dataset_and_a_copy_into_it(tmp_path): with pytest.raises(TimeNetRegistryError, match=refused + r"a copy of timenet/hello-world@1\.0\.0\."): copy_release(release, origin, registry) assert not registry.root.exists() + + +def test_a_read_only_registry_refuses_to_remove_a_release(tmp_path): + release = LocalRegistry(tmp_path).put_dataset(make_dataset()) + registry = LocalRegistry(tmp_path, read_only=True) + with pytest.raises(TimeNetRegistryError, match=r"refuses a removal of timenet/hello-world@1\.0\.0\. "): + registry.remove_release(release) + assert registry.exists(release.dataset_id, str(release.version)) diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 6a4337c35..638f317cb 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -3,6 +3,7 @@ import pickle from unittest.mock import AsyncMock, call +from _demo_child import _child_of_the_demo from _fake_registry import build_fake, build_publish_fake, range_server, read_file, registry_server import httpx import numpy as np @@ -46,7 +47,7 @@ def version_dir(tmp_path) -> tuple[Path, Manifest]: def _remote(version_dir, tmp_path, **kw): directory, _ = version_dir transport, requests = build_fake(directory, token=kw.pop("token", None)) - registry = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache", **kw) + registry = RemoteRegistry("http://api.local", transport=transport, **kw) return registry, requests @@ -68,7 +69,7 @@ def handler(request): assert request.url.path == "/api/v1/datasets" return httpx.Response(200, json={"datasets": [{"dataset_id": 3}]}) - registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=tmp_path) + registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler)) with pytest.raises(ValidationError, match=r"datasets\.0\.dataset_id"): registry.list_datasets() @@ -77,7 +78,6 @@ def test_list_datasets_exposes_pydantic_json_errors(tmp_path): registry = RemoteRegistry( "http://api.local", transport=httpx.MockTransport(lambda _request: httpx.Response(200, content=b"not json")), - cache_dir=tmp_path, ) with pytest.raises(ValidationError, match="Invalid JSON"): registry.list_datasets() @@ -104,7 +104,6 @@ def handler(request): registry = RemoteRegistry( "http://api.local", transport=httpx.MockTransport(handler), - cache_dir=tmp_path, ) metadata = registry.list_datasets()[0] assert metadata.license is License.OTHER @@ -161,7 +160,6 @@ def handler(request): registry = RemoteRegistry( "http://api.local", transport=httpx.MockTransport(handler), - cache_dir=tmp_path, ) assert registry.list_datasets() == [expected] assert paths == [ @@ -229,7 +227,7 @@ def handler(request): sleep = AsyncMock() monkeypatch.setattr("timenet.registry.remote._http.asyncio.sleep", sleep) - registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=tmp_path / "cache") + registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler)) release = DatasetRef(dataset_id=manifest.metadata.dataset_id, version=manifest.metadata.dataset_version) dest = tmp_path / "out" / release.dataset_id / str(release.version) @@ -252,7 +250,7 @@ def handler(request): return transport.handle_request(request) monkeypatch.setattr("timenet.registry.remote._http.asyncio.sleep", AsyncMock()) - registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=tmp_path / "cache") + registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler)) release = DatasetRef(dataset_id=manifest.metadata.dataset_id, version=manifest.metadata.dataset_version) dest = tmp_path / "out" / release.dataset_id / str(release.version) @@ -310,7 +308,7 @@ def test_a_remote_download_whose_size_differs_from_its_record_names_both_sizes(v def test_load_full_round_trips(version_dir, tmp_path, entry_point): directory, manifest = version_dir with registry_server(directory) as server: - registry = RemoteRegistry(server.url, cache_dir=tmp_path / "cache") + registry = RemoteRegistry(server.url) client = TimeNet(sources=[registry]) if entry_point == "open": with client.open(manifest.metadata.dataset_id) as reader: @@ -335,7 +333,7 @@ def single_use_download(request): return read_file(directory, request) with registry_server(directory, serve_file=single_use_download if expire_urls else None) as server: - registry = RemoteRegistry(server.url, cache_dir=tmp_path / "cache") + registry = RemoteRegistry(server.url) client = TimeNet(sources=[registry]) with client.open(manifest.metadata.dataset_id) as reader: # A worker must reopen its own clients, including after the main process has read values. @@ -369,7 +367,7 @@ def serve_file(request): return httpx.Response(200, content=(directory / request.url.path.lstrip("/")).read_bytes()) with registry_server(directory, serve_file=serve_file) as server: - remote = RemoteRegistry(server.url, token="private-token", cache_dir=tmp_path / "cache") + remote = RemoteRegistry(server.url, token="private-token") with TimeNet(sources=[remote]).open(dataset.metadata.dataset_id, "1.0.0") as reader: assert all(response.request.url.path.startswith("/api/v1/") for response in server.responses) if not support_ranges: @@ -394,7 +392,7 @@ def serve_file(request): def test_put_dataset_publishes_uploads_and_finalizes(tmp_path): transport, state = build_publish_fake(token="tok_rw") - registry = RemoteRegistry("http://api.local", token="tok_rw", transport=transport, cache_dir=tmp_path) + registry = RemoteRegistry("http://api.local", token="tok_rw", transport=transport) dataset = make_dataset() assert registry.put_dataset(dataset) == DatasetRef.parse(f"{dataset.metadata.dataset_id}@1.0.0") assert state["finalized"] is True @@ -404,7 +402,7 @@ def test_put_dataset_publishes_uploads_and_finalizes(tmp_path): def test_put_dataset_refuses_force_and_uploads_nothing(tmp_path): transport, state = build_publish_fake(token="tok_rw") - registry = RemoteRegistry("http://api.local", token="tok_rw", transport=transport, cache_dir=tmp_path) + registry = RemoteRegistry("http://api.local", token="tok_rw", transport=transport) with pytest.raises(TimeFValidationError, match="never writes a published version again"): registry.put_dataset(make_dataset(), force=True) assert state["manifest"] is None @@ -415,7 +413,7 @@ def test_put_dataset_refuses_a_payload_source_the_service_does_not_hold_before_p local = LocalRegistry(tmp_path / "local") local.put_dataset(make_dataset()) transport, state = build_publish_fake(token="tok_rw") - registry = RemoteRegistry("http://api.local", token="tok_rw", transport=transport, cache_dir=tmp_path / "cache") + registry = RemoteRegistry("http://api.local", token="tok_rw", transport=transport) with TimeFReader(open_local_version(local.root / "timenet/hello-world/1.0.0")) as reader: parent = reader.read() child = TimeFDataset.from_parts( @@ -446,9 +444,7 @@ def handler(request): return httpx.Response(200, json={"files": []}) return httpx.Response(200, json={}) - registry = RemoteRegistry( - "http://api.local", token="tok_rw", transport=httpx.MockTransport(handler), cache_dir=tmp_path - ) + registry = RemoteRegistry("http://api.local", token="tok_rw", transport=httpx.MockTransport(handler)) with pytest.raises(TimeNetRegistryError, match="disagrees with the manifest"): registry.put_dataset(make_dataset()) @@ -462,9 +458,7 @@ def handler(request): return httpx.Response(200, json={"files": ["../escape"]}) return httpx.Response(200, json={}) - registry = RemoteRegistry( - "http://api.local", token="tok_rw", transport=httpx.MockTransport(handler), cache_dir=tmp_path - ) + registry = RemoteRegistry("http://api.local", token="tok_rw", transport=httpx.MockTransport(handler)) with pytest.raises(ValidationError, match="stay within the dataset root"): registry.put_dataset(make_dataset()) @@ -482,9 +476,7 @@ def handler(request): return httpx.Response(200, json={"url": "not-a-url", "headers": []}) return httpx.Response(200, json={}) - registry = RemoteRegistry( - "http://api.local", token="tok_rw", transport=httpx.MockTransport(handler), cache_dir=tmp_path - ) + registry = RemoteRegistry("http://api.local", token="tok_rw", transport=httpx.MockTransport(handler)) with pytest.raises(ValidationError, match="headers"): registry.put_dataset(make_dataset()) @@ -494,27 +486,41 @@ def handler(request): assert request.url.path == "/api/v1/datasets/demo/example" return httpx.Response(200, json={"version": 3}) - registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=tmp_path) + registry = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler)) with pytest.raises(ValidationError, match="version"): registry.get_manifest("demo/example") -def test_a_registry_url_in_the_sources_caches_under_the_storage_directory(tmp_path, monkeypatch): - # A remote registry that the sources name by URL reuses the complete copies under the storage - # directory. - monkeypatch.setenv("TIMENET_STORAGE", str(tmp_path / "store")) - (registry,) = TimeNet(sources=["https://registry.example.test"])._sources - assert isinstance(registry, RemoteRegistry) - assert registry._cache_dir == tmp_path / "store" - - def test_put_dataset_without_write_token_is_rejected(tmp_path): transport, _ = build_publish_fake(token="tok_rw") - registry = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path) # anonymous + registry = RemoteRegistry("http://api.local", transport=transport) # anonymous with pytest.raises(TimeNetRegistryError): registry.put_dataset(make_dataset()) +def _remote_over(root: Path, blob_url: str, requests: list[str]) -> RemoteRegistry: + """Serve the releases under ``root`` through the registry API, with the files at ``blob_url``. + + Each request path goes into ``requests``. + """ + + def handler(request): + path = request.url.path + requests.append(path) + if request.url.host == "127.0.0.1": + return httpx.Response(200, content=(root / path.lstrip("/")).read_bytes()) + relative = path.removeprefix("/api/v1/datasets/") + if relative.endswith("/manifest"): + return httpx.Response( + 200, content=(root / relative.removesuffix("manifest") / "manifest.json").read_bytes() + ) + if "/download/" in relative: + return httpx.Response(307, headers={"location": blob_url + "/" + relative.replace("/download/", "/")}) + return httpx.Response(404) + + return RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler)) + + @pytest.mark.parametrize("backend", ["parquet", "zarr"]) def test_child_streams_physical_sources_and_downloads_for_offline_reads(tmp_path, backend): origin = tmp_path / "origin" @@ -535,35 +541,22 @@ def test_child_streams_physical_sources_and_downloads_for_offline_reads(tmp_path expected = parent.records[0].signals[0].to_arrow() signal_id = imported.records[parent.records[0].id].signals[0].id local.put_dataset(child) - requests = [] - - def handler(request): - path = request.url.path - requests.append(path) - if request.url.host == "127.0.0.1": - return httpx.Response(200, content=(origin / path.lstrip("/")).read_bytes()) - relative = path.removeprefix("/api/v1/datasets/") - if relative.endswith("/manifest"): - return httpx.Response( - 200, content=(origin / relative.removesuffix("manifest") / "manifest.json").read_bytes() - ) - if "/download/" in relative: - return httpx.Response(307, headers={"location": blob_url + "/" + relative.replace("/download/", "/")}) - return httpx.Response(404) - + requests: list[str] = [] cache = tmp_path / "cache" - remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(handler), cache_dir=cache) with range_server(origin) as blob_url: + remote = _remote_over(origin, blob_url, requests) with TimeNet(sources=[remote]).open("test/child", "1.0.0") as reader: assert all("/download/" not in path for path in requests) assert reader.get(Signal, signal_id).to_arrow().equals(expected) assert any(parent_id + "/1.0.0/download/" in path for path in requests) assert not any(parent_id + "/1.0.0/download/control.duckdb" in path for path in requests) - assert not (cache / parent_id / "1.0.0").exists() copy_release(DatasetRef.parse("test/child@1.0.0"), remote, LocalRegistry(cache)) - with TimeNet(sources=[remote]).open("test/child", "1.0.0") as reader: - assert reader.get(Signal, signal_id).to_arrow().equals(expected) assert not any("test/build-only" in path for path in requests) + requests.clear() + # The local registry holds the child and its payload source, so the read asks the remote registry nothing. + with TimeNet(sources=[LocalRegistry(cache), remote]).open("test/child", "1.0.0") as reader: + assert reader.get(Signal, signal_id).to_arrow().equals(expected) + assert requests == [] assert (cache / parent_id / "1.0.0/control.duckdb").exists() with TimeFReader(open_local_version(cache / "test/child/1.0.0")) as reader: reader.verify() @@ -577,6 +570,21 @@ def handler(request): assert requests == prepared_requests +def test_a_remote_read_takes_a_payload_source_that_the_local_registry_holds_from_it(tmp_path): + origin, demo = _child_of_the_demo(tmp_path / "origin") + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + requests: list[str] = [] + with range_server(origin.root) as blob_url: + remote = _remote_over(origin.root, blob_url, requests) + with TimeNet(sources=[store, remote]).open("test/child", "1.0.0") as reader: + signal = reader.read().records[0].signals[0] + assert signal.to_arrow().equals(make_dataset().records[0].signals[0].to_arrow()) + # The child comes from the remote registry, and the files of its payload source from the local registry. + assert any("test/child/1.0.0/download/" in path for path in requests) + assert not [path for path in requests if demo.dataset_id in path] + + def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: """Write a release that is not open into ``origin``; return its ID and the release whose card is credentialed.""" dataset = make_dataset() @@ -599,7 +607,7 @@ def _gated_release(origin: Path, page: str) -> tuple[str, DatasetRef]: def test_a_read_or_a_download_checks_access_before_it_reads_from_a_remote_registry(tmp_path, entry_point): dataset_id, release = _gated_release(tmp_path / "origin", "ancestor") transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) - remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + remote = RemoteRegistry("http://api.local", transport=transport) client = TimeNet(sources=[LocalRegistry(tmp_path / "local"), remote]) with pytest.raises(TimeNetAccessError) as caught: @@ -618,7 +626,7 @@ def test_a_remote_registry_lists_but_does_not_serve_a_release_that_is_not_open(t # The card of the release is open. A card in its lineage is not. dataset_id, release = _gated_release(tmp_path / "origin", "ancestor") transport, requests = build_fake(next((tmp_path / "origin").rglob("manifest.json")).parent) - remote = RemoteRegistry("http://api.local", transport=transport, cache_dir=tmp_path / "cache") + remote = RemoteRegistry("http://api.local", transport=transport) requirement = AccessRequirement(release, Access.CREDENTIALED, "https://example.org/dua") answer = remote.resolve(dataset_id, None) @@ -651,7 +659,7 @@ def _offline(request): def test_an_unreachable_registry_raises_a_registry_error_that_names_it(tmp_path): - remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(_offline), cache_dir=tmp_path) + remote = RemoteRegistry("http://api.local", transport=httpx.MockTransport(_offline)) with pytest.raises(TimeNetRegistryError, match=r"^cannot reach http://api\.local: the network is down$"): remote.resolve("timenet/hello-world", None) diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index a63e3a3c3..49d89cca9 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -47,8 +47,8 @@ def s3_root(monkeypatch): @pytest.mark.parametrize("entry_point", ["open", "load"]) @pytest.mark.parametrize("backend", ["parquet", "zarr"]) -def test_put_dataset_and_remote_load_round_trip(s3_root, tmp_path, backend, entry_point): - registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") +def test_put_dataset_and_remote_load_round_trip(s3_root, backend, entry_point): + registry = S3Registry(s3_root) reference = registry.put_dataset(make_dataset(), values_backend=backend) assert reference == DatasetRef.parse(f"{make_dataset().metadata.dataset_id}@1.0.0") client = TimeNet(sources=[registry]) @@ -62,14 +62,13 @@ def test_put_dataset_and_remote_load_round_trip(s3_root, tmp_path, backend, entr assert isinstance( registry.open_release(DatasetRef.parse("timenet/hello-world@1.0.0")).filesystem, pafs.S3FileSystem ) - assert not (tmp_path / "cache" / make_dataset().metadata.dataset_id).exists() def test_put_dataset_checks_payload_sources_against_the_bucket(s3_root, tmp_path): local = LocalRegistry(tmp_path / "local") local.put_dataset(make_dataset()) # moto keeps one bucket state per process, so this test uses a prefix no other test writes. - registry = S3Registry("s3://tn-test/payload-sources", cache_dir=tmp_path / "cache") + registry = S3Registry("s3://tn-test/payload-sources") with TimeFReader(open_local_version(local.root / "timenet/hello-world/1.0.0")) as reader: parent = reader.read() child = TimeFDataset.from_parts( @@ -92,24 +91,24 @@ def test_put_dataset_checks_payload_sources_against_the_bucket(s3_root, tmp_path def test_objects_live_under_a_datasets_prefix(s3_root, tmp_path): # Keys sit under /datasets/... so the bucket matches the hosted registry's layout and can # hold other top-level prefixes beside the datasets. - registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") + registry = S3Registry(s3_root) registry.put_dataset(make_dataset()) keys = [obj["Key"] for obj in boto3.client("s3").list_objects_v2(Bucket="tn-test")["Contents"]] assert keys # something was stored assert all(key.startswith("registry/datasets/") for key in keys) -def test_download_then_load_serves_from_cache(s3_root, tmp_path): - registry = S3Registry(s3_root, cache_dir=tmp_path / "cache") +def test_download_then_load_reads_the_store_copy(s3_root, tmp_path): + registry = S3Registry(s3_root) dataset_id = make_dataset().metadata.dataset_id registry.put_dataset(make_dataset()) - # The store is the cache directory of the registry, so the registry reuses each download. - client = TimeNet(sources=[LocalRegistry(tmp_path / "cache"), registry]) + store = LocalRegistry(tmp_path / "store") + client = TimeNet(sources=[store, registry]) path = client.download(dataset_id) # boto3 GetObject per file, into the store - assert path == tmp_path / "cache" / dataset_id / "1.0.0" + assert path == store.root / dataset_id / "1.0.0" assert (path / "manifest.json").exists() - handle = registry.open_release(DatasetRef.parse(f"{dataset_id}@1.0.0")) - assert isinstance(handle.filesystem, pafs.LocalFileSystem) # cache-first: no S3 reads + with client.open(dataset_id) as reader: + assert isinstance(reader._version.filesystem, pafs.LocalFileSystem) # the files come from the store, not S3 assert_datasets_equal(make_dataset(), client.load(dataset_id)) @@ -117,7 +116,7 @@ def test_get_bytes_reraises_non_404_client_errors(tmp_path, monkeypatch): # A missing object returns None, but any other S3 error (auth, throttling) must surface, not hide. import botocore.exceptions # ruff: ignore[import-outside-top-level] - registry = S3Registry("s3://tn-test/registry", cache_dir=tmp_path) + registry = S3Registry("s3://tn-test/registry") denied = botocore.exceptions.ClientError({"Error": {"Code": "AccessDenied", "Message": "no"}}, "GetObject") class _Denying: @@ -130,7 +129,7 @@ def get_object(self, **kwargs): def test_list_and_search_are_unsupported(tmp_path): - registry = S3Registry("s3://tn-test/registry", cache_dir=tmp_path) + registry = S3Registry("s3://tn-test/registry") with pytest.raises(NotImplementedError): registry.list_datasets() with pytest.raises(NotImplementedError): @@ -139,7 +138,7 @@ def test_list_and_search_are_unsupported(tmp_path): def test_rejects_a_non_s3_uri(tmp_path): with pytest.raises(TimeNetRegistryError, match="s3://"): - S3Registry("https://not-s3", cache_dir=tmp_path) + S3Registry("https://not-s3") class _Unreachable: diff --git a/packages/timenet/tests/test_cache.py b/packages/timenet/tests/test_cache.py index 1daf8484e..9a391553f 100644 --- a/packages/timenet/tests/test_cache.py +++ b/packages/timenet/tests/test_cache.py @@ -1,23 +1,74 @@ +from pathlib import Path + import pytest -from timenet.cache import cached_datasets, clear_cache, human_bytes +from timenet.cache import ( + ClearedCache, + StoredRelease, + cache_size, + clear_cache, + clear_releases, + human_bytes, + stored_releases, +) +from timenet.config import CACHE_TAG, is_cache_folder, make_cache_folder +from timenet.errors import TimeFValidationError +from timenet.registry import LocalRegistry from timenet.testing import make_dataset -from timenet.writer import TimeFWriter +from timenet.types import Version +from timenet.types.references import DatasetRef -@pytest.fixture(autouse=True) -def _home(tmp_path, monkeypatch): - for var in ("TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): - monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - return tmp_path / "home" +def _store_with_two_releases(root): + """Return a store that holds timenet/hello-world at 1.0.0 and 1.1.0.""" + store = LocalRegistry(root) + for version in (Version(1, 0, 0), Version(1, 1, 0)): + dataset = make_dataset() + dataset._metadata = dataset.metadata.model_copy(update={"dataset_version": version}) + store.put_dataset(dataset) + return store -def _write(root): +def _other_dataset(): dataset = make_dataset() - dataset.derive_schema() - with TimeFWriter(root, dataset) as writer: - writer.write() + dataset._metadata = dataset.metadata.model_copy(update={"dataset_id": "demo/other"}) + return dataset + + +def _size(directory): + return sum(path.lstat().st_size for path in directory.rglob("*") if not path.is_dir()) + + +def _raw(cache, size=10): + """Write ``size`` bytes of raw downloads into ``/downloads``, a folder that TimeNet made.""" + make_cache_folder(cache / "downloads") + raw = cache / "downloads/timenet/hello-world/raw.bin" + raw.parent.mkdir(parents=True) + raw.write_bytes(b"x" * size) + return raw + + +def test_make_cache_folder_marks_the_folder_that_it_creates(tmp_path): + make_cache_folder(tmp_path / "cache" / "downloads") + lines = (tmp_path / "cache/downloads" / CACHE_TAG).read_text().splitlines() + assert lines[0] == "Signature: 8a477f597d28d172789f06886806bc55" + assert "TimeNet" in lines[1] + assert is_cache_folder(tmp_path / "cache" / "downloads") + + +def test_make_cache_folder_leaves_an_existing_folder_unmarked(tmp_path): + (tmp_path / "downloads").mkdir() + make_cache_folder(tmp_path / "downloads") + assert list((tmp_path / "downloads").iterdir()) == [] + assert not is_cache_folder(tmp_path / "downloads") + + +def test_a_marker_of_another_tool_does_not_count(tmp_path): + (tmp_path / "downloads").mkdir() + (tmp_path / "downloads" / CACHE_TAG).write_text( + "Signature: 8a477f597d28d172789f06886806bc55\n# This file is a cache directory tag created by another tool.\n" + ) + assert not is_cache_folder(tmp_path / "downloads") @pytest.mark.parametrize( @@ -37,48 +88,185 @@ def test_human_bytes(n, expected): assert human_bytes(n) == expected -def test_cached_datasets_lists_registry_and_storage(_home): - _write(_home / "registry") - _write(_home / "storage") - cached = cached_datasets() - locations = {(c.location, c.dataset_id, c.version) for c in cached} - assert ("registry", "timenet/hello-world", "1.0.0") in locations - assert ("storage", "timenet/hello-world", "1.0.0") in locations - assert all(c.size_bytes > 0 for c in cached) +def test_stored_releases_lists_each_release_with_its_size(tmp_path): + store = _store_with_two_releases(tmp_path / "store") + first, second = (tmp_path / "store/timenet/hello-world" / version for version in ("1.0.0", "1.1.0")) + assert stored_releases(store) == [ + StoredRelease(DatasetRef.parse("timenet/hello-world@1.0.0"), first, _size(first)), + StoredRelease(DatasetRef.parse("timenet/hello-world@1.1.0"), second, _size(second)), + ] + assert _size(first) > 0 + + +def test_stored_releases_skips_what_is_not_a_release(tmp_path): + store = _store_with_two_releases(tmp_path / "store") + manifest = (tmp_path / "store/timenet/hello-world/1.0.0/manifest.json").read_bytes() + # A directory whose name is not a version, a staging directory, a hidden one, a path that is not + # a dataset ID, and a manifest at the root and one level below. + for stray in ( + "timenet/hello-world/draft", + "timenet/hello-world/2.0.0.tmp-1", + ".trash/x/1.0.0", + "a/b/c/1.0.0", + ".", + "1.0.0", + ): + (tmp_path / "store" / stray).mkdir(parents=True, exist_ok=True) + (tmp_path / "store" / stray / "manifest.json").write_bytes(manifest) + assert [str(item.release) for item in stored_releases(store)] == [ + "timenet/hello-world@1.0.0", + "timenet/hello-world@1.1.0", + ] + + +def test_clear_cache_keeps_a_store_inside_the_cache(tmp_path): + cache = tmp_path / "cache" + store = LocalRegistry(cache / "registry") + store.put_dataset(make_dataset()) + raw = _raw(cache) + other = cache / "other/keep.txt" + other.parent.mkdir() + other.write_text("kept") + size = _size(cache / "downloads") + assert cache_size(cache) == size + assert clear_cache(cache, local_registry=store) == ClearedCache(size, [cache / "downloads"], []) + assert store.exists("timenet/hello-world", "1.0.0") + assert other.exists() + assert not raw.exists() -def test_cached_datasets_empty_when_nothing(): - assert cached_datasets() == [] +def test_clear_cache_deletes_the_control_files(tmp_path): + cache = tmp_path / "cache" + make_cache_folder(cache / "control") + (cache / "control/abc.duckdb").write_bytes(b"x" * 7) + size = _size(cache / "control") + assert cache_size(cache) == size + cleared = clear_cache(cache, local_registry=LocalRegistry(tmp_path / "store")) + assert cleared == ClearedCache(size, [cache / "control"], []) + assert not (cache / "control").exists() + assert cache.is_dir() -def test_clear_removes_storage_and_cache_but_keeps_registry(_home): - _write(_home / "registry") - _write(_home / "storage") - (_home / "cache" / "raw").mkdir(parents=True) - (_home / "cache" / "raw" / "f.bin").write_bytes(b"x" * 100) +def test_clear_cache_keeps_and_reports_the_folders_without_the_marker(tmp_path): + cache = tmp_path / "cache" + (cache / "downloads").mkdir(parents=True) + (cache / "downloads/report.pdf").write_bytes(b"x" * 10) + (cache / "control").write_text("a file, not a folder") + assert cache_size(cache) == 0 + cleared = clear_cache(cache, local_registry=LocalRegistry(tmp_path / "store")) + assert cleared == ClearedCache(0, [], [cache / "downloads", cache / "control"]) + assert (cache / "downloads/report.pdf").exists() + assert (cache / "control").read_text() == "a file, not a folder" - freed, _ = clear_cache(include_registry=False) - assert freed > 0 - assert not (_home / "storage").exists() - assert not (_home / "cache").exists() - assert (_home / "registry").exists() # built outputs kept +def test_clear_releases_deletes_every_release_but_no_other_file(tmp_path): + store = _store_with_two_releases(tmp_path / "store") + store.put_dataset(_other_dataset()) + notes = tmp_path / "store/notes.txt" + notes.write_text("kept") + staging = tmp_path / "store/timenet/hello-world/2.0.0.tmp-1/part.bin" + staging.parent.mkdir() + staging.write_bytes(b"kept") + releases = stored_releases(store) -def test_clear_all_removes_registry_too(_home): - _write(_home / "registry") - clear_cache(include_registry=True) - assert not (_home / "registry").exists() + cleared = clear_releases(store) + assert cleared.freed == sum(item.size_bytes for item in releases) + assert cleared.removed == [item.path for item in releases] + assert stored_releases(store) == [] + # The dataset directories that the call emptied go, and the files that are not releases stay. + assert not (tmp_path / "store/demo").exists() + assert sorted(path.relative_to(tmp_path / "store").as_posix() for path in (tmp_path / "store").rglob("*")) == [ + "notes.txt", + "timenet", + "timenet/hello-world", + "timenet/hello-world/2.0.0.tmp-1", + "timenet/hello-world/2.0.0.tmp-1/part.bin", + ] -def test_clear_sizes_a_symlink_as_the_link_not_its_target(_home, tmp_path): + +def test_clear_sizes_a_symlink_as_the_link_not_its_target(tmp_path): target = tmp_path / "big.bin" target.write_bytes(b"x" * 10_000) - cache = _home / "cache" - cache.mkdir(parents=True) - (cache / "link.bin").symlink_to(target) + downloads = tmp_path / "cache/downloads" + make_cache_folder(downloads) + (downloads / "link.bin").symlink_to(target) - freed, _ = clear_cache(include_registry=False) + cleared = clear_cache(tmp_path / "cache", local_registry=LocalRegistry(tmp_path / "store")) - assert freed < 10_000 # counted the link itself, never followed it to the 10 KB target + assert cleared.freed < 10_000 # counted the link itself, never followed it to the 10 KB target assert target.exists() # the external target is left untouched - assert not cache.exists() + assert not downloads.exists() + + +@pytest.mark.parametrize("marked", [True, False]) +def test_clear_cache_unlinks_a_symlinked_downloads_folder_only_if_marked_and_keeps_its_target(tmp_path, marked): + # The user moved the downloads folder, and left a symlink in its place. + target = tmp_path / "elsewhere" + if marked: + make_cache_folder(target) + else: + target.mkdir() + (target / "raw.bin").write_bytes(b"x" * 10) + cache = tmp_path / "cache" + cache.mkdir() + (cache / "downloads").symlink_to(target, target_is_directory=True) + link_size = (cache / "downloads").lstat().st_size + assert cache_size(cache) == (link_size if marked else 0) # the link, not its target + + cleared = clear_cache(cache, local_registry=LocalRegistry(tmp_path / "store")) + + if marked: + assert cleared == ClearedCache(link_size, [cache / "downloads"], []) + assert not (cache / "downloads").exists() and not (cache / "downloads").is_symlink() + else: + assert cleared == ClearedCache(0, [], [cache / "downloads"]) + assert (cache / "downloads").is_symlink() + assert (target / "raw.bin").exists() + + +def test_clear_cache_refuses_a_store_in_a_folder_that_it_deletes(tmp_path): + make_cache_folder(tmp_path / "cache" / "downloads") + store = LocalRegistry(tmp_path / "cache" / "downloads" / "store") + store.put_dataset(make_dataset()) + with pytest.raises(TimeFValidationError, match="overlaps"): + clear_cache(tmp_path / "cache", local_registry=store) + assert store.exists("timenet/hello-world", "1.0.0") + + +def test_clear_cache_refuses_a_folder_that_it_deletes_in_the_store(tmp_path): + # With the cache in the store, the downloads folder can be the directory of a dataset. + store = LocalRegistry(tmp_path / "store") + make_cache_folder(tmp_path / "store" / "acme" / "downloads") + (tmp_path / "store/acme/downloads/1.0.0").mkdir() + with pytest.raises(TimeFValidationError, match="overlaps"): + clear_cache(tmp_path / "store" / "acme", local_registry=store) + assert (tmp_path / "store/acme/downloads/1.0.0").is_dir() + + +@pytest.mark.parametrize("cache", ["~", "parent"]) +def test_clear_cache_refuses_the_home_folder_or_a_folder_above_it(tmp_path, home, cache): + # With the cache at ~, a clear would delete ~/downloads, which is ~/Downloads on a volume that ignores case. + cache_dir = home.parent if cache == "parent" else home + raw = _raw(cache_dir) + with pytest.raises(TimeFValidationError, match=r"^The cache .* is the home folder or a folder above it"): + clear_cache(cache_dir, local_registry=LocalRegistry(tmp_path / "store")) + assert raw.exists() + + +def test_clear_cache_works_without_a_home_folder(tmp_path, monkeypatch): + # For example a container whose user has no home folder. + def no_home(): + raise RuntimeError("Could not determine home directory.") + + monkeypatch.setattr(Path, "home", staticmethod(no_home)) + raw = _raw(tmp_path / "cache") + cleared = clear_cache(tmp_path / "cache", local_registry=LocalRegistry(tmp_path / "store")) + assert cleared.removed == [tmp_path / "cache" / "downloads"] + assert not raw.exists() + + +def test_clear_cache_of_a_missing_cache_frees_nothing(tmp_path): + assert cache_size(tmp_path / "cache") == 0 + cleared = clear_cache(tmp_path / "cache", local_registry=LocalRegistry(tmp_path / "store")) + assert cleared == ClearedCache(0, [], []) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index e03bf46c6..d6af67f8b 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -8,12 +8,13 @@ import typer from typer.testing import CliRunner -from timenet.cache import cached_datasets, raw_cache_size +from timenet.cache import human_bytes, stored_releases from timenet.cli.app import _enum_list, app +from timenet.config import make_cache_folder from timenet.dataset import TimeFDataset -from timenet.errors import TimeNetDatasetNotFoundError +from timenet.errors import TimeFValidationError, TimeNetDatasetNotFoundError from timenet.reader import TimeFReader -from timenet.registry import open_local_version +from timenet.registry import LocalRegistry, open_local_version from timenet.testing import make_dataset from timenet.types import Access, Domain from timenet.types.reference_keys import ParentAlias @@ -36,48 +37,99 @@ def registry_root(tmp_path): return tmp_path / "reg" -@pytest.fixture -def home(tmp_path, monkeypatch): - for var in ("TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): - monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - dataset = make_dataset() - dataset.derive_schema() - with TimeFWriter(tmp_path / "home" / "registry", dataset) as writer: - writer.write() - return tmp_path / "home" +def _downloads_size(cache): + """Return the size of the raw downloads and the marker file in ``/downloads``.""" + return sum(path.stat().st_size for path in (cache / "downloads").rglob("*") if path.is_file()) -def test_cache_info_lists_datasets(home): - assert runner.invoke(app, ["cache", "info"]).exit_code == 0 - assert any(d.dataset_id == "timenet/hello-world" for d in cached_datasets()) - - -def test_cache_info_empty(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "empty")) - assert runner.invoke(app, ["cache", "info"]).exit_code == 0 - assert cached_datasets() == [] +@pytest.fixture +def state(tmp_path, monkeypatch): + """Put a release in the store and 10 bytes of raw downloads in the cache. Returns both paths.""" + store, cache = tmp_path / "store", tmp_path / "cache" + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(store)) + monkeypatch.setenv("TIMENET_CACHE", str(cache)) + LocalRegistry(store).put_dataset(make_dataset()) + make_cache_folder(cache / "downloads") + raw = cache / "downloads/timenet/hello-world/raw.bin" + raw.parent.mkdir(parents=True) + raw.write_bytes(b"x" * 10) + return store, cache + + +def test_local_info_and_local_cache_info_show_the_paths_and_sizes(state): + store, cache = state + result = runner.invoke(app, ["local", "info"]) + assert result.exit_code == 0, result.output + [release] = stored_releases(LocalRegistry(store)) + cache_line = f"Cache {cache}: {human_bytes(_downloads_size(cache))}" + assert click.unstyle(result.stdout).splitlines() == [ + cache_line, + f"Local registry {store}: 1 release, {human_bytes(release.size_bytes)}", + ] + result = runner.invoke(app, ["local", "cache", "info"]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.stdout) == f"{cache_line}\n" + + +@pytest.mark.parametrize( + ("command", "prompt"), + [ + (["local", "clear"], "Delete the cache in {cache} and every release in {store}?"), + (["local", "cache", "clear"], "Delete the raw source files and control databases in {cache}?"), + ], +) +def test_each_clear_asks_first_and_names_the_paths(state, command, prompt): + store, cache = state + result = runner.invoke(app, command, input="n\n") + assert result.exit_code != 0 # aborted + assert f"{prompt.format(cache=cache, store=store)} [y/N]" in click.unstyle(result.output) + assert (cache / "downloads").exists() + assert LocalRegistry(store).exists("timenet/hello-world", "1.0.0") -def test_cache_info_reports_raw_cache_size(home): - cache_dir = home / "cache" - cache_dir.mkdir(parents=True, exist_ok=True) - (cache_dir / "blob").write_bytes(b"x" * 10) - assert runner.invoke(app, ["cache", "info"]).exit_code == 0 - assert raw_cache_size() == 10 +def test_local_cache_clear_keeps_the_local_registry(state): + store, cache = state + size = human_bytes(_downloads_size(cache)) + result = runner.invoke(app, ["local", "cache", "clear"], input="y\n") + assert result.exit_code == 0, result.output + assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 1 location(s)." + assert not (cache / "downloads").exists() + assert LocalRegistry(store).exists("timenet/hello-world", "1.0.0") -def test_cache_clear_needs_confirmation(home): - result = runner.invoke(app, ["cache", "clear"], input="n\n") - assert result.exit_code != 0 # aborted - assert (home / "registry").exists() +def test_local_cache_clear_warns_about_a_folder_without_the_marker_and_keeps_it(state): + _, cache = state + (cache / "control").mkdir() + (cache / "control/notes.txt").write_text("not from TimeNet") + result = runner.invoke(app, ["local", "cache", "clear", "--yes"]) + assert result.exit_code == 0, result.output + # The console wraps a long line, also in a path, so the check ignores whitespace. + expected = f"Kept {cache / 'control'}: it has no CACHEDIR.TAG from TimeNet" + assert "".join(expected.split()) in "".join(click.unstyle(result.stderr).split()) + assert click.unstyle(result.stdout).splitlines()[-1].endswith("from 1 location(s).") + assert (cache / "control/notes.txt").exists() + + +def test_local_clear_deletes_the_cache_and_every_release(state): + store, cache = state + [release] = stored_releases(LocalRegistry(store)) + size = human_bytes(_downloads_size(cache) + release.size_bytes) + result = runner.invoke(app, ["local", "clear", "--yes"]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 2 location(s)." + assert stored_releases(LocalRegistry(store)) == [] + assert not (cache / "downloads").exists() -def test_cache_clear_all_removes_everything(home): - result = runner.invoke(app, ["cache", "clear", "--all", "--yes"]) - assert result.exit_code == 0 - assert "Freed" in result.stdout - assert not (home / "registry").exists() +def test_local_clear_deletes_no_release_when_the_cache_check_fails(state, monkeypatch): + _, cache = state + # The local registry lies in the downloads folder that the clear deletes. + inside = LocalRegistry(cache / "downloads" / "store") + inside.put_dataset(make_dataset()) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(inside.root)) + result = runner.invoke(app, ["local", "clear", "--yes"]) + assert isinstance(result.exception, TimeFValidationError) + assert inside.exists("timenet/hello-world", "1.0.0") def test_list(registry_root): @@ -155,8 +207,7 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): def test_download_command_copies_into_the_store_and_then_has_nothing_to_do(registry_root, tmp_path, monkeypatch): # The last line names a long path, so the console must not wrap it. monkeypatch.setattr(_cli_module.ui, "rich", Console(stderr=True, width=1000)) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "store")) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) command = ["download", "timenet/hello-world", "--registry", str(registry_root)] target = tmp_path / "store" / "timenet/hello-world" / "1.0.0" @@ -177,8 +228,7 @@ def test_download_command_copies_into_the_store_and_then_has_nothing_to_do(regis def test_download_command_with_force_copies_a_held_release_again(registry_root, tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "store")) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) command = ["download", "timenet/hello-world", "--registry", str(registry_root)] assert runner.invoke(app, command).exit_code == 0 diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 5fa13c006..dacc64a74 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from _demo_child import _child_of_the_demo +from _fake_registry import build_service_fake import httpx from pydantic import ValidationError import pytest @@ -159,7 +160,6 @@ def test_a_damaged_build_in_the_store_names_the_build_that_replaces_it(tmp_path, empty = RemoteRegistry( "http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404)), - cache_dir=tmp_path / "cache", ) client = TimeNet(sources=[store, empty, InstalledConnectors()]) client.load("timenet/hello-world@1.0.0") # the build puts the release into the store @@ -173,12 +173,6 @@ def test_a_damaged_build_in_the_store_names_the_build_that_replaces_it(tmp_path, assert len(calls) == 1 # nothing builds over the damaged copy -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch): - for var in ("TIMENET_HOME", "TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): - monkeypatch.delenv(var, raising=False) - - @pytest.fixture def registry_root(tmp_path): dataset = make_dataset() @@ -197,41 +191,52 @@ def _entry(source): return str(source) -@pytest.mark.parametrize("registry", [None, "https://registry.example.test", "local"]) -def test_the_default_list_follows_timenet_registry(tmp_path, monkeypatch, registry): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - store = ("local", tmp_path / "home" / "registry") - if registry == "local": - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "elsewhere")) - expected = [("local", tmp_path / "elsewhere"), "installed connectors"] - elif registry is not None: - monkeypatch.setenv("TIMENET_REGISTRY", registry) - expected = [store, f"registry {registry}", "installed connectors"] - else: - expected = [store, f"registry {TIMENET_REGISTRY_URL}", "installed connectors"] +def test_the_default_list_follows_the_environment(monkeypatch, tmp_path): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(tmp_path / "remote")) + monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") + store, remote = default_sources() + assert isinstance(store, LocalRegistry) and isinstance(remote, LocalRegistry) + assert (store.root, remote.root) == (tmp_path / "store", tmp_path / "remote") + # TimeNet() uses the same list. client = TimeNet() - - assert [_entry(source) for source in client._sources] == expected + assert [_entry(source) for source in client._sources] == [ + ("local", tmp_path / "store"), + ("local", tmp_path / "remote"), + ] assert client._store is client._sources[0] - # The argument comes before $TIMENET_REGISTRY, and adds a remote to a local $TIMENET_REGISTRY. - with_remote = [_entry(source) for source in default_sources(remote_registry="https://other.example.test")] - assert with_remote == [expected[0], "registry https://other.example.test", "installed connectors"] -def test_timenet_registry_env_selects_a_local_registry(tmp_path, monkeypatch): - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.setenv("TIMENET_REGISTRY", str(tmp_path / "home" / "registry")) - dataset = make_dataset() - dataset.derive_schema() - with TimeFWriter(tmp_path / "home" / "registry", dataset) as writer: - writer.write() - client = TimeNet() # the store comes from $TIMENET_REGISTRY - assert [_entry(source) for source in client._sources] == [ - ("local", tmp_path / "home" / "registry"), +@pytest.mark.usefixtures("home") +def test_the_default_list_without_variables(monkeypatch): + for name in ("LOCAL_REGISTRY", "REMOTE_REGISTRY", "AUTO_BUILD", "CACHE"): + monkeypatch.delenv(f"TIMENET_{name}", raising=False) + store, remote, connectors = default_sources() + assert isinstance(store, LocalRegistry) and isinstance(remote, RemoteRegistry) + assert store.root == Path("~/.timenet/registry").expanduser() + assert str(remote) == f"registry {TIMENET_REGISTRY_URL}" + assert type(connectors) is InstalledConnectors + + +def test_default_sources_arguments_override_the_environment(monkeypatch, tmp_path): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "env-store")) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(tmp_path / "env-remote")) + monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") + sources = default_sources(local_registry=tmp_path / "store", remote_registry=tmp_path / "remote", auto_build=True) + assert [_entry(source) for source in sources] == [ + ("local", tmp_path / "store"), + ("local", tmp_path / "remote"), "installed connectors", ] - assert {m.dataset_id for m in client.list()} == {"timenet/hello-world"} - assert client.download("timenet/hello-world") == tmp_path / "home" / "registry" / "timenet/hello-world" / "1.0.0" + + +def test_default_sources_names_the_variable_of_a_registry_that_cannot_open(monkeypatch): + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", "gs://bucket/registry") + with pytest.raises( + TimeNetRegistryError, + match=r"^TIMENET_REMOTE_REGISTRY 'gs://bucket/registry' has an unsupported registry scheme", + ): + default_sources() def test_a_client_takes_an_ordered_list_of_sources(tmp_path): @@ -440,10 +445,9 @@ def test_download_counts_a_registry_over_the_directory_of_the_store_as_the_store if case == "damaged": (directory / "control.duckdb").write_bytes(b"cut short") before = {path: path.read_bytes() for path in directory.rglob("*") if path.is_file()} - # As for `TIMENET_REGISTRY=./reg timenet download timenet/hello-world --registry ./reg`: the + # As for `TIMENET_LOCAL_REGISTRY=./reg timenet download timenet/hello-world --registry ./reg`: the # registry after the store reads the directory of the store, so it cannot replace its copy. - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - monkeypatch.setenv("TIMENET_REGISTRY", str(root)) + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(root)) client = TimeNet(sources=default_sources(remote_registry=root)) if case == "force": @@ -541,13 +545,15 @@ def test_download_repairs_a_damaged_store_copy(tmp_path): def test_download_returns_a_complete_store_copy_at_once(registry_root, tmp_path): - client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), str(registry_root)]) + transport, requests = build_service_fake(registry_root) + remote = RemoteRegistry("http://api.local", transport=transport) + client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), remote]) first = client.download("timenet/hello-world") - # The store copy is complete, so the next download needs no registry. - shutil.rmtree(registry_root) + requests.clear() + # The store copy is complete, so the next download asks the remote registry nothing. events: list[ProgressEvent] = [] assert client.download("timenet/hello-world", progress=events.append) == first - assert events == [] # nothing is copied again + assert (events, requests) == ([], []) def test_download_with_force_copies_again(registry_root, tmp_path): @@ -830,11 +836,7 @@ class _Hosted(RemoteRegistry): def get_manifest(self, dataset_id, version=None): return SimpleNamespace(metadata=metadata, lineage=lineage) - return _Hosted( - "http://api.local", - transport=httpx.MockTransport(lambda request: httpx.Response(404)), - cache_dir=tmp_path / "cache", - ) + return _Hosted("http://api.local", transport=httpx.MockTransport(lambda request: httpx.Response(404))) def test_load_rejects_a_credentialed_dataset_from_a_hosted_registry(tmp_path): diff --git a/packages/timenet/tests/test_config.py b/packages/timenet/tests/test_config.py index 82f1d4888..d3b5fe5b4 100644 --- a/packages/timenet/tests/test_config.py +++ b/packages/timenet/tests/test_config.py @@ -3,63 +3,87 @@ import pytest from timenet.config import TimeNetSettings, settings +from timenet.errors import TimeFValidationError, TimeNetRegistryError -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch): - for var in ("TIMENET_HOME", "TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): - monkeypatch.delenv(var, raising=False) +def _unset(monkeypatch): + """Delete the variables of the defaults, also the temporary paths that the root ``conftest.py`` sets.""" + for name in ("LOCAL_REGISTRY", "REMOTE_REGISTRY", "AUTO_BUILD", "CACHE"): + monkeypatch.delenv(f"TIMENET_{name}", raising=False) -def test_defaults_under_home(): - cfg = settings(home=Path("/base")) - assert cfg.home_dir == Path("/base") - assert cfg.storage_dir == Path("/base/storage") - assert cfg.cache_dir == Path("/base/cache") - assert cfg.control_cache_dir == Path("/base/cache/control") - assert cfg.registry_path == Path("/base/registry") +@pytest.mark.usefixtures("home") +def test_the_defaults_are_the_store_the_hosted_registry_the_connectors_and_the_cache(monkeypatch): + _unset(monkeypatch) + cfg = settings() + assert cfg.local_registry_dir == Path("~/.timenet/registry").expanduser() + assert (cfg.remote_registry, cfg.auto_build) == ("timenet://", True) + assert cfg.cache_dir == Path("~/.cache/timenet").expanduser() + assert cfg.downloads_dir == cfg.cache_dir / "downloads" + assert cfg.control_cache_dir == cfg.cache_dir / "control" + + +def test_each_variable_sets_its_value(monkeypatch, tmp_path): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) + monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", "https://registry.example.test") + monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") + monkeypatch.setenv("TIMENET_CACHE", str(tmp_path / "cache")) + cfg = settings() + assert cfg.local_registry_dir == tmp_path / "store" + assert (cfg.remote_registry, cfg.auto_build) == ("https://registry.example.test", False) + assert cfg.cache_dir == tmp_path / "cache" + assert cfg.downloads_dir == tmp_path / "cache" / "downloads" -def test_default_home_is_cache_timenet(): - assert settings().home_dir == Path("~/.cache/timenet").expanduser() +def test_the_local_registry_rejects_a_remote_url(monkeypatch): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", "s3://bucket/store") + with pytest.raises(TimeNetRegistryError, match="the local registry needs a local directory"): + _ = settings().local_registry_dir -def test_home_env_relocates_everything(monkeypatch): - monkeypatch.setenv("TIMENET_HOME", "/relocated") - cfg = settings() - assert cfg.storage_dir == Path("/relocated/storage") - assert cfg.cache_dir == Path("/relocated/cache") - assert cfg.control_cache_dir == Path("/relocated/cache/control") - assert cfg.registry_path == Path("/relocated/registry") +def test_an_explicit_override_beats_the_variable_and_none_lets_it_win(monkeypatch, tmp_path): + monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "from-env")) + monkeypatch.setenv("TIMENET_AUTO_BUILD", "true") + cfg = settings(local_registry=tmp_path / "from-arg", auto_build=False) + assert (cfg.local_registry_dir, cfg.auto_build) == (tmp_path / "from-arg", False) + cfg = settings(local_registry=None, auto_build=None) # a missing CLI flag falls through to the variable + assert (cfg.local_registry_dir, cfg.auto_build) == (tmp_path / "from-env", True) -def test_specific_env_overrides_only_its_path(monkeypatch): - monkeypatch.setenv("TIMENET_HOME", "/base") - monkeypatch.setenv("TIMENET_STORAGE", "/elsewhere/storage") - cfg = settings() - assert cfg.storage_dir == Path("/elsewhere/storage") - assert cfg.cache_dir == Path("/base/cache") # still under home +def test_a_path_override_works_for_each_path_setting(tmp_path): + cfg = settings(local_registry=tmp_path / "store", remote_registry=tmp_path / "remote", cache=tmp_path / "cache") + assert (cfg.local_registry_dir, cfg.remote_registry, cfg.cache_dir) == ( + tmp_path / "store", + str(tmp_path / "remote"), + tmp_path / "cache", + ) -def test_explicit_override_beats_env(monkeypatch): - monkeypatch.setenv("TIMENET_STORAGE", "/from/env") - cfg = settings(storage=Path("/from/arg")) - assert cfg.storage_dir == Path("/from/arg") +def test_an_invalid_timenet_auto_build_raises(monkeypatch): + monkeypatch.setenv("TIMENET_AUTO_BUILD", "maybe") + with pytest.raises(TimeFValidationError, match="TIMENET_AUTO_BUILD"): + settings() -def test_settings_drops_none_so_env_wins(monkeypatch): - monkeypatch.setenv("TIMENET_STORAGE", "/from/env") - cfg = settings(storage=None) # a missing CLI flag must fall through to env - assert cfg.storage_dir == Path("/from/env") +@pytest.mark.parametrize("value", ["", " "]) +@pytest.mark.parametrize("name", ["LOCAL_REGISTRY", "REMOTE_REGISTRY", "CACHE"]) +def test_an_empty_or_blank_setting_raises_instead_of_naming_a_directory(monkeypatch, name, value): + monkeypatch.setenv(f"TIMENET_{name}", value) + with pytest.raises(TimeFValidationError, match=f"TIMENET_{name} must not be empty"): + settings() -def test_registry_selector_defaults_none(): - assert settings(home=Path("/base")).registry is None +def test_an_empty_argument_raises_with_the_text_of_its_variable(): + with pytest.raises( + TimeFValidationError, + match=r"^TIMENET_REMOTE_REGISTRY must not be empty; give it a value, or leave it unset for timenet://\.$", + ): + settings(remote_registry="") -def test_registry_selector_from_env(monkeypatch): - monkeypatch.setenv("TIMENET_REGISTRY", "https://reg.example.io") - assert settings().registry == "https://reg.example.io" +def test_an_argument_replaces_an_invalid_variable(monkeypatch, tmp_path): + monkeypatch.setenv("TIMENET_CACHE", "") + assert settings(cache=tmp_path / "cache").cache_dir == tmp_path / "cache" def test_is_basesettings(): diff --git a/packages/timenet/tests/test_conftest.py b/packages/timenet/tests/test_conftest.py index a7aa29752..469af7971 100644 --- a/packages/timenet/tests/test_conftest.py +++ b/packages/timenet/tests/test_conftest.py @@ -12,17 +12,16 @@ ROOT_CONFTEST = Path(__file__).resolve().parents[3] / "conftest.py" -PATHS = ("registry_path", "storage_dir", "cache_dir", "control_cache_dir") -"""The settings properties that point at local state.""" +PATHS = ("local_registry_dir", "remote_registry", "cache_dir", "downloads_dir", "control_cache_dir") +"""The settings that point at local state during a test.""" def test_each_test_starts_without_the_timenet_variables_of_the_shell(pytester, monkeypatch): for name, value in { - "TIMENET_REGISTRY": "timenet://", - "TIMENET_STORAGE": "/shell/storage", - "TIMENET_CACHE": "/shell/cache", + "TIMENET_AUTO_BUILD": "false", "TIMENET_TOKEN": "secret", "TIMENET_ISOLATION": "off", + "TIMENET_ROW_LIMIT": "3", }.items(): monkeypatch.setenv(name, value) pytester.makeconftest(ROOT_CONFTEST.read_text()) @@ -30,26 +29,26 @@ def test_each_test_starts_without_the_timenet_variables_of_the_shell(pytester, m """ import os - def test_only_the_home_is_set(): + def test_only_the_three_paths_are_set(): names = sorted(name for name in os.environ if name.startswith("TIMENET_")) - assert names == ["TIMENET_HOME"] + assert names == ["TIMENET_CACHE", "TIMENET_LOCAL_REGISTRY", "TIMENET_REMOTE_REGISTRY"] """ ) pytester.runpytest().assert_outcomes(passed=1) -def test_every_settings_path_lies_in_the_home_of_the_test_also_in_a_child_process(tmp_path): - home = Path(os.environ["TIMENET_HOME"]) - in_process = [getattr(settings(), name) for name in PATHS] +def test_every_settings_path_lies_in_the_state_of_the_test_also_in_a_child_process(tmp_path): + state = Path(os.environ["TIMENET_LOCAL_REGISTRY"]).parent + in_process = [Path(getattr(settings(), name)) for name in PATHS] code = f"from timenet.config import settings; print(*(getattr(settings(), name) for name in {PATHS}), sep='\\n')" child = subprocess.run([sys.executable, "-c", code], cwd=tmp_path, capture_output=True, text=True, check=True) in_child = [Path(line) for line in child.stdout.splitlines()] - assert len(in_child) == len(in_process) == 4 - assert all(path.is_relative_to(home) for path in [*in_process, *in_child]) + assert len(in_child) == len(in_process) == 5 + assert all(path.is_relative_to(state) for path in [*in_process, *in_child]) def test_a_dotenv_file_in_the_working_directory_does_not_change_the_settings(monkeypatch, tmp_path): - (tmp_path / ".env").write_text("TIMENET_REGISTRY=timenet://\nTIMENET_TOKEN=secret\nTIMENET_ISOLATION=off\n") + (tmp_path / ".env").write_text("TIMENET_AUTO_BUILD=false\nTIMENET_TOKEN=secret\nTIMENET_ISOLATION=off\n") monkeypatch.chdir(tmp_path) cfg = settings() - assert (cfg.registry, cfg.token, cfg.isolation) == (None, None, "on") + assert (cfg.auto_build, cfg.token, cfg.isolation) == (True, None, "on") diff --git a/packages/timenet/tests/test_engine.py b/packages/timenet/tests/test_engine.py index 431b73b86..d7fa7fd63 100644 --- a/packages/timenet/tests/test_engine.py +++ b/packages/timenet/tests/test_engine.py @@ -4,7 +4,7 @@ import pytest -from timenet.config import settings +from timenet.config import is_cache_folder, settings from timenet.connectors import BaseConnector, BuildContext from timenet.dataset import TimeFDataset from timenet.engine import open_parents, run_pipeline, store_dataset @@ -161,30 +161,42 @@ def test_clean_cache_keeps_caller_supplied_dir(tmp_path): assert cache.is_dir() # a caller-owned cache_dir is never deleted, even when asked to clean it -def _isolated_home(tmp_path, monkeypatch) -> None: - """Point the settings at a scratch home so the default cache lands under tmp_path.""" - for var in ("TIMENET_STORAGE", "TIMENET_CACHE", "TIMENET_REGISTRY"): - monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home")) - - -def test_clean_cache_removes_the_auto_created_default(tmp_path, monkeypatch): - _isolated_home(tmp_path, monkeypatch) +def test_clean_cache_removes_the_auto_created_default(tmp_path): run_pipeline(_DemoConnector(), tmp_path / "root") # cache_dir=None -> the engine made it - assert not (settings().cache_dir / make_dataset().metadata.dataset_id).exists() + assert not (settings().downloads_dir / make_dataset().metadata.dataset_id).exists() -def test_run_pipeline_keeps_the_cache_when_asked(tmp_path, monkeypatch): - _isolated_home(tmp_path, monkeypatch) +def test_run_pipeline_keeps_the_cache_when_asked(tmp_path): run_pipeline(_DemoConnector(), tmp_path / "root", keep_cache=True) - assert (settings().cache_dir / make_dataset().metadata.dataset_id).exists() + assert (settings().downloads_dir / make_dataset().metadata.dataset_id).exists() + + +def test_the_engine_marks_the_downloads_folder_that_it_creates(tmp_path): + run_pipeline(_DemoConnector(), tmp_path / "root") + assert is_cache_folder(settings().downloads_dir) + +def test_a_build_into_the_downloads_that_it_removes_fails_before_the_download(tmp_path, monkeypatch): + monkeypatch.setattr(_DemoConnector, "download", lambda self, cache_dir: pytest.fail("downloaded")) + root = settings().downloads_dir / make_dataset().metadata.dataset_id / "registry" + with pytest.raises(TimeFValidationError, match=r"^the local registry .* overlaps .*, which TimeNet deletes"): + run_pipeline(_DemoConnector(), root) + assert not root.exists() -def test_run_pipeline_keeps_the_cache_when_download_fails(tmp_path, monkeypatch): + +def test_the_engine_deletes_nothing_in_a_downloads_folder_without_the_marker(tmp_path): + # The folder existed before TimeNet used it, for example ~/Downloads on a volume that ignores case. + # So the engine leaves it unmarked, and the folder of the dataset in it can hold files of the user. + folder = settings().downloads_dir / make_dataset().metadata.dataset_id + folder.mkdir(parents=True) + (folder / "notes.txt").write_text("mine") + run_pipeline(_DemoConnector(), tmp_path / "root") + assert (folder / "notes.txt").read_text() == "mine" + + +def test_run_pipeline_keeps_the_cache_when_download_fails(tmp_path): # Downloads resume: http writes a .part and renames it, so a file that landed is complete and the # next run skips it. Deleting the cache here would throw away good bytes. - _isolated_home(tmp_path, monkeypatch) - class _HalfDownloadedConnector(_DemoConnector): def download(self, cache_dir: Path) -> list[str]: (cache_dir / "first.bin").write_bytes(b"complete") @@ -192,14 +204,12 @@ def download(self, cache_dir: Path) -> list[str]: with pytest.raises(RuntimeError, match="download blew up"): run_pipeline(_HalfDownloadedConnector(), tmp_path / "root") - cache = settings().cache_dir / make_dataset().metadata.dataset_id + cache = settings().downloads_dir / make_dataset().metadata.dataset_id assert (cache / "first.bin").read_bytes() == b"complete" # the finished file survives the retry -def test_run_pipeline_keeps_the_cache_when_convert_fails(tmp_path, monkeypatch): +def test_run_pipeline_keeps_the_cache_when_convert_fails(tmp_path): # The rmtree runs after store_dataset, so a convert that raises keeps what download fetched. - _isolated_home(tmp_path, monkeypatch) - class _FailingConnector(_DemoConnector): def download(self, cache_dir: Path) -> list[str]: (cache_dir / "source.bin").write_bytes(b"raw") @@ -210,7 +220,7 @@ def convert(self, raw_refs: list[str]) -> TimeFDataset: with pytest.raises(RuntimeError, match="convert blew up"): run_pipeline(_FailingConnector(), tmp_path / "root") - assert (settings().cache_dir / make_dataset().metadata.dataset_id).exists() + assert (settings().downloads_dir / make_dataset().metadata.dataset_id).exists() _PARENT = DatasetRef(dataset_id=make_dataset().metadata.dataset_id, version=make_dataset().metadata.dataset_version) diff --git a/packages/timenet/tests/test_fs.py b/packages/timenet/tests/test_fs.py index c381cdb9b..0a602c72f 100644 --- a/packages/timenet/tests/test_fs.py +++ b/packages/timenet/tests/test_fs.py @@ -2,7 +2,7 @@ import pytest -from timenet._fs import publish_directory, same_directory +from timenet._fs import lies_in, publish_directory, same_directory def test_a_symlink_and_its_target_are_the_same_directory(tmp_path): @@ -11,6 +11,13 @@ def test_a_symlink_and_its_target_are_the_same_directory(tmp_path): assert same_directory(tmp_path / "link", tmp_path / "store") +def test_a_path_lies_in_the_folder_that_a_symlink_names(tmp_path): + (tmp_path / "store" / "acme").mkdir(parents=True) + (tmp_path / "link").symlink_to(tmp_path / "store" / "acme", target_is_directory=True) + assert lies_in(tmp_path / "link" / "downloads", tmp_path / "store") + assert not lies_in(tmp_path / "store", tmp_path / "link") + + def test_two_missing_paths_that_differ_only_in_case_are_different_directories(tmp_path): assert not same_directory(tmp_path / "store", tmp_path / "Store")