From 9e6fd396f2fab8fb76101f082be8f2b6d93f315f Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Wed, 4 Nov 2026 20:04:22 +0100 Subject: [PATCH 1/4] feat(cache)!: record where each release in the local registry comes from The local registry could not tell a copy of a release from a build of it. A copy is easy to get again from its registry, but a build can take hours or need credentials. So the timenet local commands could not delete the copies and keep the builds. The local registry now keeps a provenance record in the directory of each release, in the file .origin.json. LocalRegistry.record_origin() writes the record, and LocalRegistry.read_origin() reads it as an Origin. The record gives the kind of the release, an OriginKind: DOWNLOAD for a copy from a registry, or BUILD. It names the registry, or the backend package and its version, and it gives the time in UTC. A URL in the record loses its user info, query and fragment, so no credential lands in it. A damaged record raises TimeFFormatError, which names its first problem. A release without a record counts as built. run_plan() writes a record for each copy and each build, and TimeNet.download() for each release that it copies. copy_release() returns the releases that it copied or repaired. publish() writes no record, and a copy never carries one. Each registry names its location, and the InstalledBuilder of a dataset holds the package that builds it, with its version. A backend that no installed package declares raises TimeNetBuildError when its builder is looked up. The fake backends of timenet.testing come from the fake package timenet-testing 0. The new groups timenet local downloads and timenet local builds split the local registry by the record. A download is a copy from a registry. A build is a release that a connector built, or a release without a record. In each group, info lists the releases with their source and size, and clear deletes them. A clear never deletes a release that a release that stays needs: its payload sources, and the parents of a build. This rule holds in both directions. The command names each release that it keeps and a release that needs it. A damaged record, a damaged manifest of a release that stays, or a symlinked dataset or release folder stops the command before it deletes anything. timenet local info shows the size of the downloads and of the builds, and the prompt of timenet local clear counts the builds. In timenet.cache, clear_releases() takes the kinds to delete, OriginKind.DOWNLOAD, OriginKind.BUILD or both, and returns the releases that it keeps. It refuses an empty list of kinds with TimeFValidationError. releases_to_clear() gives the same answer without a delete. StoredRelease.kind tells the kind of a release, and StoredRelease.size() measures it on disk. The byte-count callback of the registries is now ByteCountCallback, so ProgressCallback means only the callback of progress events. BREAKING CHANGE: timenet.registry.base.ProgressCallback is now ByteCountCallback. Part of TNET-157. --- .../tests/test_build_plan.py | 4 + packages/timenet/src/timenet/builders.py | 22 +- packages/timenet/src/timenet/cache.py | 161 +++++++++++-- packages/timenet/src/timenet/cli/app.py | 97 +++++++- packages/timenet/src/timenet/client.py | 4 +- .../timenet/src/timenet/coordinator/build.py | 12 +- .../timenet/src/timenet/manifest/manifest.py | 23 +- .../timenet/src/timenet/registry/_copy.py | 10 +- packages/timenet/src/timenet/registry/base.py | 9 +- .../timenet/src/timenet/registry/local.py | 149 ++++++++++-- .../timenet/src/timenet/registry/origin.py | 57 +++++ .../src/timenet/registry/remote/_download.py | 8 +- .../src/timenet/registry/remote/registry.py | 9 +- packages/timenet/src/timenet/registry/s3.py | 5 + .../timenet/src/timenet/testing/backends.py | 14 +- packages/timenet/src/timenet/types/_model.py | 19 +- packages/timenet/tests/_origin_record.py | 13 + .../timenet/tests/coordinator/test_build.py | 25 ++ packages/timenet/tests/registry/test_copy.py | 16 ++ .../timenet/tests/registry/test_origin.py | 126 ++++++++++ .../timenet/tests/registry/test_registry.py | 8 + packages/timenet/tests/test_builders.py | 20 +- packages/timenet/tests/test_cache.py | 226 ++++++++++++++++-- packages/timenet/tests/test_cli.py | 114 +++++++-- packages/timenet/tests/test_client.py | 27 +++ 25 files changed, 1071 insertions(+), 107 deletions(-) create mode 100644 packages/timenet/src/timenet/registry/origin.py create mode 100644 packages/timenet/tests/_origin_record.py create mode 100644 packages/timenet/tests/registry/test_origin.py diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index c3f363a21..e11ecf8de 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -427,6 +427,10 @@ def __init__(self, root): def local_root(self): return self._local.root + @property + def location(self): + return self._local.location + def list_datasets(self): return self._local.list_datasets() diff --git a/packages/timenet/src/timenet/builders.py b/packages/timenet/src/timenet/builders.py index 8c0ad7de9..ae9984b1a 100644 --- a/packages/timenet/src/timenet/builders.py +++ b/packages/timenet/src/timenet/builders.py @@ -11,7 +11,7 @@ from importlib.metadata import EntryPoint, entry_points from typing import TYPE_CHECKING, Literal, Protocol -from timenet.errors import TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError +from timenet.errors import TimeNetBuildError, TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError from timenet.progress import ProgressCallback from timenet.types import DatasetMetadata, Version from timenet.types.references import DatasetRef @@ -112,10 +112,12 @@ class InstalledBuilder: Attributes: backend: The installed backend that builds the dataset. card: The dataset's connector card. + package: ``" "`` of the package that registers the backend. """ backend: BuilderBackend card: DatasetMetadata + package: str @property def release(self) -> DatasetRef: @@ -166,12 +168,13 @@ def installed_builder(dataset_id: str) -> InstalledBuilder | None: Raises: TimeNetInvalidCardError: If the dataset's card cannot be read. - """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend + TimeNetBuildError: If no installed distribution declares the entry point of the backend. + """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend and _package claim = next((claim for claim in claims() if claim.dataset_id == dataset_id), None) if claim is None: return None card = claim.backend.card(dataset_id) - return None if card is None else InstalledBuilder(claim.backend, card) + return None if card is None else InstalledBuilder(claim.backend, card, _package(claim)) def declared(dataset_id: str, version: Version | str | None = None) -> InstalledBuilder: @@ -213,3 +216,16 @@ def find_card(dataset_id: str) -> DatasetMetadata | None: """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend found = installed_builder(dataset_id) return None if found is None else found.card + + +def _package(claim: ConnectorClaim) -> str: + """Return ``" "`` of the package whose entry point registers the backend of a claim. + + Raises: + TimeNetBuildError: If no installed distribution declares the entry point. + """ + dist = claim.entry.dist + if dist is None: + kind = type(claim.backend) + raise TimeNetBuildError(f"the entry point of {kind.__module__}.{kind.__qualname__} names no installed package") + return f"{dist.name} {dist.version}" diff --git a/packages/timenet/src/timenet/cache.py b/packages/timenet/src/timenet/cache.py index 71fcacd7e..1abab0c7f 100644 --- a/packages/timenet/src/timenet/cache.py +++ b/packages/timenet/src/timenet/cache.py @@ -1,9 +1,9 @@ -"""Inspect and clear the cache and the stored releases. +"""Inspect and clear the cache and the downloads and builds in the local registry. Cache removal requires TimeNet's ``CACHEDIR.TAG`` marker. """ -from collections.abc import Iterable +from collections.abc import Collection, Iterable from dataclasses import dataclass from pathlib import Path @@ -12,16 +12,51 @@ from timenet.errors import TimeFValidationError from timenet.registry import LocalRegistry from timenet.registry._paths import release_dir +from timenet.registry.origin import Origin, OriginKind from timenet.types.references import DatasetRef @dataclass(frozen=True) class StoredRelease: - """A stored release with its path and size in bytes.""" + """One release in the store, with its size on disk and its provenance record.""" release: DatasetRef path: Path - size_bytes: int + origin: Origin | None + """The provenance record, or ``None`` if the release has none.""" + + def size(self) -> int: + """Measure the size on disk. + + Returns: + The size in bytes, measured on each call. + """ + return _dir_size(self.path) + + @property + def kind(self) -> OriginKind: + """The kind of the record, or :attr:`~OriginKind.BUILD` for a release without a record.""" + return OriginKind.BUILD if self.origin is None else self.origin.kind + + +@dataclass(frozen=True) +class KeptRelease: + """A release that a clear keeps, because a release that stays needs it.""" + + release: DatasetRef + """The release that stays.""" + needed_by: DatasetRef + """A release that stays and needs it, as a payload source or as the parent of a build.""" + + +@dataclass(frozen=True) +class ReleasesToClear: + """The releases that a clear deletes, and the releases of the same kinds that it keeps.""" + + releases: list[StoredRelease] + """The releases to delete, sorted by dataset ID and version.""" + kept: list[KeptRelease] + """The releases of the cleared kinds that stay, sorted by dataset ID and version.""" @dataclass(frozen=True) @@ -38,12 +73,14 @@ class ClearedCache: @dataclass(frozen=True) class ClearedReleases: - """The bytes freed and release folders removed by a release clear.""" + """The bytes freed, release folders removed and releases kept by a release clear.""" freed: int """The number of bytes freed.""" removed: list[Path] """The release folders successfully removed.""" + kept: list[KeptRelease] + """The releases of the cleared kinds that stay, because a release that stays needs them.""" def human_bytes(n: int) -> str: @@ -65,19 +102,79 @@ def human_bytes(n: int) -> str: def stored_releases(store: LocalRegistry) -> list[StoredRelease]: - """List the releases of a local registry and their sizes in bytes. + """List every release in the store, with its size and its provenance record. Args: store: The local registry to inspect. Returns: The releases, sorted by dataset ID and version. - """ - found: list[StoredRelease] = [] - for release in store.releases(): - path = release_dir(store.root, release) - found.append(StoredRelease(release, path, _dir_size(path))) - return found + + Raises: + TimeFFormatError: If a provenance record is damaged. + """ # ruff: ignore[docstring-extraneous-exception] (raised by read_origin) + return [ + StoredRelease(release, release_dir(store.root, release), store.read_origin(release)) + for release in store.releases() + ] + + +def stored_releases_by_kind(store: LocalRegistry) -> dict[OriginKind, list[StoredRelease]]: + """List stored releases with their provenance records and sizes, by kind. + + Args: + store: The local registry. + + Returns: + The releases of each kind, sorted by dataset ID and version, and an empty list for a kind without releases. + + Raises: + TimeFFormatError: If a provenance record is invalid. + """ # ruff: ignore[docstring-extraneous-exception] (raised by read_origin) + by_kind: dict[OriginKind, list[StoredRelease]] = {kind: [] for kind in OriginKind} + for item in stored_releases(store): + by_kind[item.kind].append(item) + return by_kind + + +def releases_to_clear(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> ReleasesToClear: + """Find the releases of the given kinds that no release that stays needs. + + A release that stays keeps its payload sources, and a build that stays also keeps its parents. This holds + for each release that stays, also for one that stays only because another one needs it. + + Args: + store: The local registry. + kinds: The kinds of release to delete. + + Returns: + The releases to delete, and the releases of those kinds that stay with a release that needs each. + + Raises: + TimeFValidationError: If ``kinds`` is empty, or a dataset or release folder in the local registry is a symlink. + TimeFFormatError: If a provenance record or the manifest of a release that stays is damaged. + OSError: If a folder of the store cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by check_no_linked_folders and stored_releases) + _check_kinds(kinds) + store.check_no_linked_folders() + releases = stored_releases(store) + held = {item.release: item for item in releases} + # Walk from each release that stays to the releases that it needs. If every kind goes, nothing stays. + staying = [item.release for item in releases if item.kind not in kinds] + seen = set(staying) + needed_by: dict[DatasetRef, DatasetRef] = {} + while staying: + release = staying.pop() + manifest = store.get_manifest(release.dataset_id, str(release.version)) + for need in manifest.required_releases(with_parents=held[release].kind is OriginKind.BUILD): + if need in held and need not in seen: + seen.add(need) + needed_by[need] = release + staying.append(need) + return ReleasesToClear( + [item for item in releases if item.kind in kinds and item.release not in needed_by], + [KeptRelease(item.release, needed_by[item.release]) for item in releases if item.release in needed_by], + ) def cache_size(cache_dir: Path) -> int: @@ -135,27 +232,45 @@ def clear_cache(cache_dir: Path, *, local_registry: LocalRegistry) -> ClearedCac return ClearedCache(freed, removed, [folder for folder in folders if folder not in marked]) -def clear_releases(store: LocalRegistry) -> ClearedReleases: - """Delete stored releases and the dataset directories left empty. +def clear_releases(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> ClearedReleases: + """Delete selected releases unless remaining releases need them. - Other files in the local registry stay. + Each dataset folder that this leaves empty goes too, and other files in the local registry stay. Args: store: The local registry. + kinds: The kinds of release to delete. Returns: - The bytes freed and the release folders removed. + The bytes freed, the release folders removed, and the releases kept with a release that needs each. Raises: - TimeNetRegistryError: If the local registry is read-only. - OSError: If a folder cannot be removed completely. - """ # ruff: ignore[docstring-extraneous-exception] (raised by remove_release) + TimeNetRegistryError: If the local registry is read-only. Nothing is deleted then. + TimeFValidationError: If ``kinds`` is empty, or a dataset or release folder in the local registry is a symlink. + Nothing is deleted then. + TimeFFormatError: If a provenance record or the manifest of a release that stays is damaged. + Nothing is deleted then. + OSError: If a folder cannot be read or removed completely. + """ # ruff: ignore[docstring-extraneous-exception] (raised by releases_to_clear and remove_release) + _check_kinds(kinds) + store.check_writable("a clear of its releases") + found = releases_to_clear(store, kinds=kinds) freed = 0 removed: list[Path] = [] - for release in store.releases(): - freed += store.remove_release(release) - removed.append(release_dir(store.root, release)) - return ClearedReleases(freed, removed) + for item in found.releases: + freed += store.remove_release(item.release) + removed.append(item.path) + return ClearedReleases(freed, removed, found.kept) + + +def _check_kinds(kinds: Collection[OriginKind]) -> None: + """Refuse a clear that names no kind of release. + + Raises: + TimeFValidationError: If ``kinds`` is empty. + """ + if not kinds: + raise TimeFValidationError("kinds names no kind of release") def _check_cache_deletion(cache_dir: Path, folders: Iterable[Path], *, local_registry: Path) -> None: diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 4d31eaca9..e87f96e75 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -12,7 +12,14 @@ from rich.table import Table import typer -from timenet.cache import cache_size, clear_cache, clear_releases, human_bytes, stored_releases +from timenet.cache import ( + cache_size, + clear_cache, + clear_releases, + human_bytes, + releases_to_clear, + stored_releases_by_kind, +) from timenet.cli.options import AUTO_BUILD, LOCAL_REGISTRY, REMOTE_REGISTRY from timenet.cli.runner import run_cli from timenet.cli.ui import console as ui, progress_renderer @@ -22,6 +29,7 @@ from timenet.producer import publish as publish_release from timenet.progress import ProgressCallback, ProgressEvent, StepStarted from timenet.registry import open_registry +from timenet.registry.origin import OriginKind from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access from timenet.types.parent_dataset_card import ParentDatasetCard @@ -344,30 +352,44 @@ def publish( no_args_is_help=True, ) local_app.add_typer(cache_app, name="cache") +downloads_app = typer.Typer(help="The releases in the local registry that came from a registry.", no_args_is_help=True) +local_app.add_typer(downloads_app, name="downloads") +builds_app = typer.Typer( + help="The releases in the local registry that a connector built. A release without a record counts as a build.", + no_args_is_help=True, +) +local_app.add_typer(builds_app, name="builds") _YES = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt.") +# The words of each kind in the output: the noun, and the title of its list. +_KIND_WORDS = {OriginKind.DOWNLOAD: ("download", "Downloads"), OriginKind.BUILD: ("build", "Builds")} @local_app.command("info") def local_info(local_registry: str | None = LOCAL_REGISTRY) -> None: - """Show cache and local registry paths and sizes.""" + """Show the path and size of the cache, the downloads and the builds.""" store, cache_dir = default_local_registry(local_registry=local_registry), settings().cache_dir - releases = stored_releases(store) - size = human_bytes(sum(item.size_bytes for item in releases)) + by_kind = stored_releases_by_kind(store) # Plain lines, so that a long path does not wrap. typer.echo(_cache_line(cache_dir)) - typer.echo(f"Local registry {store.root}: {_count(len(releases), 'release')}, {size}") + for kind, of_kind in by_kind.items(): + title = _KIND_WORDS[kind][1] + size = human_bytes(sum(item.size() for item in of_kind)) + typer.echo(f"{title} in {store.root}: {_count(len(of_kind), 'release')}, {size}") @local_app.command("clear") def local_clear(yes: bool = _YES, local_registry: str | None = LOCAL_REGISTRY) -> None: - """Delete the cache and every release in the local registry.""" + """Delete the cache, every download and every build.""" store, cache_dir = default_local_registry(local_registry=local_registry), settings().cache_dir if not yes: - typer.confirm(f"Delete the cache in {cache_dir} and every release in {store.root}?", abort=True) + builds = _count(len(stored_releases_by_kind(store)[OriginKind.BUILD]), "build") + typer.confirm( + f"Delete the cache in {cache_dir} and every release in {store.root}, {builds} included?", abort=True + ) # The cache clear checks the folders before it deletes anything, so it runs first. cache = clear_cache(cache_dir, local_registry=store) - releases = clear_releases(store) + releases = clear_releases(store, kinds=tuple(OriginKind)) _warn_unmarked(cache.kept) _freed(releases.freed + cache.freed, len(releases.removed) + len(cache.removed)) @@ -389,6 +411,65 @@ def cache_clear(yes: bool = _YES) -> None: _freed(cleared.freed, len(cleared.removed)) +@downloads_app.command("info") +def downloads_info(local_registry: str | None = LOCAL_REGISTRY) -> None: + """Show each download in the local registry with its registry and size.""" + _show_releases(OriginKind.DOWNLOAD, local_registry) + + +@downloads_app.command("clear") +def downloads_clear(yes: bool = _YES, local_registry: str | None = LOCAL_REGISTRY) -> None: + """Delete the downloads in the local registry, except those that a remaining release needs.""" + _clear_releases(OriginKind.DOWNLOAD, yes, local_registry) + + +@builds_app.command("info") +def builds_info(local_registry: str | None = LOCAL_REGISTRY) -> None: + """Show each build in the local registry with its connector package and size.""" + _show_releases(OriginKind.BUILD, local_registry) + + +@builds_app.command("clear") +def builds_clear(yes: bool = _YES, local_registry: str | None = LOCAL_REGISTRY) -> None: + """Delete the builds in the local registry, except those that a remaining release needs.""" + _clear_releases(OriginKind.BUILD, yes, local_registry) + + +def _show_releases(kind: OriginKind, local_registry: str | None) -> None: + """Show releases of one kind in a table, or a message if none exist.""" + store = default_local_registry(local_registry=local_registry) + releases = stored_releases_by_kind(store)[kind] + noun, title = _KIND_WORDS[kind] + if not releases: + typer.echo(f"No {noun}s in {store.root}.") + return + table = Table(title=f"{title} in {store.root}", title_justify="left", title_style="bold") + table.add_column("Dataset", style="cyan") + table.add_column("Version") + table.add_column("Source", overflow="fold") + table.add_column("Size", justify="right") + sizes = [item.size() for item in releases] # each release folder is measured once + for item, size in zip(releases, sizes, strict=True): + # A build without a record has no source. + source = "-" if item.origin is None else item.origin.source + table.add_row(item.release.dataset_id, str(item.release.version), source, human_bytes(size)) + table.add_section() + table.add_row("total", "", "", human_bytes(sum(sizes)), style="bold") + console.print(table) + + +def _clear_releases(kind: OriginKind, yes: bool, local_registry: str | None) -> None: + """Delete the releases of one kind after a prompt that counts them, and name each release that stays.""" + store = default_local_registry(local_registry=local_registry) + if not yes: + releases = releases_to_clear(store, kinds=[kind]).releases + typer.confirm(f"Delete {_count(len(releases), _KIND_WORDS[kind][0])} from {store.root}?", abort=True) + cleared = clear_releases(store, kinds=[kind]) + for item in cleared.kept: + ui.warn(f"Kept {item.release}: {item.needed_by} needs it.") + _freed(cleared.freed, len(cleared.removed)) + + def _cache_line(cache_dir: Path) -> str: """Return the path and the size of the cache.""" return f"Cache {cache_dir}: {human_bytes(cache_size(cache_dir))}" diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 4818af2ab..5d4cea036 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -355,8 +355,8 @@ def download( """Copy a dataset release and its payload sources into the local registry. The release resolves as in :meth:`open`, without a build. The release and each payload source that the - local registry lacks or holds damaged come from the first registry other than the local registry that - provides them. + store lacks or holds damaged come from the first registry other than the store that provides them, and + get a provenance record that names that registry. Args: dataset_id: The dataset ID, optionally followed by ``@version``. diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index d13243ef1..b62d30d54 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -9,6 +9,7 @@ from timenet.registry import copy_release from timenet.registry.dependencies import prepare_local_copy from timenet.registry.local import LocalRegistry +from timenet.registry.origin import OriginKind from timenet.sources import ReleaseSource from timenet.types.references import is_latest, split_ref @@ -56,7 +57,8 @@ def plan_release_build( def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = None) -> None: """Run the copy and build steps of a plan in order. - Skip read steps. + A read step runs nothing, because its release stays where it is. Each release that a copy or a + build puts into ``store`` gets a provenance record (see :mod:`timenet.registry.origin`). Args: plan: The steps to run. @@ -78,7 +80,7 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = # A read opens the release where it is, so it has nothing to run. pass case CopyStep(): - copy_release( + copies = copy_release( step.release, step.registry, store, @@ -86,6 +88,8 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = force=step.force, progress=options.progress, ) + for release in copies: + store.record_origin(release, OriginKind.DOWNLOAD, step.registry.location) case BuildStep(): _build(step, store, options=options) case _: @@ -101,12 +105,14 @@ def _build(step: BuildStep, store: LocalRegistry, *, options: BuildOptions) -> N options: The build settings. Raises: - TimeNetDatasetNotFoundError: If the result lacks the release or a payload source. + TimeNetDatasetNotFoundError: If the backend does not put the release or a payload source of it + into ``store``. """ # ruff: ignore[docstring-extraneous-exception] - raised by prepare_local_copy if options.progress is not None: options.progress(StepStarted(step.release, "build")) step.builder.backend.build(step.release, store, force=step.force, pinned_by=step.pinned_by, options=options) # After a successful build, the local registry must hold the exact release with its payload sources. prepare_local_copy(store.root, step.release) + store.record_origin(step.release, OriginKind.BUILD, step.builder.package) if options.progress is not None: options.progress(StepFinished(step.release, "build")) diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index fb1137ff9..0d48886f9 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -11,10 +11,11 @@ from timenet.manifest.files import ManifestFiles from timenet.manifest.payload_source import PayloadSource from timenet.types import DatasetMetadata, DatasetSchema -from timenet.types._model import TimeFModel +from timenet.types._model import TimeFModel, _first_problem from timenet.types.metadata_fields import DatasetId from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.reference_keys import ParentAlias, SourceKeyField +from timenet.types.references import DatasetRef from timenet.types.wire import ValueEncoding from timenet.values_backends import ValuesBackend @@ -85,6 +86,20 @@ def _one_parent_card_per_release(self) -> Self: ) return self + def required_releases(self, *, with_parents: bool) -> list[DatasetRef]: + """List the releases that this release needs: its payload sources, and its parents if asked. + + Args: + with_parents: Also list the declared parents. + + Returns: + The releases of the payload sources, then the parents. + """ + releases = [source.release for source in self.payload_sources.values()] + if with_parents: + releases.extend(self.metadata.parents.values()) + return releases + def to_dict(self) -> dict[str, Any]: """Serialize the manifest to a validated JSON-compatible dict with all keys present. @@ -135,11 +150,7 @@ def parse_manifest(text: str | bytes, where: str) -> Manifest: try: return Manifest.model_validate_json(text) except ValidationError as error: - first = error.errors()[0] - location = ".".join(str(part) for part in first["loc"]) - message = first["msg"].removeprefix("Value error, ") - detail = f"{location}: {message}" if location else message - raise TimeFFormatError(f"{where} is not a valid manifest: {detail}") from error + raise TimeFFormatError(f"{where} is not a valid manifest: {_first_problem(error)}") from error def read_manifest(path: Path) -> Manifest: diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index 1d57b575d..7b61347f2 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -28,8 +28,8 @@ def copy_release( # ruff: ignore[too-many-arguments] with_payload_sources: bool = True, force: bool = False, progress: ProgressCallback | None = None, -) -> None: - """Copy a release and its required payload sources to a registry. +) -> tuple[DatasetRef, ...]: + """Copy a release and the payload sources that it needs to another registry. Payload sources go first, a complete copy in ``dest`` is reused, and a damaged local copy is fetched again. All checks run before the first copy, so a failed check copies nothing. @@ -42,6 +42,9 @@ def copy_release( # ruff: ignore[too-many-arguments] force: Fetch ``reference`` again, even if ``dest`` has a complete local copy. progress: Receives step events for each copied release and byte counts when ``dest`` is local. + Returns: + The releases that this function copied or repaired, in the order of the copy. + Raises: TimeNetRegistryError: If ``dest`` is read-only. TimeFValidationError: If ``force`` is set and ``dest`` is not local. @@ -65,7 +68,7 @@ def copy_release( # ruff: ignore[too-many-arguments] ] _check_access(manifests, set(pending), source, dest) if not pending: - return + return () # Only a copy from a non-local source to a non-local dest downloads into this directory, each # pending release on its own. with tempfile.TemporaryDirectory(prefix="timenet-copy-", ignore_cleanup_errors=True) as staging: @@ -76,6 +79,7 @@ def copy_release( # ruff: ignore[too-many-arguments] _copy_one(ref, manifests[ref], source, dest, Path(staging), force=again, progress=progress) if progress is not None: progress(StepFinished(ref, "copy")) + return tuple(pending) def _copy_one( # ruff: ignore[too-many-arguments] diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index df1da31a6..019bbbaa4 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -29,7 +29,7 @@ # Reports download progress: called with each file (or chunk) of bytes as it lands. A ``download_release`` # implementation calls it so a caller can render a progress bar; ``None`` downloads silently. -ProgressCallback = Callable[[int], None] +ByteCountCallback = Callable[[int], None] class BaseRegistry(ABC): @@ -51,6 +51,11 @@ def read_only(self) -> bool: """Whether this registry refuses every write, set with ``read_only=True``.""" return self._read_only + @property + @abstractmethod + def location(self) -> str: + """Where this registry is: a URL or a path, else the name of the registry.""" + def check_writable(self, write: str) -> None: """Refuse a write if this registry is read-only. @@ -178,7 +183,7 @@ def download_release( *, force: bool = False, manifest: Manifest | None = None, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> None: """Download one release's files to a local directory. diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index b84d62616..eb6aae95f 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -1,11 +1,15 @@ """A registry backed by a local ``///`` directory tree.""" from collections.abc import Callable, Mapping +from datetime import UTC, datetime +import os from pathlib import Path import shutil from typing import BinaryIO +import uuid import pyarrow.fs as pafs +from pydantic import ValidationError from timenet._fs import delete_and_measure, same_directory, staged_directory from timenet.dataset import TimeFDataset @@ -15,15 +19,21 @@ from timenet.registry._paths import release_dir from timenet.registry.base import BaseRegistry from timenet.registry.dependencies import check_file_sizes, validate_manifest_reference +from timenet.registry.origin import Origin, OriginKind, without_credentials from timenet.registry.version import DatasetVersion from timenet.registry.writable import WritableRegistry from timenet.sources import FoundInRegistry, NotFound, ReleaseLookup, ReleaseSource from timenet.types import DatasetMetadata, Version +from timenet.types._model import _first_problem from timenet.types.metadata_fields import validate_dataset_id from timenet.types.references import DatasetRef, is_latest from timenet.writer import TimeFWriter, WriteProgressEvent +_ORIGIN_FILE = ".origin.json" +"""The provenance record of a release, in the folder of the release.""" + + class LocalRegistry(WritableRegistry): """Read and write dataset versions in a local directory.""" @@ -43,6 +53,11 @@ def __init__(self, root: str | Path, *, read_only: bool = False) -> None: def __str__(self) -> str: return f"local registry {self._root}" + @property + def location(self) -> str: + """The resolved root directory.""" + return str(self._root.resolve()) + @property def root(self) -> Path: """The directory this registry reads from and writes to.""" @@ -70,26 +85,72 @@ def list_datasets(self) -> list[DatasetMetadata]: return [self.get_manifest(dataset_id, str(version)).metadata for dataset_id, version in latest.items()] def releases(self) -> list[DatasetRef]: - """List the committed releases at any depth below the root. + """List the committed releases below the root. - A release is a ``//manifest.json`` outside hidden folders and unfinished writes. + A release is a ``///manifest.json`` outside hidden folders and unfinished writes. Returns: The releases, sorted by dataset ID and version. - """ - if not self._root.is_dir(): - return [] + + Raises: + OSError: If a folder cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by _walk) + return self._walk()[0] + + def check_no_linked_folders(self) -> None: + """Check that no dataset or release folder is a symlink, which :meth:`releases` would skip. + + Raises: + TimeFValidationError: If a dataset or release folder is a symlink. + OSError: If a folder cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by _walk) + linked = self._walk()[1] + if linked: + raise TimeFValidationError( + f"{linked[0]} is a symlink, so the clear cannot see the releases behind it or the " + "releases that they need. Replace the link with its folder, then clear again." + ) + + def _walk(self) -> tuple[list[DatasetRef], list[Path]]: + """Walk the dataset and release folders once, without following a symlink. + + Returns: + The releases, sorted by dataset ID and version, and the linked dataset or release folders in walk order. + + Raises: + OSError: If a folder cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by os.walk) found: list[DatasetRef] = [] - for manifest_path in self._root.rglob(MANIFEST_FILE): - parts = manifest_path.parent.relative_to(self._root).parts - # A release needs a dataset ID before its version, so a manifest at the root or one level below is none. - if not parts[:-1] or any(part.startswith(".") or ".tmp-" in part for part in parts): + linked: list[Path] = [] + if not self._root.is_dir(): + return found, linked + release_depth = 3 # // + + def fail(error: OSError) -> None: + raise error # os.walk otherwise skips a folder that it cannot read + + for parent, dirs, files in os.walk(self._root, onerror=fail): + folder = Path(parent) + parts = folder.relative_to(self._root).parts + if len(parts) == release_depth: + dirs.clear() # the walk stops at the releases + release = self._release_at_parts(parts) if MANIFEST_FILE in files else None + if release is not None: + found.append(release) continue - try: - found.append(DatasetRef.parse(f"{'/'.join(parts[:-1])}@{parts[-1]}")) - except TimeFValidationError: - continue # for example a directory whose name is not a version - return sorted(found, key=lambda release: (release.dataset_id, release.version)) + dirs[:] = sorted(name for name in dirs if not _is_hidden_or_staging(name)) + linked.extend(folder / name for name in dirs if (folder / name).is_symlink()) + return sorted(found, key=lambda release: (release.dataset_id, release.version)), linked + + def _release_at_parts(self, parts: tuple[str, ...]) -> DatasetRef | None: + """Return the release of a ``//`` folder, or ``None`` if the names are not one.""" + try: + release = DatasetRef.parse(f"{parts[0]}/{parts[1]}@{parts[2]}") + except TimeFValidationError: + return None # for example a directory whose name is not a version + if release_dir(self._root, release) != self._root.joinpath(*parts): + return None # for example 01.0.0, which parses as 1.0.0 + return release def remove_release(self, release: DatasetRef) -> int: """Delete a release and the dataset folders that this leaves empty. @@ -113,6 +174,54 @@ def remove_release(self, release: DatasetRef) -> int: parent = parent.parent return freed + def record_origin(self, release: DatasetRef, kind: OriginKind, source: str) -> None: + """Record how a stored release reached this registry, with the current time. + + The record replaces an earlier one. A URL in ``source`` loses its user info, query and fragment, so no + credential lands in the record. + + Args: + release: The release. + kind: How the release reached this registry. + source: The registry location, or the name and version of the backend package. + + Raises: + TimeNetDatasetNotFoundError: If this registry does not hold the release. + """ + if not self.exists(release.dataset_id, str(release.version)): + raise TimeNetDatasetNotFoundError(f"the local registry holds no {release}, so it gets no provenance record") + origin = Origin(kind=kind, source=without_credentials(source), at=datetime.now(UTC).replace(microsecond=0)) + directory = release_dir(self._root, release) + # Write next to the record, then rename, so a reader never sees half a record. + staging = directory / f"{_ORIGIN_FILE}.tmp-{uuid.uuid4().hex}" + try: + staging.write_text(origin.model_dump_json() + "\n") + staging.replace(directory / _ORIGIN_FILE) + finally: + staging.unlink(missing_ok=True) + + def read_origin(self, release: DatasetRef) -> Origin | None: + """Read the provenance record of a stored release. + + Args: + release: The release. + + Returns: + The record, or ``None`` if the release has none. + + Raises: + TimeFFormatError: If the record is invalid. + """ + path = release_dir(self._root, release) / _ORIGIN_FILE + try: + text = path.read_text() + except FileNotFoundError: + return None + try: + return Origin.model_validate_json(text) + except ValidationError as error: + raise TimeFFormatError(f"{path} is not a valid provenance record: {_first_problem(error)}.") from error + def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest: """Return a dataset's manifest (latest version if unspecified). @@ -368,6 +477,18 @@ def is_same_registry(source: ReleaseSource, registry: BaseRegistry) -> bool: ) +def _is_hidden_or_staging(name: str) -> bool: + """Check whether a folder name is hidden or names an unfinished write. + + Args: + name: The folder name. + + Returns: + ``True`` if the name starts with ``.`` or holds ``.tmp-``. + """ + return name.startswith(".") or ".tmp-" in name + + def _is_version(name: str) -> bool: """Return whether a directory name is a parseable version. diff --git a/packages/timenet/src/timenet/registry/origin.py b/packages/timenet/src/timenet/registry/origin.py new file mode 100644 index 000000000..f9c2267db --- /dev/null +++ b/packages/timenet/src/timenet/registry/origin.py @@ -0,0 +1,57 @@ +"""The provenance record: where and when each release in the local registry was downloaded or built. + +:class:`~timenet.registry.LocalRegistry` writes and reads the record of each release. The record stays in the +local registry when the release is copied elsewhere. +""" + +from datetime import UTC, datetime +from enum import Enum +from urllib.parse import urlsplit, urlunsplit + +from pydantic import AwareDatetime, BaseModel, ConfigDict, field_validator + + +class OriginKind(Enum): + """How a release reached the local registry.""" + + DOWNLOAD = "download" + """A copy from a registry.""" + BUILD = "build" + """A build by an installed connector.""" + + +class Origin(BaseModel): + """The kind, source, and time of a stored release's download or build.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: OriginKind + """How the release reached the local registry.""" + source: str + """The registry location, or the name and version of the backend package.""" + at: AwareDatetime + """The time that the release reached the local registry, in UTC.""" + + @field_validator("at") + @classmethod + def _in_utc(cls, value: datetime) -> datetime: + """Return the time in UTC.""" + return value.astimezone(UTC) + + +def without_credentials(location: str) -> str: + """Remove the user info, the query and the fragment of a URL. + + A string that is not a URL with a host, such as a path, comes back unchanged. + + Args: + location: A URL, a path, or another name. + + Returns: + The location without credentials. + """ + parts = urlsplit(location) + if not (parts.scheme and parts.netloc): + return location + host = parts.netloc.rpartition("@")[2] + return urlunsplit((parts.scheme, host, parts.path, "", "")) diff --git a/packages/timenet/src/timenet/registry/remote/_download.py b/packages/timenet/src/timenet/registry/remote/_download.py index 093a7089b..572b5601b 100644 --- a/packages/timenet/src/timenet/registry/remote/_download.py +++ b/packages/timenet/src/timenet/registry/remote/_download.py @@ -21,7 +21,7 @@ from timenet.manifest import Manifest from timenet.manifest.files import FilePart from timenet.registry._paths import safe_version_path -from timenet.registry.base import ProgressCallback +from timenet.registry.base import ByteCountCallback from timenet.registry.remote._http import ( RegistryHttpClient, _download_path, @@ -45,7 +45,7 @@ def download_version_files( # ruff: ignore[too-many-arguments] *, force: bool = False, max_concurrency: int = 8, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> Path: """Download every file of a version into ``dest_dir`` atomically. @@ -79,7 +79,7 @@ async def _download_all( # ruff: ignore[too-many-arguments, too-many-positional files: list[FilePart], staging: Path, max_concurrency: int, - progress_cb: ProgressCallback | None, + progress_cb: ByteCountCallback | None, ) -> None: """Download all files concurrently into the staging directory. @@ -143,7 +143,7 @@ async def _stream_one( # ruff: ignore[too-many-arguments] *, release: DatasetRef, record: FilePart, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> None: """Stream one presigned URL to ``dest`` via a ``.part`` temp file, renamed on success. diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index e439268e3..757b50757 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -17,7 +17,7 @@ from timenet.format.constants import MANIFEST_FILE from timenet.manifest import Manifest, parse_manifest from timenet.registry.lazy_read import LazyReadRegistry -from timenet.registry.remote._download import ProgressCallback, download_version_files +from timenet.registry.remote._download import ByteCountCallback, download_version_files from timenet.registry.remote._http import RegistryHttpClient, _reaching from timenet.registry.remote.filesystem import RegistryFileSystem from timenet.registry.remote.models import ( @@ -58,6 +58,11 @@ def __init__( def __str__(self) -> str: return f"registry {self._base_url}" + @property + def location(self) -> str: + """The base URL of the service.""" + return self._base_url + def list_datasets(self) -> list[DatasetMetadata]: """Return the latest-version metadata of every dataset, sorted by id. @@ -137,7 +142,7 @@ def download_release( *, force: bool = False, manifest: Manifest | None = None, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> None: """Download one release's files to a local directory. diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index b9b227a14..7872ed94a 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -62,6 +62,11 @@ def __init__(self, uri: str, *, read_only: bool = False) -> None: def __str__(self) -> str: return f"S3 registry {self._display_root()}" + @property + def location(self) -> str: + """The ``s3://bucket/prefix`` root.""" + return self._display_root() + def list_datasets(self) -> list[DatasetMetadata]: """Unsupported: the S3 backend has no catalog. diff --git a/packages/timenet/src/timenet/testing/backends.py b/packages/timenet/src/timenet/testing/backends.py index 71f4ece70..a421b2b5a 100644 --- a/packages/timenet/src/timenet/testing/backends.py +++ b/packages/timenet/src/timenet/testing/backends.py @@ -1,7 +1,7 @@ """A fake set of installed builder backends.""" from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING from timenet import builders @@ -11,12 +11,20 @@ import pytest +@dataclass(frozen=True) +class _FakeDistribution: + """The distribution that declares each fake entry point.""" + + name: str = "timenet-testing" + version: str = "0" + + @dataclass(frozen=True) class _FakeEntryPoint: - """An entry point that no installed distribution declares.""" + """An entry point of the fake distribution ``timenet-testing 0``.""" backend: object - dist: None = None + dist: _FakeDistribution = field(default_factory=_FakeDistribution) def load(self) -> Callable[[], object]: """Return the factory of the backend.""" diff --git a/packages/timenet/src/timenet/types/_model.py b/packages/timenet/src/timenet/types/_model.py index 6157bec07..1963529da 100644 --- a/packages/timenet/src/timenet/types/_model.py +++ b/packages/timenet/src/timenet/types/_model.py @@ -1,8 +1,8 @@ -"""Shared configuration for immutable TimeF data models.""" +"""Shared configuration for immutable TimeF data models, and the text of their validation errors.""" from typing import Any -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, ValidationError class TimeFModel(BaseModel): @@ -24,3 +24,18 @@ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]: The rebuild callable and its arguments: this class and the serialized fields. """ return type(self).model_validate, (self.model_dump(),) + + +def _first_problem(error: ValidationError) -> str: + """Describe the first problem of a validation error as ``: ``. + + Args: + error: The validation error. + + Returns: + The location and message of the first problem, or the message alone if it has no location. + """ + first = error.errors()[0] + location = ".".join(str(part) for part in first["loc"]) + message = first["msg"].removeprefix("Value error, ") + return f"{location}: {message}" if location else message diff --git a/packages/timenet/tests/_origin_record.py b/packages/timenet/tests/_origin_record.py new file mode 100644 index 000000000..f01dd783c --- /dev/null +++ b/packages/timenet/tests/_origin_record.py @@ -0,0 +1,13 @@ +"""The file of a release's provenance record, for tests that damage, delete or read it.""" + +from pathlib import Path + +from timenet.registry import LocalRegistry +from timenet.registry._paths import release_dir +from timenet.registry.local import _ORIGIN_FILE +from timenet.types.references import DatasetRef + + +def origin_file(store: LocalRegistry, release: DatasetRef) -> Path: + """Return the file of the provenance record of a release in a local registry.""" + return release_dir(store.root, release) / _ORIGIN_FILE diff --git a/packages/timenet/tests/coordinator/test_build.py b/packages/timenet/tests/coordinator/test_build.py index 4236a1a74..70cd4310c 100644 --- a/packages/timenet/tests/coordinator/test_build.py +++ b/packages/timenet/tests/coordinator/test_build.py @@ -13,6 +13,7 @@ from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version from timenet.registry._paths import release_dir +from timenet.registry.origin import OriginKind from timenet.sources import InstalledConnectors from timenet.testing import install_backends, make_dataset from timenet.types import Access @@ -201,6 +202,30 @@ def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( assert store.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) +def test_run_plan_records_each_copy_and_each_build(tmp_path, store, backends): + # The registry to copy from holds the child and its payload source, the parent. + origin = LocalRegistry(tmp_path / "origin") + _store_child_without_parent(origin) + origin.put_dataset(make_dataset()) + backends.extend([_Backend(_child_card()), _Backend(_grandchild_card())]) + plan = plan_release_build(GRANDCHILD_REF.dataset_id, store=store, parents_from=[origin]) + assert [(step.action, step.release) for step in plan.steps] == [("copy", CHILD_REF), ("build", GRANDCHILD_REF)] + + run_plan(plan, store) + + records = {} + for release in (PARENT_REF, CHILD_REF, GRANDCHILD_REF): + record = store.read_origin(release) + records[release] = None if record is None else (record.kind, record.source) + # The copy of the child brings its payload source along, and both name the registry. The build + # names the fake distribution of the fake backend. + assert records == { + PARENT_REF: (OriginKind.DOWNLOAD, str(origin.root.resolve())), + CHILD_REF: (OriginKind.DOWNLOAD, str(origin.root.resolve())), + GRANDCHILD_REF: (OriginKind.BUILD, "timenet-testing 0"), + } + + def test_a_forced_build_forces_only_the_requested_release(store, fake_parent, fake_child): store.put_dataset(_demo(_child_card())) diff --git a/packages/timenet/tests/registry/test_copy.py b/packages/timenet/tests/registry/test_copy.py index 37b1e2628..871a76098 100644 --- a/packages/timenet/tests/registry/test_copy.py +++ b/packages/timenet/tests/registry/test_copy.py @@ -25,6 +25,7 @@ from timenet.progress import BytesMoved, ProgressEvent, StepFinished, StepStarted from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, WritableRegistry, copy_release, open_local_version +from timenet.registry._paths import release_dir from timenet.registry.shared import complete_copy from timenet.testing import make_dataset from timenet.types import Access, Annotation, DatasetMetadata, DatasetSchema, License, TimeSeriesSpec, Version, ureg @@ -406,6 +407,21 @@ def test_copy_release_without_payload_sources_copies_the_release_alone(chain_sou assert not dest.exists(*chain[0]) +def test_copy_release_returns_the_releases_that_it_copied(chain_source, dest): + copy_release(grandparent_ref, chain_source, dest) + # Payload sources come first. dest holds the grandparent already, so the tuple leaves it out. + assert copy_release(grandchild_ref, chain_source, dest) == (middle_ref, grandchild_ref) + assert copy_release(grandchild_ref, chain_source, dest) == () + + +def test_copy_release_returns_each_release_that_it_repairs_or_copies_again(source, tmp_path): + dest = LocalRegistry(tmp_path / "dest") + copy_release(child_ref, source, dest) + (release_dir(dest.root, parent_ref) / dest.get_manifest(*parent).files.control[0].path).unlink() + assert copy_release(child_ref, source, dest) == (parent_ref,) + assert copy_release(child_ref, source, dest, force=True) == (child_ref,) + + def test_copy_release_copies_a_shared_payload_source_once(tmp_path): root = tmp_path / "chain" _write_chain(root, grandchild_reads_grandparent=True) diff --git a/packages/timenet/tests/registry/test_origin.py b/packages/timenet/tests/registry/test_origin.py new file mode 100644 index 000000000..6a96c72b6 --- /dev/null +++ b/packages/timenet/tests/registry/test_origin.py @@ -0,0 +1,126 @@ +"""The provenance record says where each release in the store comes from, and stays in the store.""" + +from datetime import UTC, datetime +import json + +from _demo_child import _child_of_the_demo +from _origin_record import origin_file +from pydantic import ValidationError +import pytest + +import timenet +from timenet.errors import TimeFFormatError, TimeNetDatasetNotFoundError +from timenet.registry import LocalRegistry +from timenet.registry.origin import Origin, OriginKind, without_credentials +from timenet.testing import make_dataset +from timenet.types.references import DatasetRef + + +DEMO = DatasetRef(dataset_id=make_dataset().metadata.dataset_id, version=make_dataset().metadata.dataset_version) + + +@pytest.fixture +def store(tmp_path) -> LocalRegistry: + registry = LocalRegistry(tmp_path / "store") + registry.put_dataset(make_dataset()) + return registry + + +def _read(store: LocalRegistry) -> Origin: + """Return the record of the demo release, which must have one.""" + origin = store.read_origin(DEMO) + assert origin is not None + return origin + + +def test_a_record_round_trips(store): + before = datetime.now(UTC).replace(microsecond=0) + store.record_origin(DEMO, OriginKind.BUILD, "timenet-connectors 0.9.0") + origin = _read(store) + + assert (origin.kind, origin.source) == (OriginKind.BUILD, "timenet-connectors 0.9.0") + assert origin.at.tzinfo is UTC + assert before <= origin.at <= datetime.now(UTC) + # The file holds the three keys, with the time in UTC to the second. + record = json.loads(origin_file(store, DEMO).read_text()) + assert record == { + "kind": "build", + "source": "timenet-connectors 0.9.0", + "at": origin.at.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +def test_a_new_record_replaces_the_old_one(store): + store.record_origin(DEMO, OriginKind.DOWNLOAD, "https://registry.example.org") + store.record_origin(DEMO, OriginKind.BUILD, "timenet-connectors 0.9.0") + assert _read(store).kind is OriginKind.BUILD + # The write leaves no temporary file next to the record. + path = origin_file(store, DEMO) + assert list(path.parent.glob(f"{path.name}*")) == [path] + + +@pytest.mark.parametrize( + ("location", "expected"), + [ + ("https://user:secret@registry.example.org:8443/r?sig=abc#top", "https://registry.example.org:8443/r"), + ("s3://bucket/prefix?versionId=1", "s3://bucket/prefix"), + ("https://registry.example.org", "https://registry.example.org"), + ("/data/registry", "/data/registry"), + ("timenet-connectors 0.9.0", "timenet-connectors 0.9.0"), + ], +) +def test_without_credentials_keeps_only_scheme_host_and_path_of_a_url(location, expected): + assert without_credentials(location) == expected + + +def test_a_release_without_a_record_has_none(store): + assert store.read_origin(DEMO) is None + + +def test_a_record_needs_the_release_in_the_store(tmp_path): + empty = LocalRegistry(tmp_path / "store") + with pytest.raises(TimeNetDatasetNotFoundError, match=f"the local registry holds no {DEMO}"): + empty.record_origin(DEMO, OriginKind.DOWNLOAD, "https://registry.example.org") + + +@pytest.mark.parametrize( + ("text", "problem"), + [ + ("not json", "Invalid JSON: expected ident at line 1 column 2"), + ( + '{"kind": "other", "source": "x", "at": "2026-11-03T14:02:11Z"}', + "kind: Input should be 'download' or 'build'", + ), + ], + ids=["invalid-json", "other-kind"], +) +def test_a_damaged_record_raises_with_its_first_problem(store, text, problem): + path = origin_file(store, DEMO) + path.write_text(text) + with pytest.raises(TimeFFormatError) as caught: + store.read_origin(DEMO) + assert str(caught.value) == f"{path} is not a valid provenance record: {problem}." + assert isinstance(caught.value.__cause__, ValidationError) + + +def test_a_time_with_another_offset_reads_as_utc(store): + path = origin_file(store, DEMO) + path.write_text('{"kind": "download", "source": "x", "at": "2026-11-03T16:02:11+02:00"}') + at = _read(store).at + assert at == datetime(2026, 11, 3, 14, 2, 11, tzinfo=UTC) + assert at.tzinfo is UTC + + +def test_publish_never_copies_the_record(tmp_path): + store = LocalRegistry(tmp_path / "store") + _, parent = _child_of_the_demo(store.root) + child = DatasetRef.parse("test/child@1.0.0") + store.record_origin(parent, OriginKind.DOWNLOAD, "https://registry.example.org") + store.record_origin(child, OriginKind.BUILD, "timenet-connectors 0.9.0") + + timenet.publish(str(child), dest=tmp_path / "dest", source=store) + + dest = LocalRegistry(tmp_path / "dest") + assert dest.exists(child.dataset_id, str(child.version)) + assert dest.exists(parent.dataset_id, str(parent.version)) + assert not list(dest.root.rglob(f"{origin_file(dest, child).name}*")) diff --git a/packages/timenet/tests/registry/test_registry.py b/packages/timenet/tests/registry/test_registry.py index a4ea35ec1..bfc1bbfab 100644 --- a/packages/timenet/tests/registry/test_registry.py +++ b/packages/timenet/tests/registry/test_registry.py @@ -300,6 +300,14 @@ def test_put_release_refuses_a_directory_without_a_readable_manifest(tmp_path, m # ---- s3 catalog ------------------------------------------------------------------------------- +def test_each_registry_names_its_location(tmp_path): + assert RemoteRegistry("https://registry.example.test").location == "https://registry.example.test" + assert open_registry("timenet://").location == TIMENET_REGISTRY_URL + assert S3Registry("s3://bucket/prefix").location == "s3://bucket/prefix" + assert S3Registry("s3://bucket").location == "s3://bucket" + assert LocalRegistry(tmp_path / "other" / ".." / "store").location == str((tmp_path / "store").resolve()) + + def test_s3_registry_has_no_catalog(): # The S3 backend does store + download by explicit id, but has no catalog: list/search raise. s3 = S3Registry("s3://bucket/registry") diff --git a/packages/timenet/tests/test_builders.py b/packages/timenet/tests/test_builders.py index 1f1727342..d93f42af1 100644 --- a/packages/timenet/tests/test_builders.py +++ b/packages/timenet/tests/test_builders.py @@ -1,7 +1,10 @@ +from types import SimpleNamespace + import pytest +from timenet import builders from timenet.builders import declared, installed_builder -from timenet.errors import TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError +from timenet.errors import TimeNetBuildError, TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError from timenet.testing import install_backends, make_dataset from timenet.types.references import DatasetRef @@ -57,3 +60,18 @@ def test_declared_returns_the_release_of_the_card_and_refuses_another_version(mo ) with pytest.raises(TimeNetDatasetNotFoundError, match="no installed connector builds 'x/y'"): declared("x/y") + + +def test_an_installed_builder_names_the_distribution_of_the_backend(monkeypatch): + install_backends(monkeypatch, _Backend("x/y", card=make_dataset().metadata)) + builder = installed_builder("x/y") + assert builder is not None + assert builder.package == "timenet-testing 0" + + +def test_a_backend_without_a_distribution_gives_no_installed_builder(monkeypatch): + backend = _Backend("a/b", card=make_dataset().metadata) + entry = SimpleNamespace(dist=None, load=lambda: lambda: backend) + monkeypatch.setattr(builders, "entry_points", lambda group: [entry]) + with pytest.raises(TimeNetBuildError, match=f"the entry point of {__name__}._Backend names no installed package"): + installed_builder("a/b") diff --git a/packages/timenet/tests/test_cache.py b/packages/timenet/tests/test_cache.py index 9a391553f..87100611e 100644 --- a/packages/timenet/tests/test_cache.py +++ b/packages/timenet/tests/test_cache.py @@ -1,24 +1,39 @@ from pathlib import Path +from _demo_child import _child_of_the_demo +from _origin_record import origin_file import pytest from timenet.cache import ( ClearedCache, + KeptRelease, StoredRelease, cache_size, clear_cache, clear_releases, human_bytes, + releases_to_clear, stored_releases, ) +from timenet.client import TimeNet from timenet.config import CACHE_TAG, is_cache_folder, make_cache_folder -from timenet.errors import TimeFValidationError +from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetRegistryError from timenet.registry import LocalRegistry +from timenet.registry._paths import release_dir +from timenet.registry.origin import OriginKind from timenet.testing import make_dataset from timenet.types import Version +from timenet.types.reference_keys import ParentAlias from timenet.types.references import DatasetRef +PARENT = DatasetRef.parse("test/child@1.0.0") +GRAND = DatasetRef.parse("timenet/hello-world@1.0.0") +BUILT = DatasetRef.parse("test/built@1.0.0") +OTHER = DatasetRef.parse("demo/other@1.0.0") +HAND = DatasetRef.parse("demo/hand@1.0.0") + + def _store_with_two_releases(root): """Return a store that holds timenet/hello-world at 1.0.0 and 1.1.0.""" store = LocalRegistry(root) @@ -29,10 +44,40 @@ def _store_with_two_releases(root): return store -def _other_dataset(): - dataset = make_dataset() - dataset._metadata = dataset.metadata.model_copy(update={"dataset_id": "demo/other"}) - return dataset +def test_clear_releases_refuses_a_read_only_local_registry(tmp_path): + store = LocalRegistry(_store_with_two_releases(tmp_path / "store").root, read_only=True) + before = stored_releases(store) + with pytest.raises(TimeNetRegistryError, match=r"read_only=True, so it refuses a clear of its releases\."): + clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) + assert stored_releases(store) == before + + +def _layout(store): + """Store the five releases of the layout. + + PARENT reads GRAND as a payload source, BUILT declares PARENT as a parent, and OTHER and HAND + stand alone. + """ + _child_of_the_demo(store.root) + store.put_dataset(make_dataset(dataset_id=BUILT.dataset_id, parents={ParentAlias("parent"): PARENT})) + store.put_dataset(make_dataset(dataset_id=OTHER.dataset_id)) + store.put_dataset(make_dataset(dataset_id=HAND.dataset_id)) + + +def _record(store): + """Record PARENT, GRAND and OTHER as copies and BUILT as a build. HAND gets no record.""" + for release in (PARENT, GRAND, OTHER): + store.record_origin(release, OriginKind.DOWNLOAD, "https://registry.example.org") + store.record_origin(BUILT, OriginKind.BUILD, "timenet-connectors 0.9.0") + + +def _held(store): + """Return the releases of the layout that the store holds.""" + return { + release + for release in (BUILT, PARENT, GRAND, OTHER, HAND) + if store.exists(release.dataset_id, str(release.version)) + } def _size(directory): @@ -91,26 +136,48 @@ def test_human_bytes(n, expected): def test_stored_releases_lists_each_release_with_its_size(tmp_path): store = _store_with_two_releases(tmp_path / "store") first, second = (tmp_path / "store/timenet/hello-world" / version for version in ("1.0.0", "1.1.0")) - assert stored_releases(store) == [ - StoredRelease(DatasetRef.parse("timenet/hello-world@1.0.0"), first, _size(first)), - StoredRelease(DatasetRef.parse("timenet/hello-world@1.1.0"), second, _size(second)), + releases = stored_releases(store) + assert releases == [ + StoredRelease(DatasetRef.parse("timenet/hello-world@1.0.0"), first, None), + StoredRelease(DatasetRef.parse("timenet/hello-world@1.1.0"), second, None), ] + assert [item.size() for item in releases] == [_size(first), _size(second)] assert _size(first) > 0 +def test_stored_releases_carry_their_origin(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + releases = stored_releases(store) + assert {item.release: item.origin for item in releases} == { + release: store.read_origin(release) for release in (BUILT, PARENT, GRAND, OTHER, HAND) + } + # A release without a record counts as a build. + assert {item.release: item.kind for item in releases} == { + BUILT: OriginKind.BUILD, + HAND: OriginKind.BUILD, + PARENT: OriginKind.DOWNLOAD, + GRAND: OriginKind.DOWNLOAD, + OTHER: OriginKind.DOWNLOAD, + } + + def test_stored_releases_skips_what_is_not_a_release(tmp_path): store = _store_with_two_releases(tmp_path / "store") manifest = (tmp_path / "store/timenet/hello-world/1.0.0/manifest.json").read_bytes() - # A directory whose name is not a version, a staging directory, a hidden one, a path that is not - # a dataset ID, and a manifest at the root and one level below. - for stray in ( + # A directory whose name is not a version, a version in another spelling, a staging directory, a + # hidden one, a path that is not a dataset ID, and a manifest at the root and one level below. + strays = ( "timenet/hello-world/draft", + "timenet/hello-world/01.0.0", "timenet/hello-world/2.0.0.tmp-1", ".trash/x/1.0.0", "a/b/c/1.0.0", ".", "1.0.0", - ): + ) + for stray in strays: (tmp_path / "store" / stray).mkdir(parents=True, exist_ok=True) (tmp_path / "store" / stray / "manifest.json").write_bytes(manifest) assert [str(item.release) for item in stored_releases(store)] == [ @@ -119,6 +186,16 @@ def test_stored_releases_skips_what_is_not_a_release(tmp_path): ] +def test_releases_skips_an_unfinished_write_of_a_dataset_folder(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + # The staging folder of a dataset has a name that is also a valid dataset ID. + staging = store.root / "timenet/hello-world.tmp-1/1.0.0" + staging.mkdir(parents=True) + (staging / "manifest.json").write_bytes((release_dir(store.root, GRAND) / "manifest.json").read_bytes()) + assert [str(release) for release in store.releases()] == ["timenet/hello-world@1.0.0"] + + def test_clear_cache_keeps_a_store_inside_the_cache(tmp_path): cache = tmp_path / "cache" store = LocalRegistry(cache / "registry") @@ -161,18 +238,22 @@ def test_clear_cache_keeps_and_reports_the_folders_without_the_marker(tmp_path): def test_clear_releases_deletes_every_release_but_no_other_file(tmp_path): store = _store_with_two_releases(tmp_path / "store") - store.put_dataset(_other_dataset()) + store.put_dataset(make_dataset(dataset_id=OTHER.dataset_id)) + # A download goes as well as the builds. + store.record_origin(OTHER, OriginKind.DOWNLOAD, "https://registry.example.org") notes = tmp_path / "store/notes.txt" notes.write_text("kept") staging = tmp_path / "store/timenet/hello-world/2.0.0.tmp-1/part.bin" staging.parent.mkdir() staging.write_bytes(b"kept") releases = stored_releases(store) + size = sum(item.size() for item in releases) - cleared = clear_releases(store) + cleared = clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) - assert cleared.freed == sum(item.size_bytes for item in releases) + assert cleared.freed == size assert cleared.removed == [item.path for item in releases] + assert cleared.kept == [] assert stored_releases(store) == [] # The dataset directories that the call emptied go, and the files that are not releases stay. assert not (tmp_path / "store/demo").exists() @@ -266,6 +347,121 @@ def no_home(): assert not raw.exists() +def test_clearing_the_downloads_keeps_each_download_that_a_build_needs(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + cleared = clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert _held(store) == {BUILT, PARENT, GRAND, HAND} + # BUILT declares PARENT as a parent, and PARENT reads files of GRAND. + assert cleared.kept == [KeptRelease(PARENT, BUILT), KeptRelease(GRAND, PARENT)] + + +def test_clearing_the_downloads_deletes_a_parent_that_only_a_download_declares(tmp_path): + # A download needs its payload sources to be read, but not its parents. + store = LocalRegistry(tmp_path / "store") + copy = DatasetRef.parse("test/copy@1.0.0") + store.put_dataset(make_dataset(dataset_id=OTHER.dataset_id)) + store.put_dataset(make_dataset(dataset_id=copy.dataset_id, parents={ParentAlias("parent"): OTHER})) + store.put_dataset(make_dataset(dataset_id=BUILT.dataset_id, parents={ParentAlias("parent"): copy})) + for release in (OTHER, copy): + store.record_origin(release, OriginKind.DOWNLOAD, "https://registry.example.org") + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert [item.release for item in stored_releases(store)] == [BUILT, copy] + + +def test_clearing_the_builds_keeps_each_build_that_a_download_needs(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + # PARENT, a download, reads files of GRAND, a build. + store.record_origin(GRAND, OriginKind.BUILD, "timenet-connectors 0.9.0") + cleared = clear_releases(store, kinds=[OriginKind.BUILD]) + assert _held(store) == {PARENT, GRAND, OTHER} + assert cleared.kept == [KeptRelease(GRAND, PARENT)] + + +def test_clearing_every_kind_deletes_every_release_without_reading_a_manifest(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + # Nothing stays, so no release needs another. + (release_dir(store.root, BUILT) / "manifest.json").write_text("{}") + releases = stored_releases(store) + + cleared = clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) + + assert cleared.removed == [item.path for item in releases] + assert cleared.kept == [] + assert list((tmp_path / "store").iterdir()) == [] + + +def test_a_release_clear_with_a_damaged_record_deletes_nothing(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + origin_file(store, OTHER).write_text("{}") + with pytest.raises(TimeFFormatError, match="is not a valid provenance record"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} + + +def test_clearing_the_downloads_with_a_damaged_manifest_of_a_build_deletes_nothing(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + (release_dir(store.root, BUILT) / "manifest.json").write_text("{}") + with pytest.raises(TimeFFormatError, match="is not a valid manifest"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert {item.release for item in stored_releases(store)} == {BUILT, PARENT, GRAND, OTHER, HAND} + + +def test_clearing_the_downloads_refuses_a_store_with_a_symlinked_dataset_folder(tmp_path): + # The listing skips a symlinked folder, so it would miss BUILT and delete the downloads that it needs. + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + moved = tmp_path / "elsewhere/built" + moved.parent.mkdir() + (store.root / "test/built").rename(moved) + (store.root / "test/built").symlink_to(moved, target_is_directory=True) + with pytest.raises(TimeFValidationError, match="test/built is a symlink"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} + + +def test_clearing_every_kind_refuses_a_store_with_a_symlinked_release_folder(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + moved = tmp_path / "elsewhere/1.0.0" + moved.parent.mkdir() + release_dir(store.root, OTHER).rename(moved) + release_dir(store.root, OTHER).symlink_to(moved, target_is_directory=True) + with pytest.raises(TimeFValidationError, match=r"demo/other/1\.0\.0 is a symlink"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) + assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} + # The link hides the release from the listing, not from a read of its exact version. + with TimeNet(sources=[store]).open(str(OTHER)) as reader: + assert reader.metadata.dataset_id == OTHER.dataset_id + + +@pytest.mark.parametrize("clear", [releases_to_clear, clear_releases]) +def test_a_release_clear_refuses_an_empty_list_of_kinds_first(tmp_path, clear): + # A read-only local registry refuses a clear too, so the kinds must come first. + store = LocalRegistry(tmp_path / "store", read_only=True) + with pytest.raises(TimeFValidationError, match=r"^kinds names no kind of release$"): + clear(store, kinds=[]) + + +def test_the_symlink_check_skips_a_hidden_folder(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + (tmp_path / "elsewhere").mkdir() + (store.root / ".trash").symlink_to(tmp_path / "elsewhere", target_is_directory=True) + store.check_no_linked_folders() + + def test_clear_cache_of_a_missing_cache_frees_nothing(tmp_path): assert cache_size(tmp_path / "cache") == 0 cleared = clear_cache(tmp_path / "cache", local_registry=LocalRegistry(tmp_path / "store")) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index f943a6dcb..87658e067 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -23,6 +23,7 @@ ) from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version +from timenet.registry.origin import OriginKind from timenet.testing import install_backends, make_dataset from timenet.types import Access, Domain from timenet.types.reference_keys import ParentAlias @@ -70,35 +71,92 @@ def state(tmp_path, monkeypatch): return store, cache +def _put(store, dataset, kind): + """Put ``dataset`` into the store, with a provenance record of ``kind``. + + Returns: + The release. + """ + registry = LocalRegistry(store) + release = registry.put_dataset(dataset) + registry.record_origin(release, kind, f"https://{kind.value}.example.org") + return release + + +def _stderr_has(result, text): + """Return whether stderr holds ``text``. + + The console wraps a long line, also in a path, so the check ignores whitespace. + """ + return "".join(text.split()) in "".join(click.unstyle(result.stderr).split()) + + def test_local_info_and_local_cache_info_show_the_paths_and_sizes(state): store, cache = state + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) result = runner.invoke(app, ["local", "info"]) assert result.exit_code == 0, result.output - [release] = stored_releases(LocalRegistry(store)) + copy, build = stored_releases(LocalRegistry(store)) cache_line = f"Cache {cache}: {human_bytes(_downloads_size(cache))}" assert click.unstyle(result.stdout).splitlines() == [ cache_line, - f"Local registry {store}: 1 release, {human_bytes(release.size_bytes)}", + f"Downloads in {store}: 1 release, {human_bytes(copy.size())}", + f"Builds in {store}: 1 release, {human_bytes(build.size())}", ] result = runner.invoke(app, ["local", "cache", "info"]) assert result.exit_code == 0, result.output assert click.unstyle(result.stdout) == f"{cache_line}\n" +@pytest.mark.parametrize( + ("command", "rows"), + [ + (["local", "downloads", "info"], [["demo/copy", "1.0.0", "https://download.example.org"]]), + ( + ["local", "builds", "info"], + [["demo/built", "1.0.0", "https://build.example.org"], ["timenet/hello-world", "1.0.0", "-"]], + ), + ], +) +def test_downloads_info_and_builds_info_list_the_releases_of_their_kind(state, command, rows): + store, _ = state + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + _put(store, make_dataset(dataset_id="demo/built"), OriginKind.BUILD) + result = runner.invoke(app, command) + assert result.exit_code == 0, result.output + cells = _table_cells(click.unstyle(result.stdout)) + sizes = {item.release.dataset_id: item.size() for item in stored_releases(LocalRegistry(store))} + total = sum(sizes[row[0]] for row in rows) + # The header uses another border, so the cells are the body rows. A build without a record has no source. + assert cells == [*([*row, human_bytes(sizes[row[0]])] for row in rows), ["total", "", "", human_bytes(total)]] + + +def test_downloads_info_of_a_local_registry_without_downloads(state): + store, _ = state + result = runner.invoke(app, ["local", "downloads", "info"]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.stdout) == f"No downloads in {store}.\n" + + @pytest.mark.parametrize( ("command", "prompt"), [ - (["local", "clear"], "Delete the cache in {cache} and every release in {store}?"), + (["local", "clear"], "Delete the cache in {cache} and every release in {store}, 2 builds included?"), (["local", "cache", "clear"], "Delete the raw source files and control databases in {cache}?"), + (["local", "downloads", "clear"], "Delete 1 download from {store}?"), + (["local", "builds", "clear"], "Delete 2 builds from {store}?"), ], ) -def test_each_clear_asks_first_and_names_the_paths(state, command, prompt): +def test_each_clear_asks_first_and_counts_what_it_deletes(state, command, prompt): store, cache = state + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + _put(store, make_dataset(dataset_id="demo/built"), OriginKind.BUILD) + releases = stored_releases(LocalRegistry(store)) result = runner.invoke(app, command, input="n\n") assert result.exit_code != 0 # aborted assert f"{prompt.format(cache=cache, store=store)} [y/N]" in click.unstyle(result.output) assert (cache / "downloads").exists() - assert LocalRegistry(store).exists("timenet/hello-world", "1.0.0") + assert stored_releases(LocalRegistry(store)) == releases def test_local_cache_clear_keeps_the_local_registry(state): @@ -108,7 +166,7 @@ def test_local_cache_clear_keeps_the_local_registry(state): assert result.exit_code == 0, result.output assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 1 location(s)." assert not (cache / "downloads").exists() - assert LocalRegistry(store).exists("timenet/hello-world", "1.0.0") + assert [item.release for item in stored_releases(LocalRegistry(store))] == [HELLO] def test_local_cache_clear_warns_about_a_folder_without_the_marker_and_keeps_it(state): @@ -117,20 +175,36 @@ def test_local_cache_clear_warns_about_a_folder_without_the_marker_and_keeps_it( (cache / "control/notes.txt").write_text("not from TimeNet") result = runner.invoke(app, ["local", "cache", "clear", "--yes"]) assert result.exit_code == 0, result.output - # The console wraps a long line, also in a path, so the check ignores whitespace. - expected = f"Kept {cache / 'control'}: it has no CACHEDIR.TAG from TimeNet" - assert "".join(expected.split()) in "".join(click.unstyle(result.stderr).split()) + assert _stderr_has(result, f"Kept {cache / 'control'}: it has no CACHEDIR.TAG from TimeNet") assert click.unstyle(result.stdout).splitlines()[-1].endswith("from 1 location(s).") assert (cache / "control/notes.txt").exists() +def test_downloads_clear_keeps_each_download_that_a_build_needs_and_names_it(state): + store, cache = state + parent = _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + other = _put(store, make_dataset(dataset_id="demo/other"), OriginKind.DOWNLOAD) + child = _put( + store, make_dataset(dataset_id="demo/built", parents={ParentAlias("parent"): parent}), OriginKind.BUILD + ) + [gone] = [item for item in stored_releases(LocalRegistry(store)) if item.release == other] + size = human_bytes(gone.size()) + result = runner.invoke(app, ["local", "downloads", "clear", "--yes"]) + assert result.exit_code == 0, result.output + assert _stderr_has(result, f"Kept {parent}: {child} needs it.") + assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 1 location(s)." + assert [item.release for item in stored_releases(LocalRegistry(store))] == [child, parent, HELLO] + assert (cache / "downloads").exists() + + def test_local_clear_deletes_the_cache_and_every_release(state): store, cache = state - [release] = stored_releases(LocalRegistry(store)) - size = human_bytes(_downloads_size(cache) + release.size_bytes) + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + releases = stored_releases(LocalRegistry(store)) + size = human_bytes(_downloads_size(cache) + sum(item.size() for item in releases)) result = runner.invoke(app, ["local", "clear", "--yes"]) assert result.exit_code == 0, result.output - assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 2 location(s)." + assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 3 location(s)." assert stored_releases(LocalRegistry(store)) == [] assert not (cache / "downloads").exists() @@ -147,13 +221,21 @@ def test_local_clear_deletes_no_release_when_the_cache_check_fails(state, monkey def test_the_local_commands_take_the_local_registry_flag(tmp_path): - # The variable of the autouse fixture names an empty store, so the release comes from the flag. + # The variable of the autouse fixture names an empty store, so the releases come from the flag. store = tmp_path / "store" LocalRegistry(store).put_dataset(make_dataset()) - result = runner.invoke(app, ["local", "info", "--local-registry", str(store)]) + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + flag = ["--local-registry", str(store)] + result = runner.invoke(app, ["local", "info", *flag]) + assert result.exit_code == 0, result.output + assert f"Builds in {store}: 1 release" in click.unstyle(result.stdout) + result = runner.invoke(app, ["local", "builds", "info", *flag]) + assert result.exit_code == 0, result.output + assert "timenet/hello-world" in click.unstyle(result.stdout) + result = runner.invoke(app, ["local", "downloads", "clear", "--yes", *flag]) assert result.exit_code == 0, result.output - assert f"Local registry {store}: 1 release" in click.unstyle(result.stdout) - result = runner.invoke(app, ["local", "clear", "--yes", "--local-registry", str(store)]) + assert [item.release for item in stored_releases(LocalRegistry(store))] == [HELLO] + result = runner.invoke(app, ["local", "clear", "--yes", *flag]) assert result.exit_code == 0, result.output assert stored_releases(LocalRegistry(store)) == [] diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 0f24ad853..7b0850d87 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -5,6 +5,7 @@ from _demo_child import _child_of_the_demo from _fake_registry import build_service_fake +from _origin_record import origin_file import httpx from pydantic import ValidationError import pytest @@ -33,6 +34,7 @@ from timenet.progress import BytesMoved, ProgressEvent, StepFinished, StepStarted from timenet.reader import Loading, LoadPlan, TimeFReader from timenet.registry import TIMENET_REGISTRY_URL, LocalRegistry, RemoteRegistry, open_local_version +from timenet.registry.origin import OriginKind from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal, install_backends, make_dataset from timenet.types import Access, Annotation, AnswerTask, DatasetMetadata, Domain, License, Version @@ -554,6 +556,25 @@ def test_get_manifest_names_every_source_when_none_provides_the_release(tmp_path ) +def test_download_records_each_copied_release(tmp_path): + origin, parent = _child_of_the_demo(tmp_path / "origin") + child = DatasetRef.parse("test/child@1.0.0") + store = LocalRegistry(tmp_path / "store") + client = TimeNet(sources=[store, origin.root]) + client.download("test/child") + for release in (parent, child): + record = store.read_origin(release) + assert record is not None + assert (record.kind, record.source) == (OriginKind.DOWNLOAD, str(origin.root.resolve())) + + # With force, the call copies only the requested release again, so only that release gets a record. + for release in (parent, child): + origin_file(store, release).unlink() + client.download("test/child", force=True) + assert store.read_origin(parent) is None + assert store.read_origin(child) is not None + + def test_download_needs_a_local_registry_that_is_not_read_only(registry_root): with pytest.raises(TimeFValidationError) as caught: TimeNet(sources=[LocalRegistry(registry_root, read_only=True)]).download("timenet/hello-world") @@ -624,6 +645,12 @@ def test_download_copies_each_release_from_the_first_registry_that_provides_it(t assert path == store.root / "test/child/1.0.0" with TimeNet(sources=[store]).open("test/child") as reader: reader.verify() + # Each provenance record names the registry that the release came from. + records = {release: store.read_origin(release) for release in (DatasetRef.parse("test/child@1.0.0"), reference)} + assert {release: (record.kind, record.source) for release, record in records.items() if record} == { + DatasetRef.parse("test/child@1.0.0"): (OriginKind.DOWNLOAD, str((tmp_path / "first").resolve())), + reference: (OriginKind.DOWNLOAD, str((tmp_path / "second").resolve())), + } @pytest.mark.parametrize("case", ["force", "damaged"]) From acf410b10e0a1175e489d0c290ef7d1989a3324b Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 00:34:24 +0100 Subject: [PATCH 2/4] docs: shorten the docstrings of the store records Shorter docstrings for the code that this change adds or changes. Part of TNET-157. --- packages/timenet/src/timenet/cache.py | 22 +++++++++---------- packages/timenet/src/timenet/cli/app.py | 2 +- packages/timenet/src/timenet/client.py | 21 +++++++++--------- .../timenet/src/timenet/coordinator/build.py | 20 ++++++++--------- .../timenet/src/timenet/registry/_copy.py | 16 +++++++------- packages/timenet/src/timenet/registry/base.py | 2 +- .../timenet/src/timenet/registry/local.py | 2 +- .../timenet/src/timenet/registry/origin.py | 8 +++---- .../src/timenet/registry/remote/registry.py | 2 +- packages/timenet/src/timenet/registry/s3.py | 2 +- 10 files changed, 46 insertions(+), 51 deletions(-) diff --git a/packages/timenet/src/timenet/cache.py b/packages/timenet/src/timenet/cache.py index 1abab0c7f..3452b1e54 100644 --- a/packages/timenet/src/timenet/cache.py +++ b/packages/timenet/src/timenet/cache.py @@ -18,15 +18,15 @@ @dataclass(frozen=True) class StoredRelease: - """One release in the store, with its size on disk and its provenance record.""" + """A stored release with its path, provenance record, and size on disk.""" release: DatasetRef path: Path origin: Origin | None - """The provenance record, or ``None`` if the release has none.""" + """The record of how the release reached the local registry, or ``None`` if it has no record.""" def size(self) -> int: - """Measure the size on disk. + """Measure the size of the release on disk. Returns: The size in bytes, measured on each call. @@ -102,16 +102,16 @@ def human_bytes(n: int) -> str: def stored_releases(store: LocalRegistry) -> list[StoredRelease]: - """List every release in the store, with its size and its provenance record. + """List stored releases with their provenance records and sizes. Args: - store: The local registry to inspect. + store: The local registry. Returns: The releases, sorted by dataset ID and version. Raises: - TimeFFormatError: If a provenance record is damaged. + TimeFFormatError: If a provenance record is invalid. """ # ruff: ignore[docstring-extraneous-exception] (raised by read_origin) return [ StoredRelease(release, release_dir(store.root, release), store.read_origin(release)) @@ -153,7 +153,7 @@ def releases_to_clear(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> Raises: TimeFValidationError: If ``kinds`` is empty, or a dataset or release folder in the local registry is a symlink. TimeFFormatError: If a provenance record or the manifest of a release that stays is damaged. - OSError: If a folder of the store cannot be read. + OSError: If a folder of the local registry cannot be read. """ # ruff: ignore[docstring-extraneous-exception] (raised by check_no_linked_folders and stored_releases) _check_kinds(kinds) store.check_no_linked_folders() @@ -245,11 +245,9 @@ def clear_releases(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> Cl The bytes freed, the release folders removed, and the releases kept with a release that needs each. Raises: - TimeNetRegistryError: If the local registry is read-only. Nothing is deleted then. - TimeFValidationError: If ``kinds`` is empty, or a dataset or release folder in the local registry is a symlink. - Nothing is deleted then. - TimeFFormatError: If a provenance record or the manifest of a release that stays is damaged. - Nothing is deleted then. + TimeNetRegistryError: Before any deletion, if the local registry is read-only. + TimeFValidationError: Before any deletion, if ``kinds`` is empty or a dataset or release folder is a symlink. + TimeFFormatError: Before any deletion, if a provenance record or a kept release's manifest is damaged. OSError: If a folder cannot be read or removed completely. """ # ruff: ignore[docstring-extraneous-exception] (raised by releases_to_clear and remove_release) _check_kinds(kinds) diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index e87f96e75..3d9af8c59 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -367,7 +367,7 @@ def publish( @local_app.command("info") def local_info(local_registry: str | None = LOCAL_REGISTRY) -> None: - """Show the path and size of the cache, the downloads and the builds.""" + """Show paths and sizes for the cache, downloads, and builds.""" store, cache_dir = default_local_registry(local_registry=local_registry), settings().cache_dir by_kind = stored_releases_by_kind(store) # Plain lines, so that a long path does not wrap. diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 5d4cea036..4bc063221 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -352,31 +352,30 @@ def download( force: bool = False, progress: ProgressCallback | None = None, ) -> Path: - """Copy a dataset release and its payload sources into the local registry. + """Copy a release and its payload sources into the local registry. - The release resolves as in :meth:`open`, without a build. The release and each payload source that the - store lacks or holds damaged come from the first registry other than the store that provides them, and - get a provenance record that names that registry. + The release resolves as in :meth:`open`, without a build. Missing or damaged releases come from the first + other registry that provides them, with a provenance record that names it. Args: - dataset_id: The dataset ID, optionally followed by ``@version``. + dataset_id: The dataset ID. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. force: Copy the release again, even if the local registry holds a complete copy. - progress: A callback for copy events and byte counts. + progress: Receives step events and byte counts for each copied release. Returns: The release directory in the local registry. Raises: - TimeFValidationError: If no source is a writable local registry. - TimeNetAccessError: If a required release is only listed or requires access in a remote registry. - TimeNetNotBuiltError: If only an installed connector provides the release. + TimeFValidationError: If the sources have no writable local registry. + TimeNetAccessError: If a required release is only listed or lacks open access in a remote registry. + TimeNetNotBuiltError: If only installed connectors provide the release. TimeNetDatasetNotFoundError: If no registry other than the local registry provides the release. TimeNetVersionUnavailableError: If the version is missing and its connector builds another version. TimeNetMissingParentError: If no registry provides a missing payload source. TimeNetVersionConflictError: If the local registry holds the same release with different files. - TimeNetFileError: If no registry can replace a damaged copy, or a copied file is damaged. - TimeFFormatError: If a payload source differs from its recorded files or the source links form a cycle. + TimeNetFileError: If a stored copy cannot be repaired or a copied file is damaged. + TimeFFormatError: If a payload source breaks a pin or the payload sources form a cycle. """ # ruff: ignore[docstring-extraneous-exception] (raised by the plan and the copies) dataset_id, version = _resolve_ref(dataset_id, version) store = self._store diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index b62d30d54..45468873e 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -55,20 +55,19 @@ def plan_release_build( def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = None) -> None: - """Run the copy and build steps of a plan in order. + """Run a plan's copy and build steps in order. - A read step runs nothing, because its release stays where it is. Each release that a copy or a - build puts into ``store`` gets a provenance record (see :mod:`timenet.registry.origin`). + Each copied or built release gets a record of its source. Args: - plan: The steps to run. + plan: The plan to run. store: The local registry to copy and build into. - options: Build settings, including the progress callback. + options: Build settings, such as the progress callback. Raises: - TimeNetRegistryError: If the plan copies or builds a release and ``store`` is read-only. - TimeNetVersionConflictError: If a copied release conflicts with the files in ``store``. - TimeFFormatError: If a build changes files pinned by other releases. + TimeNetRegistryError: If the plan copies or builds a release and the local registry is read-only. + TimeNetVersionConflictError: If the local registry holds the same release with different files. + TimeFFormatError: If a build changes files that other releases pin. TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the copy and the backends if any(not isinstance(step, ReadStep) for step in plan.steps): @@ -97,7 +96,7 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = def _build(step: BuildStep, store: LocalRegistry, *, options: BuildOptions) -> None: - """Build an exact release with the builder of its step and check its required files. + """Build the requested release with the builder of its step and record the builder's package. Args: step: The build step, with ``force`` set to rebuild an existing release. @@ -105,8 +104,7 @@ def _build(step: BuildStep, store: LocalRegistry, *, options: BuildOptions) -> N options: The build settings. Raises: - TimeNetDatasetNotFoundError: If the backend does not put the release or a payload source of it - into ``store``. + TimeNetDatasetNotFoundError: If the build omits the release or a payload source. """ # ruff: ignore[docstring-extraneous-exception] - raised by prepare_local_copy if options.progress is not None: options.progress(StepStarted(step.release, "build")) diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index 7b61347f2..a1b060ed4 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -29,27 +29,27 @@ def copy_release( # ruff: ignore[too-many-arguments] force: bool = False, progress: ProgressCallback | None = None, ) -> tuple[DatasetRef, ...]: - """Copy a release and the payload sources that it needs to another registry. + """Copy a release and its payload sources to another registry. - Payload sources go first, a complete copy in ``dest`` is reused, and a damaged local copy is fetched again. - All checks run before the first copy, so a failed check copies nothing. + Copy payload sources first, reuse complete copies, and repair damaged local copies. + Check all releases before copying any files. Args: reference: The exact release to copy. source: The registry to copy from. dest: The registry to copy to. with_payload_sources: Also copy the payload sources that the manifest of the release lists. - force: Fetch ``reference`` again, even if ``dest`` has a complete local copy. - progress: Receives step events for each copied release and byte counts when ``dest`` is local. + force: Fetch ``reference`` again even if the local destination has a complete copy. + progress: Receives step events for each copy and byte counts for local destinations. Returns: - The releases that this function copied or repaired, in the order of the copy. + The copied or repaired releases, in copy order. Raises: TimeNetRegistryError: If ``dest`` is read-only. TimeFValidationError: If ``force`` is set and ``dest`` is not local. - TimeNetVersionConflictError: If ``dest`` holds a release of the copy with other files or another card. - TimeNetAccessError: If a release is not open and ``dest`` or ``source`` requires open access. + TimeNetVersionConflictError: If ``dest`` holds the same release with other files or another card. + TimeNetAccessError: If a release lacks open access and either registry requires it. TimeNetDatasetNotFoundError: If ``source`` lacks the release. TimeNetMissingParentError: If neither registry holds a complete payload source. TimeFFormatError: If a manifest names the wrong release or a pin fails. diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index 019bbbaa4..870a89128 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -54,7 +54,7 @@ def read_only(self) -> bool: @property @abstractmethod def location(self) -> str: - """Where this registry is: a URL or a path, else the name of the registry.""" + """The registry's URL, path, or name.""" def check_writable(self, write: str) -> None: """Refuse a write if this registry is read-only. diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index eb6aae95f..3f752bcdb 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -55,7 +55,7 @@ def __str__(self) -> str: @property def location(self) -> str: - """The resolved root directory.""" + """The registry's absolute root path.""" return str(self._root.resolve()) @property diff --git a/packages/timenet/src/timenet/registry/origin.py b/packages/timenet/src/timenet/registry/origin.py index f9c2267db..315714c57 100644 --- a/packages/timenet/src/timenet/registry/origin.py +++ b/packages/timenet/src/timenet/registry/origin.py @@ -40,15 +40,15 @@ def _in_utc(cls, value: datetime) -> datetime: def without_credentials(location: str) -> str: - """Remove the user info, the query and the fragment of a URL. + """Remove user information, the query, and the fragment from a URL. - A string that is not a URL with a host, such as a path, comes back unchanged. + Return strings unchanged unless they have both a URL scheme and a host. Args: - location: A URL, a path, or another name. + location: A URL, path, or name. Returns: - The location without credentials. + The URL without user information, query, or fragment, or the original non-URL string. """ parts = urlsplit(location) if not (parts.scheme and parts.netloc): diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index 757b50757..74143b5eb 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -60,7 +60,7 @@ def __str__(self) -> str: @property def location(self) -> str: - """The base URL of the service.""" + """The registry service's base URL.""" return self._base_url def list_datasets(self) -> list[DatasetMetadata]: diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 7872ed94a..41fc3bce4 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -64,7 +64,7 @@ def __str__(self) -> str: @property def location(self) -> str: - """The ``s3://bucket/prefix`` root.""" + """The registry's root URL as ``s3://bucket/prefix``.""" return self._display_root() def list_datasets(self) -> list[DatasetMetadata]: From b3a168a14dc2fca4c559ae9693c92165c519b8aa Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 11:19:01 +0100 Subject: [PATCH 3/4] test: trim the tests of the store records Drop tests that cannot fail or repeat other tests, and sharpen the rest. Part of TNET-157. --- packages/timenet/tests/registry/test_origin.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/timenet/tests/registry/test_origin.py b/packages/timenet/tests/registry/test_origin.py index 6a96c72b6..88d8dff54 100644 --- a/packages/timenet/tests/registry/test_origin.py +++ b/packages/timenet/tests/registry/test_origin.py @@ -63,10 +63,7 @@ def test_a_new_record_replaces_the_old_one(store): ("location", "expected"), [ ("https://user:secret@registry.example.org:8443/r?sig=abc#top", "https://registry.example.org:8443/r"), - ("s3://bucket/prefix?versionId=1", "s3://bucket/prefix"), - ("https://registry.example.org", "https://registry.example.org"), ("/data/registry", "/data/registry"), - ("timenet-connectors 0.9.0", "timenet-connectors 0.9.0"), ], ) def test_without_credentials_keeps_only_scheme_host_and_path_of_a_url(location, expected): From 952a19df3f648f08f78327d0028b044b7dabbf09 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 17:39:50 +0100 Subject: [PATCH 4/4] refactor: name the target registry in the coordinator and the local registry in the client The coordinator used three names for the registry that its copies and builds go into: "store", "output" and "local registry". Now it uses one name, the target registry. The parameters of plan_release_build(), run_plan(), plan_steps(), plan_download(), plan_read() and read_release() are now target_registry. In the planner, _Graph.output is now target_registry, in_output() is now in_target(), and needed_in_local_registry is now needed_in_target. The docstrings and comments say "the target registry" in place of "the output" and "the store". In the client, TimeNet._store is now TimeNet._local_registry. The local variables and the parameters of the client helpers that hold it follow. In timenet-build build, the opened local registry is now local. The tests follow. In the coordinator tests, the fixtures, the variables, the test names and the case names for the registry that copies and builds go into say target_registry. Where a test means the local registry of the client, the SDK functions or the CLI, its names say local_registry. Other meanings of "store" keep their names: the store_dataset() stage of a connector, stored_releases and StoredRelease, and the parameter store of Backend.build() in the fake backends. The public names do not change, and the behavior does not change. Part of TNET-157. --- .../src/timenet_connectors/builder/cli.py | 14 +- .../tests/test_build_env.py | 2 +- .../tests/test_build_plan.py | 281 +++++++++-------- packages/timenet/src/timenet/client.py | 50 +-- .../timenet/src/timenet/coordinator/build.py | 30 +- .../timenet/src/timenet/coordinator/plan.py | 146 ++++----- .../timenet/src/timenet/coordinator/read.py | 26 +- .../timenet/tests/coordinator/test_build.py | 143 +++++---- .../timenet/tests/coordinator/test_read.py | 290 ++++++++++-------- .../timenet/tests/registry/test_origin.py | 2 +- .../timenet/tests/registry/test_remote.py | 4 +- packages/timenet/tests/registry/test_s3.py | 2 +- packages/timenet/tests/test_cache.py | 10 +- packages/timenet/tests/test_cli.py | 10 +- packages/timenet/tests/test_client.py | 28 +- packages/timenet/tests/test_config.py | 2 +- packages/timenet/tests/test_producer.py | 4 +- 17 files changed, 560 insertions(+), 484 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index dd6e88260..8be73fadb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -113,7 +113,7 @@ def build( sources = default_sources(local_registry=local_registry, remote_registry=remote_registry, auto_build=auto_build) # The default list starts with the local registry, which the build writes into. The planner leaves it out of # the sources of the copies and builds. - store = cast("LocalRegistry", sources[0]) + local = cast("LocalRegistry", sources[0]) dest = None if publish_to is None else open_registry(publish_to, label="--publish-to") has_connectors = any(isinstance(source, InstalledConnectors) for source in sources) try: @@ -126,7 +126,7 @@ def build( # plan refuses another release before any download. plan = plan_release_build( str(wanted), - store=store, + target_registry=local, parents_from=sources, force=force, open_access_only=dest is not None and dest.requires_open_access, @@ -136,7 +136,7 @@ def build( if dest is not None: lines.append(f"then publish {wanted} to {dest}") elif not plan.steps: - console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {store}") + console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {local}") if lines: typer.echo("\n".join(lines)) return @@ -149,9 +149,9 @@ def build( quiet=console.quiet, isolation=None if isolation is None else ("on" if isolation else "off"), ) - run_plan(plan, store, options=options) + run_plan(plan, local, options=options) if dest is not None: - publish(wanted, dest=dest, source=store, progress=progress) + publish(wanted, dest=dest, source=local, progress=progress) except TimeNetMissingParentError as error: error.hint = _missing_hint(has_connectors=has_connectors) raise @@ -167,9 +167,9 @@ def build( error.hint = _access_hint(dest, has_connectors=has_connectors) raise if dest is None and not plan.steps: - console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {store}") + console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {local}") # A publish prints the version, and a local build prints the release directory. - typer.echo(str(wanted.version) if dest is not None else str(release_dir(store.root, wanted))) + typer.echo(str(wanted.version) if dest is not None else str(release_dir(local.root, wanted))) def _missing_hint(*, has_connectors: bool) -> str: diff --git a/packages/timenet-connectors/tests/test_build_env.py b/packages/timenet-connectors/tests/test_build_env.py index 7b26a698c..f42bb2038 100644 --- a/packages/timenet-connectors/tests/test_build_env.py +++ b/packages/timenet-connectors/tests/test_build_env.py @@ -165,7 +165,7 @@ def _missing(): uv_command(_spec(), ["timenet-build"]) -def test_run_isolated_build_runs_build_release_with_the_store_and_the_pins(monkeypatch, tmp_path): +def test_run_isolated_build_runs_build_release_with_the_target_registry_and_the_pins(monkeypatch, tmp_path): release = DatasetRef.parse("timenet/hello-world@1.0.0") pinned = PayloadSource(release=release, files=(FilePart(path="a.bin", checksum="sha256:" + "0" * 64, size=1),)) captured = {} diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index e11ecf8de..cfc5b3c3a 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -45,9 +45,9 @@ runner = CliRunner() -def _parent_steps(dataset_id: str, store: LocalRegistry, **options) -> list[Step]: +def _parent_steps(dataset_id: str, target_registry: LocalRegistry, **options) -> list[Step]: """Plan the build of ``dataset_id`` and return each step but its own build.""" - steps = plan_release_build(dataset_id, store=store, **options).steps + steps = plan_release_build(dataset_id, target_registry=target_registry, **options).steps return [step for step in steps if not (step.action == "build" and step.release.dataset_id == dataset_id)] @@ -110,7 +110,7 @@ def _install_cards(monkeypatch, **parents: dict[str, DatasetRef]) -> dict: @pytest.mark.parametrize( ("grand_in", "expected"), [ - ("output", lambda origin: [_build(PARENT)]), + ("target", lambda origin: [_build(PARENT)]), ( "origin", lambda origin: [ @@ -129,12 +129,12 @@ def _install_cards(monkeypatch, **parents: dict[str, DatasetRef]) -> dict: ) def test_plan_copies_or_builds_each_parent_once_before_its_children(tmp_path, monkeypatch, grand_in, expected): _install_cards(monkeypatch, child={"parent": PARENT, "grand": GRAND}, parent={"grand": GRAND}) - registries = {name: LocalRegistry(tmp_path / name) for name in ("output", "origin")} + registries = {name: LocalRegistry(tmp_path / name) for name in ("target", "origin")} if grand_in in registries: registries[grand_in].put_dataset(make_dataset()) steps = _parent_steps( - CHILD.dataset_id, registries["output"], parents_from=[registries["origin"], InstalledConnectors()] + CHILD.dataset_id, registries["target"], parents_from=[registries["origin"], InstalledConnectors()] ) assert steps == expected(registries["origin"]) @@ -161,10 +161,10 @@ def test_plan_copies_or_builds_each_parent_once_before_its_children(tmp_path, mo # ruff: ignore[too-many-arguments, too-many-positional-arguments] def test_plan_rejects_unobtainable_parents_before_anything_runs(tmp_path, monkeypatch, cards, options, error, match): _install_cards(monkeypatch, **cards) - output = LocalRegistry(tmp_path / "output") + target_registry = LocalRegistry(tmp_path / "output") with pytest.raises(error, match=match): - _parent_steps(CHILD.dataset_id, output, **options) + _parent_steps(CHILD.dataset_id, target_registry, **options) _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) @@ -202,27 +202,29 @@ def _declared_without_its_sources_source(registry: LocalRegistry) -> None: shutil.rmtree(release_dir(registry.root, GRAND)) -@pytest.mark.parametrize("where", ["output", "origin", "source-in-origin"]) +@pytest.mark.parametrize("where", ["target", "origin", "source-in-origin"]) def test_a_declared_parent_needs_only_its_direct_payload_sources(tmp_path, monkeypatch, child_builds, where): _install_cards(monkeypatch, child={"declared": DECLARED}) - output, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") - _declared_without_its_sources_source(origin if where == "origin" else output) + target_registry, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") + _declared_without_its_sources_source(origin if where == "origin" else target_registry) if where == "source-in-origin": release_dir(origin.root, SOURCE).parent.mkdir(parents=True) - shutil.move(release_dir(output.root, SOURCE), release_dir(origin.root, SOURCE)) + shutil.move(release_dir(target_registry.root, SOURCE), release_dir(origin.root, SOURCE)) # DECLARED opens with SOURCE, its direct payload source. So the plan needs neither GRAND, which - # SOURCE reads and no registry holds, nor a step for SOURCE where the output holds it. + # SOURCE reads and no registry holds, nor a step for SOURCE where the target registry holds it. steps = { - "output": [], + "target": [], "origin": [CopyStep(DECLARED, origin)], "source-in-origin": [CopyStep(SOURCE, origin, with_payload_sources=False)], } - assert _parent_steps(CHILD.dataset_id, output, parents_from=[origin]) == steps[where] - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[origin]), output) + assert _parent_steps(CHILD.dataset_id, target_registry, parents_from=[origin]) == steps[where] + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[origin]), target_registry + ) for reference in (DECLARED, SOURCE): - assert output.exists(reference.dataset_id, str(reference.version)) - assert not output.exists(GRAND.dataset_id, str(GRAND.version)) + assert target_registry.exists(reference.dataset_id, str(reference.version)) + assert not target_registry.exists(GRAND.dataset_id, str(GRAND.version)) # Only the requested release builds. assert child_builds == [(CHILD.dataset_id, False)] @@ -231,11 +233,11 @@ def test_a_payload_source_that_is_later_a_declared_parent_brings_its_own_payload # The child declares DECLARED first, then SOURCE. The walk finds SOURCE first as the payload source # of DECLARED. As a declared parent, SOURCE also needs GRAND, its own payload source, which is missing. _install_cards(monkeypatch, child={"declared": DECLARED, "source": SOURCE}) - output = LocalRegistry(tmp_path / "output") - _declared_without_its_sources_source(output) - pins = output.get_manifest(SOURCE.dataset_id).payload_sources[SourceKey(str(GRAND))] + target_registry = LocalRegistry(tmp_path / "output") + _declared_without_its_sources_source(target_registry) + pins = target_registry.get_manifest(SOURCE.dataset_id).payload_sources[SourceKey(str(GRAND))] - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert steps == [_build(GRAND, pinned_by={SOURCE: pins})] @@ -275,22 +277,22 @@ def convert(self, raw_refs): return make_dataset() -def _parent_without_its_source(output: LocalRegistry) -> None: +def _parent_without_its_source(target_registry: LocalRegistry) -> None: """Store GRAND and PARENT, which reads a GRAND Record, then remove GRAND.""" - output.put_dataset(make_dataset()) - _child_of(output, PARENT, GRAND, make_dataset().records[0].id) - shutil.rmtree(release_dir(output.root, GRAND)) + target_registry.put_dataset(make_dataset()) + _child_of(target_registry, PARENT, GRAND, make_dataset().records[0].id) + shutil.rmtree(release_dir(target_registry.root, GRAND)) @pytest.mark.parametrize("backend", [ValuesBackend.PARQUET, ValuesBackend.ZARR]) def test_a_pinned_source_is_rebuilt_in_staging_and_published_only_if_it_matches(tmp_path, monkeypatch, backend): _install_cards(monkeypatch, child={"parent": PARENT}, parent={"parent": GRAND}) monkeypatch.setattr(_Grand, "values_backend", backend) - output = LocalRegistry(tmp_path / "output") - _parent_without_its_source(output) - pins = output.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] + target_registry = LocalRegistry(tmp_path / "output") + _parent_without_its_source(target_registry) + pins = target_registry.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert steps == [_build(GRAND, pinned_by={PARENT: pins})] built = [] @@ -305,33 +307,39 @@ def child(release, store_root, *, pinned_by, **_): monkeypatch.setattr(backend_module, "run_isolated_build", child) if backend == ValuesBackend.PARQUET: - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[InstalledConnectors()]), output) - assert output.exists(GRAND.dataset_id, str(GRAND.version)) + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()]), + target_registry, + ) + assert target_registry.exists(GRAND.dataset_id, str(GRAND.version)) assert built == [GRAND, CHILD] return # A Zarr rebuild writes other files than the Parquet files PARENT reads, so nothing is published. with pytest.raises(TimeFFormatError, match=r"Nothing was published\. Fetch the exact release .* or rebuild"): - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[InstalledConnectors()]), output) + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()]), + target_registry, + ) assert built == [GRAND] - assert not output.exists(GRAND.dataset_id, str(GRAND.version)) - assert not list(release_dir(output.root, GRAND).parent.glob("*")) + assert not target_registry.exists(GRAND.dataset_id, str(GRAND.version)) + assert not list(release_dir(target_registry.root, GRAND).parent.glob("*")) def test_a_payload_source_that_two_releases_read_is_built_against_the_pins_of_both(tmp_path, monkeypatch): sibling = DatasetRef.parse("test/sibling@1.0.0") _install_cards(monkeypatch, child={"parent": PARENT, "sibling": sibling}) - output = LocalRegistry(tmp_path / "output") - output.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "output") + target_registry.put_dataset(make_dataset()) # PARENT and the sibling each read another Record of GRAND, so their pins differ. - _child_of(output, PARENT, GRAND, make_dataset().records[0].id) - _child_of(output, sibling, GRAND, make_dataset().records[1].id) - shutil.rmtree(release_dir(output.root, GRAND)) + _child_of(target_registry, PARENT, GRAND, make_dataset().records[0].id) + _child_of(target_registry, sibling, GRAND, make_dataset().records[1].id) + shutil.rmtree(release_dir(target_registry.root, GRAND)) pins = { - reader: output.get_manifest(reader.dataset_id).payload_sources[SourceKey(str(GRAND))] + reader: target_registry.get_manifest(reader.dataset_id).payload_sources[SourceKey(str(GRAND))] for reader in (PARENT, sibling) } - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert steps == [_build(GRAND, pinned_by=pins)] @@ -350,70 +358,72 @@ def _repin(registry: LocalRegistry, reader: DatasetRef) -> None: def test_two_readers_that_pin_one_file_differently_fail_the_plan_of_its_build(tmp_path, monkeypatch): sibling = DatasetRef.parse("test/sibling@1.0.0") _install_cards(monkeypatch, child={"parent": PARENT, "sibling": sibling}) - output = LocalRegistry(tmp_path / "output") - output.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "output") + target_registry.put_dataset(make_dataset()) for reader in (PARENT, sibling): - _child_of(output, reader, GRAND, make_dataset().records[0].id) - _repin(output, sibling) - shutil.rmtree(release_dir(output.root, GRAND)) + _child_of(target_registry, reader, GRAND, make_dataset().records[0].id) + _repin(target_registry, sibling) + shutil.rmtree(release_dir(target_registry.root, GRAND)) # No build of GRAND can match both readers, so the plan fails before any step runs. with pytest.raises(TimeFFormatError, match=f"pin .* of {GRAND} as different files, so no build"): - plan_release_build(CHILD.dataset_id, store=output, parents_from=[InstalledConnectors()]) + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()]) def test_a_held_payload_source_that_breaks_a_pin_fails_the_plan(tmp_path, monkeypatch): _install_cards(monkeypatch, child={"parent": PARENT}) - output = LocalRegistry(tmp_path / "output") - output.put_dataset(make_dataset()) - _child_of(output, PARENT, GRAND, make_dataset().records[0].id) - _repin(output, PARENT) + target_registry = LocalRegistry(tmp_path / "output") + target_registry.put_dataset(make_dataset()) + _child_of(target_registry, PARENT, GRAND, make_dataset().records[0].id) + _repin(target_registry, PARENT) with pytest.raises(TimeFFormatError, match=f"payload source {GRAND} differs from the release {PARENT}"): - plan_release_build(CHILD.dataset_id, store=output) + plan_release_build(CHILD.dataset_id, target_registry=target_registry) def test_a_built_release_needs_only_its_payload_sources(tmp_path, monkeypatch): _install_cards(monkeypatch, parent={"parent": GRAND}) - output = LocalRegistry(tmp_path / "output") - _parent_without_its_source(output) - pins = output.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] + target_registry = LocalRegistry(tmp_path / "output") + _parent_without_its_source(target_registry) + pins = target_registry.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] with pytest.raises(TimeNetMissingParentError, match=r"^payload source timenet/hello-world@1\.0\.0 of test/parent"): - _parent_steps(PARENT.dataset_id, output) + _parent_steps(PARENT.dataset_id, target_registry) connectors = [InstalledConnectors()] # The source that PARENT reads builds against its pins. - assert _parent_steps(PARENT.dataset_id, output, parents_from=connectors) == [ + assert _parent_steps(PARENT.dataset_id, target_registry, parents_from=connectors) == [ _build(GRAND, pinned_by={PARENT: pins}) ] # A rebuild of PARENT needs its declared parents instead, and pins the new files. - assert _parent_steps(PARENT.dataset_id, output, parents_from=connectors, force=True) == [_build(GRAND)] + assert _parent_steps(PARENT.dataset_id, target_registry, parents_from=connectors, force=True) == [_build(GRAND)] # A registry that holds the source fills the gap: the local PARENT is reused and GRAND is copied. origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - assert _parent_steps(PARENT.dataset_id, output, parents_from=[origin]) == [ + assert _parent_steps(PARENT.dataset_id, target_registry, parents_from=[origin]) == [ CopyStep(GRAND, origin, with_payload_sources=False) ] def test_a_damaged_parent_is_copied_again_but_never_built_over(tmp_path, monkeypatch, child_builds): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - output, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") + target_registry, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] - for registry in (output, origin): + for registry in (target_registry, origin): registry.put_dataset(parent) - (release_dir(output.root, PARENT) / "control.duckdb").write_bytes(b"cut short") + (release_dir(target_registry.root, PARENT) / "control.duckdb").write_bytes(b"cut short") with pytest.raises(TimeNetFileError, match=r"'control\.duckdb' has 9 bytes, expected [\d,]+ bytes\.$") as caught: - _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert caught.value.release == PARENT - assert _parent_steps(CHILD.dataset_id, output, parents_from=[origin, InstalledConnectors()]) == [ + assert _parent_steps(CHILD.dataset_id, target_registry, parents_from=[origin, InstalledConnectors()]) == [ CopyStep(PARENT, origin) ] - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[origin]), output) - assert _parent_steps(CHILD.dataset_id, output) == [] + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[origin]), target_registry + ) + assert _parent_steps(CHILD.dataset_id, target_registry) == [] assert child_builds == [(CHILD.dataset_id, False)] @@ -456,10 +466,10 @@ def put_release(self, from_directory): def test_open_access_only_refuses_a_parent_to_build_whose_card_is_not_open(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) cards[PARENT.dataset_id] = cards[PARENT.dataset_id].model_copy(update=_GATED) - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") with pytest.raises(TimeNetAccessError) as caught: - _parent_steps(CHILD.dataset_id, store, parents_from=[InstalledConnectors()], open_access_only=True) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()], open_access_only=True) error = caught.value requirement = AccessRequirement(PARENT, Access.CREDENTIALED, _GATED["access_url"]) @@ -468,15 +478,15 @@ def test_open_access_only_refuses_a_parent_to_build_whose_card_is_not_open(tmp_p def test_open_access_only_names_the_access_page_of_a_held_parent_from_its_lineage(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") # The card of the held parent stays open. Its lineage holds GRAND, which is credentialed. held = TimeFDataset(metadata=cards[PARENT.dataset_id]) grand = make_dataset().metadata.model_copy(update=_GATED) held._import_state.lineage[str(GRAND)] = ParentDatasetCard(metadata=grand) - store.put_dataset(held) + target_registry.put_dataset(held) with pytest.raises(TimeNetAccessError) as caught: - _parent_steps(CHILD.dataset_id, store, parents_from=[InstalledConnectors()], open_access_only=True) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()], open_access_only=True) assert (caught.value.dataset, caught.value.access) == (PARENT, Access.CREDENTIALED) assert caught.value.requirements == (AccessRequirement(GRAND, Access.CREDENTIALED, _GATED["access_url"]),) @@ -548,9 +558,12 @@ def test_every_build_entry_point_prepares_parents(tmp_path, monkeypatch, child_b def build(origin=None): # The client and timenet.build() build through the coordinator. - store = LocalRegistry(out) + target_registry = LocalRegistry(out) run_plan( - plan_release_build(CHILD.dataset_id, store=store, parents_from=[] if origin is None else [origin]), store + plan_release_build( + CHILD.dataset_id, target_registry=target_registry, parents_from=[] if origin is None else [origin] + ), + target_registry, ) with pytest.raises(TimeNetMissingParentError, match=r"parent release test/parent@1\.0\.0 of test/child@1\.0\.0"): @@ -569,14 +582,14 @@ def build(origin=None): def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, parent={"parent": GRAND}) - output = LocalRegistry(tmp_path / "output") - _parent_without_its_source(output) - pins = output.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] + local_registry = LocalRegistry(tmp_path / "output") + _parent_without_its_source(local_registry) + pins = local_registry.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] received, pin_files = [], [] def run_pipeline(connector, root, *, pinned_by, **_): received.append(pinned_by) - output.put_dataset(make_dataset(), pinned_by=pinned_by) + local_registry.put_dataset(make_dataset(), pinned_by=pinned_by) def child(command, env): # Stands in for the uv child: run the forwarded command line through the CLI in this process. @@ -590,15 +603,15 @@ def child(command, env): monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) monkeypatch.setattr(backend_module, "run_pipeline", run_pipeline) - # The output holds PARENT, so the build gets only GRAND, the payload source of PARENT. - result = runner.invoke(app, ["build", PARENT.dataset_id, "--local-registry", str(output.root)]) + # The local registry holds PARENT, so the build gets only GRAND, the payload source of PARENT. + result = runner.invoke(app, ["build", PARENT.dataset_id, "--local-registry", str(local_registry.root)]) assert result.exit_code == 0, result.output # The rebuild of GRAND gets the pins of PARENT, through a file that is gone afterwards. assert received == [{PARENT: pins}] assert len(pin_files) == 1 assert not pin_files[0].exists() - assert output.exists(GRAND.dataset_id, str(GRAND.version)) + assert local_registry.exists(GRAND.dataset_id, str(GRAND.version)) _COPY_HINT = ( @@ -611,7 +624,7 @@ def child(command, env): def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - store = LocalRegistry(tmp_path / "store") + local_registry = LocalRegistry(tmp_path / "store") options = [] if case == "missing": # No registry holds the parent, and no installed connector builds it. @@ -619,8 +632,8 @@ def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): if case == "missing-without-auto-build": options = ["--no-auto-build"] if case == "damaged": - store.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) - (release_dir(store.root, PARENT) / "control.duckdb").write_bytes(b"cut short") + local_registry.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) + (release_dir(local_registry.root, PARENT) / "control.duckdb").write_bytes(b"cut short") options = ["--remote-registry", str(tmp_path / "remote registry")] if case == "not-open": cards[CHILD.dataset_id] = cards[CHILD.dataset_id].model_copy(update=_GATED) @@ -633,11 +646,11 @@ def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): monkeypatch.setattr(client_module, "open_registry", lambda uri, **_: hosted) options = ["--no-auto-build"] - result = runner.invoke(app, ["build", CHILD.dataset_id, "--local-registry", str(store.root), *options]) + result = runner.invoke(app, ["build", CHILD.dataset_id, "--local-registry", str(local_registry.root), *options]) # The hint of a damaged copy repeats the registry flags of the command, quoted for the shell. remote = str(tmp_path / "remote registry") - flags = f" --local-registry {shlex.quote(str(store.root))} --remote-registry {shlex.quote(remote)}" + flags = f" --local-registry {shlex.quote(str(local_registry.root))} --remote-registry {shlex.quote(remote)}" hints = { "missing": (TimeNetMissingParentError, f". {_COPY_HINT[0].upper()}{_COPY_HINT[1:]}"), "missing-without-auto-build": ( @@ -667,7 +680,7 @@ def _publish_to(tmp_path, monkeypatch, cards) -> tuple[_Hosted, LocalRegistry, l """Make each ``--publish-to`` of the CLI open a registry that is not local, and record each isolated build. Returns: - The registry that ``--publish-to`` opens, the store, and the list of the builds. + The registry that ``--publish-to`` opens, the local registry, and the list of the builds. """ remote = _Hosted(tmp_path / "hosted") monkeypatch.setattr(cli_module, "open_registry", lambda url, **_: remote) @@ -685,25 +698,25 @@ def test_publish_to_refuses_a_credentialed_release_before_any_download(tmp_path, cards[gated_ref.dataset_id] = cards[gated_ref.dataset_id].model_copy(update=_GATED) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) - _, store, calls = _publish_to(tmp_path, monkeypatch, cards) + _, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) command = ["build", CHILD.dataset_id, "--publish-to", "s3://bucket/registry"] result = runner.invoke(app, [*command, "--remote-registry", str(origin.root), "--no-auto-build"]) - # The build into the store gets the access checks of the destination. + # The build into the local registry gets the access checks of the destination. assert isinstance(result.exception, TimeNetAccessError), result.output assert result.exception.dataset == gated_ref assert calls == [] - assert not store.exists(PARENT.dataset_id, str(PARENT.version)) + assert not local_registry.exists(PARENT.dataset_id, str(PARENT.version)) -def test_publish_to_builds_into_the_store_and_publishes(tmp_path, monkeypatch): +def test_publish_to_builds_into_the_local_registry_and_publishes(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) origin = LocalRegistry(tmp_path / "origin") parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] origin.put_dataset(parent) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) command = ["build", CHILD.dataset_id, "--publish-to", "s3://bucket/registry"] command += ["--remote-registry", str(origin.root), "--no-auto-build"] @@ -711,24 +724,24 @@ def test_publish_to_builds_into_the_store_and_publishes(tmp_path, monkeypatch): assert result.exit_code == 0, result.output assert result.stdout == f"{CHILD.version}\n" - # The parent lands in the store first, and the child builds there. Then the publish takes the + # The parent lands in the local registry first, and the child builds there. Then the publish takes the # child and its payload source, the parent, to the destination. assert calls == [(CHILD.dataset_id, False)] - for registry in (store, remote): + for registry in (local_registry, remote): assert registry.exists(CHILD.dataset_id, str(CHILD.version)) assert registry.exists(PARENT.dataset_id, str(PARENT.version)) # A destination that holds the version gets nothing, and nothing builds. - shutil.rmtree(store.root) + shutil.rmtree(local_registry.root) result = runner.invoke(app, command) assert (result.exit_code, result.stdout) == (0, f"{CHILD.version}\n") assert result.stderr.splitlines() == [f"✅ Nothing to do: 'test/child' 1.0.0 is already in {remote}"] assert calls == [(CHILD.dataset_id, False)] - assert not store.root.exists() + assert not local_registry.root.exists() def test_publish_to_copies_parents_only_from_the_sources(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] remote.put_dataset(parent) @@ -745,7 +758,7 @@ def test_publish_to_copies_parents_only_from_the_sources(tmp_path, monkeypatch): result = runner.invoke(app, [*command, "--remote-registry", str(remote.local_root)]) assert result.exit_code == 0, result.output assert calls == [(CHILD.dataset_id, False)] - assert store.exists(PARENT.dataset_id, str(PARENT.version)) + assert local_registry.exists(PARENT.dataset_id, str(PARENT.version)) assert remote.exists(CHILD.dataset_id, str(CHILD.version)) @@ -761,7 +774,7 @@ def test_publish_to_a_local_directory_publishes_there(tmp_path, monkeypatch, chi assert result.exit_code == 0, result.output assert result.stdout == f"{CHILD.version}\n" assert child_builds == [(PARENT.dataset_id, False), (CHILD.dataset_id, False)] - # The build lands in the store, and the publish copies the child with its payload source. + # The build lands in the local registry, and the publish copies the child with its payload source. for registry in (default_local_registry(), published): assert registry.exists(CHILD.dataset_id, str(CHILD.version)) assert registry.exists(PARENT.dataset_id, str(PARENT.version)) @@ -817,7 +830,7 @@ def test_force_with_publish_to_does_nothing_for_the_same_files(tmp_path, monkeyp _store_from_card(CHILD.dataset_id, saved.root, force=False) def copy_the_saved_release(release, store_root, *, force=False, **_): - # Each build puts the same release directory into the store, so both builds write the same files. + # Each build puts the same release directory into the local registry, so both builds write the same files. calls.append((release.dataset_id, force)) LocalRegistry(store_root).put_release(release_dir(saved.root, CHILD)) @@ -835,61 +848,65 @@ def copy_the_saved_release(release, store_root, *, force=False, **_): assert _files(remote.local_root) == published -def test_the_plan_drops_the_store_from_parents_from(tmp_path, monkeypatch): +def test_the_plan_drops_the_target_registry_from_parents_from(tmp_path, monkeypatch): _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") - # The list holds the store itself and another registry over its directory. The plan drops both, - # so the error names the store once. + # The list holds the target registry itself and another registry over its directory. The plan drops both, + # so the error names the target registry once. with pytest.raises(TimeNetMissingParentError) as missing: - _parent_steps(CHILD.dataset_id, store, parents_from=[store, LocalRegistry(store.root)]) + _parent_steps( + CHILD.dataset_id, target_registry, parents_from=[target_registry, LocalRegistry(target_registry.root)] + ) - assert missing.value.searched == str(store) + assert missing.value.searched == str(target_registry) def test_a_parent_that_a_registry_lists_without_serving_is_built_by_the_connectors(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) hosted = _Hosted(tmp_path / "hosted") hosted.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id].model_copy(update=_GATED))) - output = LocalRegistry(tmp_path / "out") + target_registry = LocalRegistry(tmp_path / "out") - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[hosted, InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[hosted, InstalledConnectors()]) assert steps == [_build(PARENT)] # Without the connectors, the plan names the access page of the release. with pytest.raises(TimeNetAccessError) as caught: - _parent_steps(CHILD.dataset_id, output, parents_from=[hosted]) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[hosted]) assert caught.value.requirements == (AccessRequirement(PARENT, Access.CREDENTIALED, _GATED["access_url"]),) @pytest.mark.parametrize("answer", ["not-found", "not-served"]) -def test_a_damaged_copy_in_the_output_stops_the_plan_if_no_registry_serves_the_release(tmp_path, monkeypatch, answer): +def test_a_damaged_copy_in_the_target_registry_stops_the_plan_if_no_registry_serves_the_release( + tmp_path, monkeypatch, answer +): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - output, hosted = LocalRegistry(tmp_path / "output"), _Hosted(tmp_path / "hosted") + target_registry, hosted = LocalRegistry(tmp_path / "output"), _Hosted(tmp_path / "hosted") parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] - output.put_dataset(parent) - (release_dir(output.root, PARENT) / "control.duckdb").write_bytes(b"cut short") + target_registry.put_dataset(parent) + (release_dir(target_registry.root, PARENT) / "control.duckdb").write_bytes(b"cut short") if answer == "not-served": hosted.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id].model_copy(update=_GATED))) # The damage, not the missing release or its access page, is what the caller must fix. with pytest.raises(TimeNetFileError) as caught: - _parent_steps(CHILD.dataset_id, output, parents_from=[hosted]) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[hosted]) assert caught.value.release == PARENT def test_the_requested_build_waits_for_a_payload_source_of_a_held_parent(tmp_path, monkeypatch): _install_cards(monkeypatch, child={"declared": DECLARED}) - output, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") + target_registry, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) _child_of(origin, SOURCE, GRAND, make_dataset().records[0].id) _child_of(origin, DECLARED, SOURCE, 2) - # The output holds DECLARED, but not SOURCE, whose files DECLARED reads. - shutil.copytree(release_dir(origin.root, DECLARED), release_dir(output.root, DECLARED)) + # The target registry holds DECLARED, but not SOURCE, whose files DECLARED reads. + shutil.copytree(release_dir(origin.root, DECLARED), release_dir(target_registry.root, DECLARED)) - plan = plan_release_build(CHILD.dataset_id, store=output, parents_from=[origin]) + plan = plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[origin]) # DECLARED has no step, so the build of CHILD waits for the step of SOURCE behind it. assert plan.steps == ( @@ -919,19 +936,19 @@ def test_a_plan_reads_each_manifest_once_from_each_registry(tmp_path, monkeypatc layout.put_dataset(make_dataset()) _child_of(layout, SOURCE, GRAND, make_dataset().records[0].id) _child_of(layout, DECLARED, SOURCE, 2) - hosted, output = _Hosted(layout.root), LocalRegistry(tmp_path / "output") - hosted_reads, output_reads = _count_reads(monkeypatch, hosted), _count_reads(monkeypatch, output) + hosted, target_registry = _Hosted(layout.root), LocalRegistry(tmp_path / "output") + hosted_reads, target_reads = _count_reads(monkeypatch, hosted), _count_reads(monkeypatch, target_registry) - plan = plan_release_build(CHILD.dataset_id, store=output, parents_from=[hosted]) + plan = plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[hosted]) assert plan.steps[:-1] == tuple(CopyStep(reference, hosted) for reference in parents.values()) # The registry that is not local answers through its manifest, and the plan reads that manifest # only once. assert hosted_reads == {str(reference): 1 for reference in (DECLARED, SOURCE, GRAND)} - # The output is read for the requested release too, which it does not hold. - assert output_reads == {str(reference): 1 for reference in (CHILD, DECLARED, SOURCE)} + # The target registry is read for the requested release too, which it does not hold. + assert target_reads == {str(reference): 1 for reference in (CHILD, DECLARED, SOURCE)} hosted_reads.clear() - run_plan(plan, output) + run_plan(plan, target_registry) # Each copy reads the manifests of its release and of the payload sources that it brings once # more, from the registry as it is when the copy runs. SOURCE is in both copies. assert hosted_reads == {str(DECLARED): 1, str(SOURCE): 2, str(GRAND): 1} @@ -970,7 +987,7 @@ def test_plan_lines_name_read_steps(tmp_path): def test_dry_run_prints_the_plan_and_changes_nothing(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) result = runner.invoke(app, ["build", CHILD.dataset_id, "--dry-run", "--publish-to", "s3://bucket/registry"]) assert result.exit_code == 0, result.output assert result.stdout.splitlines() == [ @@ -978,7 +995,7 @@ def test_dry_run_prints_the_plan_and_changes_nothing(tmp_path, monkeypatch): "└── test/parent@1.0.0: build with installed connectors", f"then publish test/child@1.0.0 to {remote}", ] - assert calls == [] and not store.root.exists() and not remote.exists(CHILD.dataset_id, "1.0.0") + assert calls == [] and not local_registry.root.exists() and not remote.exists(CHILD.dataset_id, "1.0.0") def test_a_build_prints_the_plan_before_the_first_step(tmp_path, monkeypatch, child_builds): @@ -1013,21 +1030,21 @@ def test_quiet_hides_the_plan(tmp_path, monkeypatch, child_builds): assert child_builds == [(PARENT.dataset_id, False), (CHILD.dataset_id, False)] -@pytest.mark.parametrize("held_in", ["store", "destination", "store-only"]) +@pytest.mark.parametrize("held_in", ["local-registry", "destination", "local-registry-only"]) def test_a_dry_run_of_a_held_release_says_nothing_to_do(tmp_path, monkeypatch, held_in): cards = _install_cards(monkeypatch, child={}, parent={}) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) # The line names a long path, so the console must not wrap it. monkeypatch.setattr(cli_module.console, "rich", Console(stderr=True, width=1000)) - held_by = remote if held_in == "destination" else store - _store_from_card(CHILD.dataset_id, held_by.root if held_by is store else remote.local_root, force=False) - options = [] if held_in == "store" else ["--publish-to", "s3://bucket/registry"] + held_by = remote if held_in == "destination" else local_registry + _store_from_card(CHILD.dataset_id, held_by.root if held_by is local_registry else remote.local_root, force=False) + options = [] if held_in == "local-registry" else ["--publish-to", "s3://bucket/registry"] result = runner.invoke(app, ["build", CHILD.dataset_id, "--dry-run", *options]) assert result.exit_code == 0, result.output - if held_in == "store-only": - # The store holds the release and the destination does not, so only the publish is left to do. + if held_in == "local-registry-only": + # The local registry holds the release and the destination does not, so only the publish is left to do. assert result.stdout.splitlines() == [f"then publish test/child@1.0.0 to {remote}"] assert result.stderr == "" else: diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 4bc063221..56c94244c 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -105,9 +105,9 @@ def default_sources( TimeNetRegistryError: If ``local_registry`` names a remote registry, or the remote registry cannot be opened. """ # ruff: ignore[docstring-extraneous-exception] (raised by settings and open_registry) cfg = settings(local_registry=local_registry, remote_registry=remote_registry, auto_build=auto_build) - store = LocalRegistry(cfg.local_registry_dir) + local = LocalRegistry(cfg.local_registry_dir) remote = open_registry(cfg.remote_registry, label="TIMENET_REMOTE_REGISTRY") - return [store, remote, *([InstalledConnectors()] if cfg.auto_build else [])] + return [local, remote, *([InstalledConnectors()] if cfg.auto_build else [])] def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None]: @@ -190,10 +190,10 @@ def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) self._sources = tuple(as_release_sources(sources, keyword="sources")) if not self._sources: raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") - self._store = next( + self._local_registry = next( (source for source in self._sources if isinstance(source, LocalRegistry) and not source.read_only), None ) - if self._store is None and any(isinstance(source, InstalledConnectors) for source in self._sources): + if self._local_registry is None and any(isinstance(source, InstalledConnectors) for source in self._sources): raise TimeFValidationError( "InstalledConnectors() needs a writable local registry to build into. " "Add a LocalRegistry without read_only=True to the sources" @@ -378,8 +378,8 @@ def download( TimeFFormatError: If a payload source breaks a pin or the payload sources form a cycle. """ # ruff: ignore[docstring-extraneous-exception] (raised by the plan and the copies) dataset_id, version = _resolve_ref(dataset_id, version) - store = self._store - if store is None: + local_registry = self._local_registry + if local_registry is None: # Without a local registry, a download has no place to go. raise TimeFValidationError( "download() needs a writable local registry to copy into. " @@ -403,13 +403,13 @@ def download( raise TimeNetDatasetNotFoundError(f"{error} Add a registry that holds it to the sources.") from error release = DatasetRef(dataset_id=dataset_id, version=answer.version) try: - plan = plan_download(release, store, sources=self._sources, force=force, lookups=lookups) + plan = plan_download(release, local_registry, sources=self._sources, force=force, lookups=lookups) except TimeNetFileError as error: # The local registry holds a damaged copy that no other registry provides. error.hint = self._file_hint(error) raise - run_plan(plan, store, options=BuildOptions(progress=progress)) - return release_dir(store.root, release) + run_plan(plan, local_registry, options=BuildOptions(progress=progress)) + return release_dir(local_registry.root, release) def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: """Open a dataset version for lazy reads, where it is. @@ -464,7 +464,9 @@ def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: """ # ruff: ignore[docstring-extraneous-exception] (raised by read_release and the builder) dataset_id, version = _resolve_ref(dataset_id, version) with self._hints(dataset_id): - return TimeFReader(read_release(dataset_id, version, sources=self._sources, store=self._store)) + return TimeFReader( + read_release(dataset_id, version, sources=self._sources, target_registry=self._local_registry) + ) def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: """Read a whole dataset version into memory, with every stored value loaded. @@ -569,20 +571,20 @@ def _file_hint(self, error: TimeNetFileError) -> str | None: Returns: A repair hint, or ``None`` if the local registry has no damaged copy or the lookup raises a TimeNet error. """ - store = self._store - if store is None: + local_registry = self._local_registry + if local_registry is None: return None damaged = error.release try: - held = store.find_manifest(damaged) - if held is None or store.damage(damaged, held) is None: + held = local_registry.find_manifest(damaged) + if held is None or local_registry.damage(damaged, held) is None: # The local registry holds no damaged copy, so the error is about another registry. return None - found = _copy_source(self._sources, store, damaged) + found = _copy_source(self._sources, local_registry, damaged) except TimeNetError: # The hint must not hide the error that it explains, for example if a registry is offline. return None - return _repair_hint(damaged, found, store) + return _repair_hint(damaged, found, local_registry) def open_torch( self, dataset_id: str, version: str | None = None, *, load: LoadPlan = LoadPlan.ALL_EAGER @@ -642,39 +644,41 @@ def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: return f"No source serves its files. Add InstalledConnectors() to the sources, or build it locally with {build}." -def _copy_source(sources: Sequence[ReleaseSource], store: LocalRegistry, release: DatasetRef) -> Found | None: +def _copy_source(sources: Sequence[ReleaseSource], local_registry: LocalRegistry, release: DatasetRef) -> Found | None: """Find the answer of a release source outside the local registry. Args: sources: The sources to search, in order. - store: The local registry to exclude. + local_registry: The local registry to exclude. release: The dataset ID and exact version. Returns: The answer of the source that provides or builds the release, or ``None`` if none exists. """ - others = [source for source in sources if not is_same_registry(source, store)] + others = [source for source in sources if not is_same_registry(source, local_registry)] answer = resolve_release(others, release.dataset_id, str(release.version)) return answer if isinstance(answer, Found) else None -def _repair_hint(release: DatasetRef, found: Found | None, store: LocalRegistry) -> str: +def _repair_hint(release: DatasetRef, found: Found | None, local_registry: LocalRegistry) -> str: """Suggest how to replace a damaged local release. Args: release: The damaged release. found: The answer of a source for the release, or ``None``. - store: The local registry that holds the damaged release. + local_registry: The local registry that holds the damaged release. Returns: A download or rebuild hint, or an explanation that no replacement is available. """ - if isinstance(found, FoundInRegistry) and not is_same_registry(found.registry, store): + if isinstance(found, FoundInRegistry) and not is_same_registry(found.registry, local_registry): # A registry other than the local registry provides the release. return f"Copy an intact release into the local registry with download('{release}')." if isinstance(found, FoundInConnector): # Only an installed connector provides the release. - command = f"timenet-build build {release.dataset_id} --local-registry {shlex.quote(str(store.root))} --force" + command = ( + f"timenet-build build {release.dataset_id} --local-registry {shlex.quote(str(local_registry.root))} --force" + ) return f"Build it again with `{command}`." # Only the local registry holds the release. return "No registry other than the local registry provides it, so download() cannot replace it." diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index 45468873e..43a2df98d 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -17,7 +17,7 @@ def plan_release_build( dataset_id: str, *, - store: LocalRegistry, + target_registry: LocalRegistry, parents_from: Sequence[ReleaseSource] = (), force: bool = False, open_access_only: bool = False, @@ -29,7 +29,7 @@ def plan_release_build( Args: dataset_id: A dataset ID such as ``org/name``, or an exact release such as ``org/name@1.2.0``. - store: The local registry to write to. + target_registry: The local registry to write to. parents_from: Sources to copy or build missing dependencies from, in preference order. force: Plan a forced rebuild and its parents even if the requested release already exists. open_access_only: Require ``access: open`` for every needed release. @@ -46,7 +46,7 @@ def plan_release_build( _, steps = plan_steps( # A connector builds only the version of its card, so "latest" asks for that version. declared(dataset_id, None if is_latest(version) else version), - store, + target_registry, parents_from=parents_from, force_build=force, open_access_only=open_access_only, @@ -54,14 +54,14 @@ def plan_release_build( return Plan(tuple(steps)) -def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = None) -> None: +def run_plan(plan: Plan, target_registry: LocalRegistry, *, options: BuildOptions | None = None) -> None: """Run a plan's copy and build steps in order. Each copied or built release gets a record of its source. Args: plan: The plan to run. - store: The local registry to copy and build into. + target_registry: The local registry to copy and build into. options: Build settings, such as the progress callback. Raises: @@ -71,7 +71,7 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the copy and the backends if any(not isinstance(step, ReadStep) for step in plan.steps): - store.check_writable("the copies and builds of a plan") + target_registry.check_writable("the copies and builds of a plan") options = options or BuildOptions() for step in plan.steps: match step: @@ -82,25 +82,25 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = copies = copy_release( step.release, step.registry, - store, + target_registry, with_payload_sources=step.with_payload_sources, force=step.force, progress=options.progress, ) for release in copies: - store.record_origin(release, OriginKind.DOWNLOAD, step.registry.location) + target_registry.record_origin(release, OriginKind.DOWNLOAD, step.registry.location) case BuildStep(): - _build(step, store, options=options) + _build(step, target_registry, options=options) case _: assert_never(step) -def _build(step: BuildStep, store: LocalRegistry, *, options: BuildOptions) -> None: +def _build(step: BuildStep, target_registry: LocalRegistry, *, options: BuildOptions) -> None: """Build the requested release with the builder of its step and record the builder's package. Args: step: The build step, with ``force`` set to rebuild an existing release. - store: The local registry to build into. + target_registry: The local registry to build into. options: The build settings. Raises: @@ -108,9 +108,11 @@ def _build(step: BuildStep, store: LocalRegistry, *, options: BuildOptions) -> N """ # ruff: ignore[docstring-extraneous-exception] - raised by prepare_local_copy if options.progress is not None: options.progress(StepStarted(step.release, "build")) - step.builder.backend.build(step.release, store, force=step.force, pinned_by=step.pinned_by, options=options) + step.builder.backend.build( + step.release, target_registry, force=step.force, pinned_by=step.pinned_by, options=options + ) # After a successful build, the local registry must hold the exact release with its payload sources. - prepare_local_copy(store.root, step.release) - store.record_origin(step.release, OriginKind.BUILD, step.builder.package) + prepare_local_copy(target_registry.root, step.release) + target_registry.record_origin(step.release, OriginKind.BUILD, step.builder.package) if options.progress is not None: options.progress(StepFinished(step.release, "build")) diff --git a/packages/timenet/src/timenet/coordinator/plan.py b/packages/timenet/src/timenet/coordinator/plan.py index 69ef4bd56..f6918d353 100644 --- a/packages/timenet/src/timenet/coordinator/plan.py +++ b/packages/timenet/src/timenet/coordinator/plan.py @@ -117,10 +117,11 @@ class _Release: role: Role """The release's role as a dataset or a source of payload files.""" - needed_in_local_registry: bool - """True if the plan needs a complete copy in the output, False if a source that serves the release is enough.""" + needed_in_target: bool + """True if the plan needs a complete copy in the target registry, False if a source that serves the release is + enough.""" step: Step | None - """The step before dependencies are set, or ``None`` for a release that the output holds.""" + """The step before dependencies are set, or ``None`` for a release that the target registry holds.""" dependencies: tuple[DatasetRef, ...] """The parent or payload source releases that must be ready first.""" @@ -201,10 +202,10 @@ def _ask(self, source: ReleaseSource, dataset_id: str, version: Version | None) class _Graph: """A graph of required releases and their read, copy or build steps.""" - output: LocalRegistry | None + target_registry: LocalRegistry | None """The local registry for copies and builds, or ``None`` for a read that copies and builds nothing.""" sources: Sequence[ReleaseSource] - """The sources in preference order. A read reads the output like any other source.""" + """The sources in preference order. A read reads the target registry like any other source.""" open_only: bool """True if each release that the walk visits must declare ``access: open``.""" builds: bool = True @@ -220,9 +221,13 @@ class _Graph: @cached_property def build_sources(self) -> list[ReleaseSource]: - """The sources without the output, which is never a source of its own copies and builds.""" - output = self.output - return [source for source in self.sources if output is None or not is_same_registry(source, output)] + """The sources without the target registry, which is never a source of its own copies and builds.""" + target_registry = self.target_registry + return [ + source + for source in self.sources + if target_registry is None or not is_same_registry(source, target_registry) + ] def visit( # ruff: ignore[too-many-arguments] self, @@ -230,22 +235,22 @@ def visit( # ruff: ignore[too-many-arguments] needed_by: DatasetRef | None, pinned: PayloadSource | None = None, *, - needed_in_local_registry: bool, + needed_in_target: bool, answer: ReleaseLookup | None = None, force: bool = False, ) -> None: """Add a required release and its dependencies to the graph. A node is planned again when a later visit needs more of it: a dataset instead of a payload source, or a - copy in the output instead of a read. + copy in the target registry instead of a read. Args: reference: The release to add. needed_by: The release that requires ``reference``, or ``None`` for the requested release. pinned: Required file records, or ``None`` for a build parent. - needed_in_local_registry: True if the plan needs a complete copy in the output. + needed_in_target: True if the plan needs a complete copy in the target registry. answer: The answer of the sources for ``reference``, or ``None`` to ask them. - force: Copy the release again, even if the output holds a complete copy. + force: Copy the release again, even if the target registry holds a complete copy. """ role: Role = "dataset" if pinned is None else "payload-source" if pinned is not None and needed_by is not None: @@ -253,10 +258,10 @@ def visit( # ruff: ignore[too-many-arguments] found = self.releases.get(reference) if found is not None: role = "dataset" if "dataset" in {found.role, role} else "payload-source" - needed_in_local_registry = found.needed_in_local_registry or needed_in_local_registry - if (role, needed_in_local_registry) == (found.role, found.needed_in_local_registry): + needed_in_target = found.needed_in_target or needed_in_target + if (role, needed_in_target) == (found.role, found.needed_in_target): return - if needed_in_local_registry: + if needed_in_target: self.local(reference, needed_by, role, force=force) else: self.read(reference, needed_by, role, answer) @@ -273,9 +278,9 @@ def read( answer: The answer of the sources for ``reference``, or ``None`` to ask them. Raises: - TimeNetFileError: If a source other than a registry provides the release and the output holds a + TimeNetFileError: If a source other than a registry provides the release and the target registry holds a damaged copy. - TimeFValidationError: If only installed connectors provide the release and the plan has no output. + TimeFValidationError: If only installed connectors provide the release and the plan has no target registry. TimeNetDatasetNotFoundError: If the registry of the answer no longer holds the release. TimeNetAccessError: If a source lists the release but no source serves it. TimeNetMissingParentError: If no source provides the payload source. @@ -291,14 +296,14 @@ def read( step = ReadStep(reference, answer.registry) self.releases[reference] = _Release(role, False, step, tuple(source.release for source in sources)) for source in sources: - self.visit(source.release, reference, source, needed_in_local_registry=False) + self.visit(source.release, reference, source, needed_in_target=False) return - # Nothing but a registry replaces a damaged copy in the output. - damage = self.in_output(reference)[1] + # Nothing but a registry replaces a damaged copy in the target registry. + damage = self.in_target(reference)[1] if damage is not None: raise damage if isinstance(answer, FoundInConnector): - if self.output is None: + if self.target_registry is None: raise TimeFValidationError( f"{reference} needs a build by {InstalledConnectors()}, but the read has no local registry to " "build into" @@ -310,34 +315,34 @@ def read( raise self.missing(reference, needed_by, role, self.sources) def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, *, force: bool = False) -> None: - """Plan a complete copy of a release in the output, from the output itself, a copy or a build. + """Plan a complete copy of a release in the target registry, from the target registry itself, a copy or a build. Args: reference: The exact release. needed_by: The release that needs ``reference``, or ``None`` for the requested release. role: Whether the release is a dataset or a payload source. - force: Copy the release again, even if the output holds a complete copy. + force: Copy the release again, even if the target registry holds a complete copy. Raises: - TimeFValidationError: If the plan has no output. + TimeFValidationError: If the plan has no target registry. TimeNetNotBuiltError: If the plan builds nothing and only installed connectors provide the release. TimeNetDatasetNotFoundError: If no source provides the requested release, or ``force`` is set and only - the output holds it. - TimeNetFileError: If the output holds a damaged copy that no other registry provides. + the target registry holds it. + TimeNetFileError: If the target registry holds a damaged copy that no other registry provides. TimeNetAccessError: If a source lists the release but no source serves it. TimeNetMissingParentError: If no source provides the release. """ # ruff: ignore[docstring-extraneous-exception] - the file, access and missing errors are raised as values - output = self.output - if output is None: + target_registry = self.target_registry + if target_registry is None: raise TimeFValidationError(f"{reference} must be in a local registry, but the read has none") - held, damage = self.in_output(reference) + held, damage = self.in_target(reference) if held is not None and not force: if self.open_only: require_open_access(held.metadata, held.lineage) sources = list(held.payload_sources.values()) if role == "dataset" else [] self.releases[reference] = _Release(role, True, None, tuple(source.release for source in sources)) for source in sources: - self.visit(source.release, reference, source, needed_in_local_registry=True) + self.visit(source.release, reference, source, needed_in_target=True) return answer = self.lookups.resolve(self.build_sources, reference.dataset_id, str(reference.version)) if isinstance(answer, FoundInRegistry): @@ -346,7 +351,7 @@ def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, # A copy from another registry repairs a damaged copy. Nothing else replaces it. raise damage elif held is not None: - # With force, the copy needs a registry other than the output, and none provides the release. + # With force, the copy needs a registry other than the target registry, and none provides the release. raise TimeNetDatasetNotFoundError( f"No registry other than the local registry provides {reference}, so download(force=True) cannot copy " "it again." @@ -359,20 +364,20 @@ def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, elif isinstance(answer, NotServed): raise answer.access_error(reference.dataset_id) else: - # No source provides the release, or only the output does. - raise self.missing(reference, needed_by, role, (output, *self.build_sources)) + # No source provides the release, or only the target registry does. + raise self.missing(reference, needed_by, role, (target_registry, *self.build_sources)) def copy(self, reference: DatasetRef, role: Role, registry: BaseRegistry, *, force: bool = False) -> None: """Add a copy step and plan the payload sources that the copy does not bring. - A dataset copy brings each payload source that ``registry`` serves. Each other payload source that the output - lacks gets its own step first. + A dataset copy brings each payload source that ``registry`` serves. Each other payload source that the target + registry lacks gets its own step first. Args: reference: The exact release to copy. role: Whether the release is a dataset or a payload source. registry: The registry to copy from. - force: Copy the release again, even if the output holds a complete copy. + force: Copy the release again, even if the target registry holds a complete copy. Raises: TimeNetDatasetNotFoundError: If the registry lacks the release. @@ -390,7 +395,7 @@ def copy(self, reference: DatasetRef, role: Role, registry: BaseRegistry, *, for manifest, registry, find=lambda source: self.listed(registry, source), - held=lambda source: self.in_output(source)[0], + held=lambda source: self.in_target(source)[0], ) if self.open_only: for found in (*sources.fetched.values(), *sources.held.values()): @@ -400,7 +405,7 @@ def copy(self, reference: DatasetRef, role: Role, registry: BaseRegistry, *, for # that depends on it closes a cycle and does not look it up again. self.releases[reference] = _Release(role, True, step, tuple(sources.missing)) for source, pinned in sources.missing.items(): - self.visit(source, reference, pinned, needed_in_local_registry=True) + self.visit(source, reference, pinned, needed_in_target=True) def build(self, reference: DatasetRef, role: Role, builder: InstalledBuilder) -> None: """Add a build step and plan the parents declared in its card. @@ -417,7 +422,7 @@ def build(self, reference: DatasetRef, role: Role, builder: InstalledBuilder) -> self.releases[reference] = _Release(role, True, BuildStep(reference, builder), parents) # A build opens each of its parents with the payload sources of that parent. for parent in parents: - self.visit(parent, reference, needed_in_local_registry=True) + self.visit(parent, reference, needed_in_target=True) @staticmethod def missing( @@ -450,22 +455,22 @@ def listed(self, registry: BaseRegistry, reference: DatasetRef) -> Manifest | No return None return self.lookups.manifest(registry, reference) - def in_output(self, reference: DatasetRef) -> tuple[Manifest | None, TimeNetFileError | None]: - """Find a release in the output and check local files for damage. + def in_target(self, reference: DatasetRef) -> tuple[Manifest | None, TimeNetFileError | None]: + """Find a release in the target registry and check local files for damage. Missing files and incorrect file sizes count as damage in a local registry. Returns: ``(manifest, None)`` if found, ``(None, error)`` if damaged, or ``(None, None)`` if absent or if the - plan has no output. + plan has no target registry. """ - if self.output is None: + if self.target_registry is None: return None, None - manifest = self.lookups.manifest(self.output, reference) + manifest = self.lookups.manifest(self.target_registry, reference) if manifest is None: return manifest, None if reference not in self.damage: - self.damage[reference] = self.output.damage(reference, manifest) + self.damage[reference] = self.target_registry.damage(reference, manifest) damage = self.damage[reference] return (manifest, None) if damage is None else (None, damage) @@ -484,8 +489,9 @@ def check(self) -> None: # One build must match every reader, so the readers must pin each file alike. _check_agreement(reference, readers) continue - # A read or a copy uses the release in its registry. A release without a step is intact in the output. - registry = self.output if step is None else step.registry + # A read or a copy uses the release in its registry. A release without a step is intact in the target + # registry. + registry = self.target_registry if step is None else step.registry manifest = None if registry is None else self.lookups.manifest(registry, reference) if manifest is None: raise TimeNetDatasetNotFoundError(f"{registry} no longer holds {reference}") @@ -501,19 +507,19 @@ def open(self, reference: DatasetRef) -> DatasetVersion: """Open a release of the plan without its payload sources, after the steps of the plan ran. Returns: - The release in the registry of its read step, else in the output. + The release in the registry of its read step, else in the target registry. Raises: - TimeFValidationError: If the release has no read step and the plan has no output. + TimeFValidationError: If the release has no read step and the plan has no target registry. """ step = self.releases[reference].step if isinstance(step, ReadStep): return step.registry.open_release(reference, manifest=self.lookups.manifest(step.registry, reference)) - if self.output is None: - # Without an output, the planner plans only read steps. So this is a bug of the planner. - raise TimeFValidationError(f"{reference} has no read step and the plan of the read has no output") - # The steps changed the output, so the release opens with a manifest that is read again. - return self.output.open_release(reference) + if self.target_registry is None: + # Without a target registry, the planner plans only read steps. So this is a bug of the planner. + raise TimeFValidationError(f"{reference} has no read step and the plan of the read has no target registry") + # The steps changed the target registry, so the release opens with a manifest that is read again. + return self.target_registry.open_release(reference) def order(self) -> list[DatasetRef]: """Order the releases so that each release comes after the releases that it depends on. @@ -589,7 +595,7 @@ def _check_agreement(release: DatasetRef, readers: Mapping[DatasetRef, PayloadSo def plan_steps( builder: InstalledBuilder, - output: LocalRegistry, + target_registry: LocalRegistry, *, parents_from: Sequence[ReleaseSource], force_build: bool, @@ -597,13 +603,13 @@ def plan_steps( ) -> tuple[DatasetRef, list[Step]]: """Plan the steps to build a dataset and its dependencies. - A damaged copy of the requested release in the output gets a forced build. + A damaged copy of the requested release in the target registry gets a forced build. Args: builder: The installed connector that builds the dataset. - output: The destination registry. + target_registry: The destination registry. parents_from: The sources to search for missing releases, in order. - force_build: Rebuild the requested release, even if the output holds it. + force_build: Rebuild the requested release, even if the target registry holds it. open_access_only: Require open access for every release. Returns: @@ -613,11 +619,11 @@ def plan_steps( if open_access_only: require_open_access(card) root = builder.release - # The graph leaves the output out of the sources of its copies and builds. - graph = _Graph(output, parents_from, open_only=open_access_only) - # If the output already holds the requested release, a reader needs only its payload sources to + # The graph leaves the target registry out of the sources of its copies and builds. + graph = _Graph(target_registry, parents_from, open_only=open_access_only) + # If the target registry already holds the requested release, a reader needs only its payload sources to # open it. Its declared parents matter only to a rebuild. - held, damage = (None, None) if force_build else graph.in_output(root) + held, damage = (None, None) if force_build else graph.in_target(root) if held is not None and open_access_only: # The installed card has no lineage, so the check above does not see an ancestor that is not open. require_open_access(held.metadata, held.lineage) @@ -627,19 +633,19 @@ def plan_steps( step = BuildStep(root, builder, force=force_build or damage is not None) graph.releases[root] = _Release("dataset", True, step, tuple(card.parents.values())) for parent in card.parents.values(): - graph.visit(parent, root, needed_in_local_registry=True) + graph.visit(parent, root, needed_in_target=True) else: sources = list(held.payload_sources.values()) graph.releases[root] = _Release("dataset", True, None, tuple(source.release for source in sources)) for source in sources: - graph.visit(source.release, root, source, needed_in_local_registry=True) + graph.visit(source.release, root, source, needed_in_target=True) graph.check() return root, graph.create_ordered_steps() def plan_download( release: DatasetRef, - store: LocalRegistry, + target_registry: LocalRegistry, *, sources: Sequence[ReleaseSource], force: bool = False, @@ -652,8 +658,8 @@ def plan_download( Args: release: The exact release. - store: The local registry to copy into. - sources: The sources to search in order, excluding registries at the directory of ``store``. + target_registry: The local registry to copy into. + sources: The sources to search in order, excluding registries at the directory of ``target_registry``. force: Copy the release again, even if the local registry holds a complete copy. lookups: Cached source answers and manifests, or ``None`` to query sources. @@ -668,9 +674,9 @@ def plan_download( TimeNetMissingParentError: If no registry provides a payload source that the local registry lacks. TimeFFormatError: If a payload source differs from the files that a release pins. """ # ruff: ignore[docstring-extraneous-exception] - raised by the walk and as values - others = [source for source in sources if not is_same_registry(source, store)] - graph = _Graph(store, others, open_only=False, builds=False, lookups=lookups or Lookups()) - graph.visit(release, None, needed_in_local_registry=True, force=force) + others = [source for source in sources if not is_same_registry(source, target_registry)] + graph = _Graph(target_registry, others, open_only=False, builds=False, lookups=lookups or Lookups()) + graph.visit(release, None, needed_in_target=True, force=force) graph.check() return Plan(tuple(graph.create_ordered_steps())) diff --git a/packages/timenet/src/timenet/coordinator/read.py b/packages/timenet/src/timenet/coordinator/read.py index f5dc30a11..ae55bf2ad 100644 --- a/packages/timenet/src/timenet/coordinator/read.py +++ b/packages/timenet/src/timenet/coordinator/read.py @@ -19,19 +19,19 @@ def plan_read( - dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], store: LocalRegistry | None + dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], target_registry: LocalRegistry | None ) -> Plan: """Plan and validate the steps needed to read a release. A release that a registry provides gets a ``"read"`` step, and a release that only installed connectors - provide gets the steps of its build into ``store``. This checks local files and payload source pins before + provide gets the steps of its build into ``target_registry``. This checks local files and payload source pins before any step runs. Args: dataset_id: The dataset ID. version: An exact version, ``"latest"``, or ``None``, as in :func:`~timenet.sources.resolve_release`. sources: Registries and ``InstalledConnectors()`` to search, in order. - store: The local registry for copies and builds, or ``None`` to copy and build nothing. + target_registry: The local registry for copies and builds, or ``None`` to copy and build nothing. Returns: The ordered plan, ending with the requested release's step if it has one. @@ -45,7 +45,7 @@ def plan_read( TimeFFormatError: If a payload source conflicts with a pin or two pins conflict. TimeFValidationError: If a build has no local registry to build into, or the releases form a cycle. """ # ruff: ignore[docstring-extraneous-exception] - raised by the planner and the checks - graph, _ = _walk(dataset_id, version, sources=sources, store=store) + graph, _ = _walk(dataset_id, version, sources=sources, target_registry=target_registry) return Plan(tuple(graph.create_ordered_steps())) @@ -54,7 +54,7 @@ def read_release( version: str | None, *, sources: Sequence[ReleaseSource], - store: LocalRegistry | None, + target_registry: LocalRegistry | None, ) -> DatasetVersion: """Open a release with its payload sources after completing required copies and builds. @@ -65,22 +65,22 @@ def read_release( dataset_id: The dataset ID. version: An exact version, ``"latest"``, or ``None``, as in :func:`~timenet.sources.resolve_release`. sources: Registries and ``InstalledConnectors()`` to search, in order. - store: The local registry for copies and builds, or ``None`` to copy and build nothing. + target_registry: The local registry for copies and builds, or ``None`` to copy and build nothing. Returns: The release with attached payload sources that match its file pins. Raises: TimeNetDatasetNotFoundError: If no source provides the release. - TimeNetVersionConflictError: If a copied release conflicts with the files in ``store``. + TimeNetVersionConflictError: If a copied release conflicts with the files in ``target_registry``. TimeFFormatError: If a build changes files pinned by other releases. TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the plan, the run and the open - graph, release = _walk(dataset_id, version, sources=sources, store=store) + graph, release = _walk(dataset_id, version, sources=sources, target_registry=target_registry) plan = Plan(tuple(graph.create_ordered_steps())) if any(not isinstance(step, ReadStep) for step in plan.steps): # The plan copies and builds only into a local registry. - run_plan(plan, cast("LocalRegistry", store)) + run_plan(plan, cast("LocalRegistry", target_registry)) return attach_payload_sources( graph.open(release), graph.open, searched=lambda: describe_searched_sources(sources, release) ) @@ -112,7 +112,7 @@ def raise_not_found(dataset_id: str, version: str | None, sources: Sequence[Rele def _walk( - dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], store: LocalRegistry | None + dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], target_registry: LocalRegistry | None ) -> tuple[_Graph, DatasetRef]: """Plan a read in one walk of the release graph, and check the plan. @@ -122,17 +122,17 @@ def _walk( Raises: TimeNetDatasetNotFoundError: If no source provides the release. """ # ruff: ignore[docstring-extraneous-exception] - raised by raise_not_found - graph = _Graph(store, sources, open_only=False) + graph = _Graph(target_registry, sources, open_only=False) answer = graph.lookups.resolve(sources, dataset_id, version) if isinstance(answer, NotFound): if not is_latest(version): # Nothing but a registry replaces a damaged copy in the local registry. - damage = graph.in_output(DatasetRef.parse(f"{dataset_id}@{version}"))[1] + damage = graph.in_target(DatasetRef.parse(f"{dataset_id}@{version}"))[1] if damage is not None: raise damage raise_not_found(dataset_id, version, sources) release = DatasetRef(dataset_id=dataset_id, version=answer.version) # The walk takes the answer, so that a read without an exact version does not ask again for the exact version. - graph.visit(release, None, needed_in_local_registry=False, answer=answer) + graph.visit(release, None, needed_in_target=False, answer=answer) graph.check() return graph, release diff --git a/packages/timenet/tests/coordinator/test_build.py b/packages/timenet/tests/coordinator/test_build.py index 70cd4310c..188b688f2 100644 --- a/packages/timenet/tests/coordinator/test_build.py +++ b/packages/timenet/tests/coordinator/test_build.py @@ -80,7 +80,7 @@ def backends(monkeypatch): @pytest.fixture -def store(tmp_path): +def target_registry(tmp_path): return LocalRegistry(tmp_path / "store") @@ -121,26 +121,26 @@ def _store_child_without_parent(registry): return registry.get_manifest(CHILD_REF.dataset_id).payload_sources[SourceKey(str(PARENT_REF))] -def test_plan_release_build_rejects_a_version_the_connector_does_not_declare(store, fake): +def test_plan_release_build_rejects_a_version_the_connector_does_not_declare(target_registry, fake): with pytest.raises(TimeNetVersionUnavailableError) as caught: - plan_release_build("a/b@2.0.0", store=store) + plan_release_build("a/b@2.0.0", target_registry=target_registry) assert (caught.value.release, caught.value.declared) == (DatasetRef.parse("a/b@2.0.0"), REF_AB.version) def test_a_plan_that_builds_refuses_a_read_only_local_registry_before_any_step(tmp_path, fake): - store = LocalRegistry(tmp_path / "store", read_only=True) + target_registry = LocalRegistry(tmp_path / "store", read_only=True) with pytest.raises(TimeNetRegistryError, match=r"read_only=True, so it refuses the copies and builds of a plan\."): - run_plan(Plan((_build(REF_AB),)), store) + run_plan(Plan((_build(REF_AB),)), target_registry) assert fake.calls == [] -def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, store, fake_parent, fake_child): +def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, target_registry, fake_parent, fake_child): origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) events = [] - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[origin]) - run_plan(plan, store, options=BuildOptions(progress=events.append)) + plan = plan_release_build(CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin]) + run_plan(plan, target_registry, options=BuildOptions(progress=events.append)) assert plan.steps == ( CopyStep(PARENT_REF, origin), @@ -154,21 +154,21 @@ def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, st StepFinished(CHILD_REF, "build"), ] assert fake_parent.calls == [] - assert store.exists(PARENT_REF.dataset_id, str(PARENT_REF.version)) + assert target_registry.exists(PARENT_REF.dataset_id, str(PARENT_REF.version)) -def test_a_build_copies_a_release_whose_payload_source_only_the_store_holds( - tmp_path, store, fake_parent, fake_child, backends +def test_a_build_copies_a_release_whose_payload_source_only_the_target_registry_holds( + tmp_path, target_registry, fake_parent, fake_child, backends ): # The registry to copy from holds the child, but not the parent whose files the child reads. origin = LocalRegistry(tmp_path / "origin") _store_child_without_parent(origin) - store.put_dataset(make_dataset()) + target_registry.put_dataset(make_dataset()) fake_grandchild = _Backend(_grandchild_card()) backends.append(fake_grandchild) - plan = plan_release_build(GRANDCHILD_REF.dataset_id, store=store, parents_from=[origin]) - run_plan(plan, store) + plan = plan_release_build(GRANDCHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin]) + run_plan(plan, target_registry) assert plan.steps == ( CopyStep(CHILD_REF, origin), @@ -176,11 +176,11 @@ def test_a_build_copies_a_release_whose_payload_source_only_the_store_holds( ) assert fake_parent.calls == fake_child.calls == [] assert fake_grandchild.calls == [(GRANDCHILD_REF, False, {})] - assert store.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) + assert target_registry.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( - tmp_path, store, fake_parent, fake_child, backends + tmp_path, target_registry, fake_parent, fake_child, backends ): # The registry to copy from holds the child, but not the parent whose files the child reads. origin = LocalRegistry(tmp_path / "origin") @@ -188,8 +188,10 @@ def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( fake_grandchild = _Backend(_grandchild_card()) backends.append(fake_grandchild) - plan = plan_release_build(GRANDCHILD_REF.dataset_id, store=store, parents_from=[origin, InstalledConnectors()]) - run_plan(plan, store) + plan = plan_release_build( + GRANDCHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin, InstalledConnectors()] + ) + run_plan(plan, target_registry) assert plan.steps == ( _build(PARENT_REF, pinned_by={CHILD_REF: pins}), @@ -199,23 +201,23 @@ def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( assert fake_parent.calls == [(PARENT_REF, False, {CHILD_REF: pins})] assert fake_child.calls == [] assert fake_grandchild.calls == [(GRANDCHILD_REF, False, {})] - assert store.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) + assert target_registry.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) -def test_run_plan_records_each_copy_and_each_build(tmp_path, store, backends): +def test_run_plan_records_each_copy_and_each_build(tmp_path, target_registry, backends): # The registry to copy from holds the child and its payload source, the parent. origin = LocalRegistry(tmp_path / "origin") _store_child_without_parent(origin) origin.put_dataset(make_dataset()) backends.extend([_Backend(_child_card()), _Backend(_grandchild_card())]) - plan = plan_release_build(GRANDCHILD_REF.dataset_id, store=store, parents_from=[origin]) + plan = plan_release_build(GRANDCHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin]) assert [(step.action, step.release) for step in plan.steps] == [("copy", CHILD_REF), ("build", GRANDCHILD_REF)] - run_plan(plan, store) + run_plan(plan, target_registry) records = {} for release in (PARENT_REF, CHILD_REF, GRANDCHILD_REF): - record = store.read_origin(release) + record = target_registry.read_origin(release) records[release] = None if record is None else (record.kind, record.source) # The copy of the child brings its payload source along, and both name the registry. The build # names the fake distribution of the fake backend. @@ -226,108 +228,119 @@ def test_run_plan_records_each_copy_and_each_build(tmp_path, store, backends): } -def test_a_forced_build_forces_only_the_requested_release(store, fake_parent, fake_child): - store.put_dataset(_demo(_child_card())) +def test_a_forced_build_forces_only_the_requested_release(target_registry, fake_parent, fake_child): + target_registry.put_dataset(_demo(_child_card())) - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()], force=True) - run_plan(plan, store) + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()], force=True + ) + run_plan(plan, target_registry) assert plan.steps == (_build(PARENT_REF), _build(CHILD_REF, depends_on=(PARENT_REF,), force=True)) assert fake_parent.calls == [(PARENT_REF, False, {})] assert fake_child.calls == [(CHILD_REF, True, {})] -def test_a_build_takes_a_parent_that_the_store_holds_and_asks_a_later_registry_nothing( - tmp_path, store, fake_parent, fake_child +def test_a_build_takes_a_parent_that_the_target_registry_holds_and_asks_a_later_registry_nothing( + tmp_path, target_registry, fake_parent, fake_child ): - store.put_dataset(make_dataset()) + target_registry.put_dataset(make_dataset()) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) transport, requests = build_service_fake(origin.root) remote = RemoteRegistry("http://api.local", transport=transport) - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[store, remote]) - run_plan(plan, store) + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[target_registry, remote] + ) + run_plan(plan, target_registry) assert plan.steps == (_build(CHILD_REF),) assert requests == [] -def test_a_build_skips_a_release_the_store_holds(store, fake): - store.put_dataset(_demo(_card(REF_AB))) - assert plan_release_build("a/b", store=store).steps == () +def test_a_build_skips_a_release_the_target_registry_holds(target_registry, fake): + target_registry.put_dataset(_demo(_card(REF_AB))) + assert plan_release_build("a/b", target_registry=target_registry).steps == () -def test_a_damaged_copy_of_the_requested_release_gets_a_forced_build(store, fake): - store.put_dataset(_demo(_card(REF_AB))) - (release_dir(store.root, REF_AB) / "control.duckdb").write_bytes(b"cut short") +def test_a_damaged_copy_of_the_requested_release_gets_a_forced_build(target_registry, fake): + target_registry.put_dataset(_demo(_card(REF_AB))) + (release_dir(target_registry.root, REF_AB) / "control.duckdb").write_bytes(b"cut short") - plan = plan_release_build("a/b", store=store) + plan = plan_release_build("a/b", target_registry=target_registry) assert plan.steps == (_build(REF_AB, force=True),) - run_plan(plan, store) + run_plan(plan, target_registry) assert fake.calls == [(REF_AB, True, {})] - assert store.damage(REF_AB) is None + assert target_registry.damage(REF_AB) is None -def test_plan_release_build_with_open_access_only_refuses_a_credentialed_card(store, backends): +def test_plan_release_build_with_open_access_only_refuses_a_credentialed_card(target_registry, backends): fake_credentialed = _Backend(_card(REF_AB, **_GATED)) backends.append(fake_credentialed) with pytest.raises(TimeNetAccessError) as caught: - plan_release_build("a/b", store=store, open_access_only=True) + plan_release_build("a/b", target_registry=target_registry, open_access_only=True) assert caught.value.dataset == REF_AB - # Without the flag, the local store takes a release that is not open. - run_plan(plan_release_build("a/b", store=store), store) + # Without the flag, the target registry takes a release that is not open. + run_plan(plan_release_build("a/b", target_registry=target_registry), target_registry) assert fake_credentialed.calls == [(REF_AB, False, {})] -def test_plan_release_build_with_open_access_only_refuses_a_held_release_with_a_credentialed_ancestor(store, backends): - # The card of the child is open. The child keeps a Record of the open release a/b, which the store +def test_plan_release_build_with_open_access_only_refuses_a_held_release_with_a_credentialed_ancestor( + target_registry, backends +): + # The card of the child is open. The child keeps a Record of the open release a/b, which the target registry # lacks. It reads nothing from the credentialed parent, which is only in its lineage. child_card = _card(CHILD_REF, parents={ParentAlias("parent"): PARENT_REF, ParentAlias("open"): REF_AB}) other, child = _Backend(_card(REF_AB)), _Backend(child_card) backends.extend([other, child]) - store.put_dataset(_demo(make_dataset().metadata.model_copy(update=_GATED))) - store.put_dataset(_demo(_card(REF_AB))) + target_registry.put_dataset(_demo(make_dataset().metadata.model_copy(update=_GATED))) + target_registry.put_dataset(_demo(_card(REF_AB))) dataset = TimeFDataset(metadata=child_card) with ( - TimeFReader(open_local_version(release_dir(store.root, PARENT_REF))) as gated_reader, - TimeFReader(open_local_version(release_dir(store.root, REF_AB))) as open_reader, + TimeFReader(open_local_version(release_dir(target_registry.root, PARENT_REF))) as gated_reader, + TimeFReader(open_local_version(release_dir(target_registry.root, REF_AB))) as open_reader, ): dataset.import_from(gated_reader, alias=ParentAlias("parent")) dataset.import_from(open_reader, alias=ParentAlias("open"), records=[make_dataset().records[0].id]) - store.put_dataset(dataset) - shutil.rmtree(store.root / PARENT_REF.dataset_id) - shutil.rmtree(release_dir(store.root, REF_AB)) - held = store.get_manifest(CHILD_REF.dataset_id) + target_registry.put_dataset(dataset) + shutil.rmtree(target_registry.root / PARENT_REF.dataset_id) + shutil.rmtree(release_dir(target_registry.root, REF_AB)) + held = target_registry.get_manifest(CHILD_REF.dataset_id) assert [card.source for card in held.lineage] == [PARENT_REF, REF_AB] assert list(held.payload_sources) == [SourceKey(str(REF_AB))] with pytest.raises(TimeNetAccessError) as caught: plan_release_build( - CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()], open_access_only=True + CHILD_REF.dataset_id, + target_registry=target_registry, + parents_from=[InstalledConnectors()], + open_access_only=True, ) assert caught.value.dataset == CHILD_REF -def test_a_plan_runs_only_through_run_plan(store, fake_parent, fake_child): - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()]) +def test_a_plan_runs_only_through_run_plan(target_registry, fake_parent, fake_child): + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()] + ) assert plan.steps == ( _build(PARENT_REF), _build(CHILD_REF, depends_on=(PARENT_REF,)), ) - assert not store.root.exists() + assert not target_registry.root.exists() assert fake_parent.calls == fake_child.calls == [] - run_plan(plan, store) + run_plan(plan, target_registry) assert fake_parent.calls == [(PARENT_REF, False, {})] assert fake_child.calls == [(CHILD_REF, False, {})] -def test_a_plan_reads_the_card_of_each_connector_once(store, backends): +def test_a_plan_reads_the_card_of_each_connector_once(target_registry, backends): reads = [] class _Counted(_Backend): @@ -336,7 +349,9 @@ def card(self, dataset_id): return super().card(dataset_id) backends.extend([_Counted(make_dataset().metadata), _Counted(_child_card())]) - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()]) - run_plan(plan, store) + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()] + ) + run_plan(plan, target_registry) assert reads == [CHILD_REF.dataset_id, PARENT_REF.dataset_id] - assert store.exists(CHILD_REF.dataset_id, str(CHILD_REF.version)) + assert target_registry.exists(CHILD_REF.dataset_id, str(CHILD_REF.version)) diff --git a/packages/timenet/tests/coordinator/test_read.py b/packages/timenet/tests/coordinator/test_read.py index fb3f8ef89..7da5b4bf0 100644 --- a/packages/timenet/tests/coordinator/test_read.py +++ b/packages/timenet/tests/coordinator/test_read.py @@ -95,11 +95,11 @@ def _install_demo_backend(monkeypatch, *cards) -> list: return calls -def _child_reading(store, *cards, child_release=CHILD): +def _child_reading(target_registry, *cards, child_release=CHILD): """Store a child, which reads one Record of the demo dataset under each card, without those releases. Args: - store: The registry to store the child in. + target_registry: The registry to store the child in. *cards: The cards of the releases that the child reads. child_release: The release of the child. @@ -112,13 +112,13 @@ def _child_reading(store, *cards, child_release=CHILD): } child = TimeFDataset(metadata=_card(child_release, parents=parents)) for (alias, parent), card in zip(parents.items(), cards, strict=True): - store.put_dataset(_demo(card)) - with TimeFReader(open_local_version(release_dir(store.root, parent))) as reader: + target_registry.put_dataset(_demo(card)) + with TimeFReader(open_local_version(release_dir(target_registry.root, parent))) as reader: child.import_from(reader, alias=alias, records=[make_dataset().records[0].id]) - store.put_dataset(child) + target_registry.put_dataset(child) for parent in parents.values(): - shutil.rmtree(release_dir(store.root, parent)) - manifest = store.get_manifest(child_release.dataset_id) + shutil.rmtree(release_dir(target_registry.root, parent)) + manifest = target_registry.get_manifest(child_release.dataset_id) return {pinned.release: pinned for pinned in manifest.payload_sources.values()} @@ -147,35 +147,40 @@ def counted(dataset_id, version=None): def test_a_read_reads_in_place_from_the_first_source_that_holds_the_release(tmp_path): - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - plan = plan_read(DEMO.dataset_id, None, sources=[store, origin], store=store) + plan = plan_read(DEMO.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) assert plan.steps == (ReadStep(DEMO, origin),) - version = read_release(DEMO.dataset_id, None, sources=[store, origin], store=store) + version = read_release(DEMO.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) assert Path(version.root).is_relative_to(origin.root) - assert not store.root.exists() + assert not target_registry.root.exists() def test_a_damaged_copy_fails_the_read_before_any_step_runs(tmp_path, monkeypatch): - store, _ = _child_of_the_demo(tmp_path / "store") + target_registry, _ = _child_of_the_demo(tmp_path / "store") # No registry holds DEMO, so the plan builds it before the read opens CHILD. - shutil.rmtree(release_dir(store.root, DEMO)) + shutil.rmtree(release_dir(target_registry.root, DEMO)) # Origin and the connector also provide CHILD, but the read stops at the damaged copy, which comes first. origin = LocalRegistry(tmp_path / "origin") - shutil.copytree(store.root, origin.root) - (release_dir(store.root, CHILD) / "control.duckdb").write_bytes(b"cut short") + shutil.copytree(target_registry.root, origin.root) + (release_dir(target_registry.root, CHILD) / "control.duckdb").write_bytes(b"cut short") calls = _install_demo_backend(monkeypatch, _card(CHILD, parents={ParentAlias("parent"): DEMO})) with pytest.raises(TimeNetFileError, match=r"^test/child@1\.0\.0: file 'control\.duckdb' has 9 bytes"): - read_release(CHILD.dataset_id, None, sources=[store, origin, InstalledConnectors()], store=store) + read_release( + CHILD.dataset_id, + None, + sources=[target_registry, origin, InstalledConnectors()], + target_registry=target_registry, + ) assert calls == [] @pytest.mark.parametrize("answer", ["build", "not-found", "not-served"]) -def test_a_damaged_copy_in_the_store_fails_a_read_that_no_registry_serves(tmp_path, monkeypatch, answer): - # The store is not in the list, so no registry answers for its damaged copy. - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) - (release_dir(store.root, DEMO) / "control.duckdb").write_bytes(b"cut short") +def test_a_damaged_copy_in_the_target_registry_fails_a_read_that_no_registry_serves(tmp_path, monkeypatch, answer): + # The target registry is not in the list, so no registry answers for its damaged copy. + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) + (release_dir(target_registry.root, DEMO) / "control.duckdb").write_bytes(b"cut short") _install_demo_backend(monkeypatch) sources: list[ReleaseSource] if answer == "build": @@ -186,70 +191,70 @@ def test_a_damaged_copy_in_the_store_fails_a_read_that_no_registry_serves(tmp_pa # The damage error comes before the access error of the registry that lists DEMO. sources = [_gated_remote(tmp_path)] with pytest.raises(TimeNetFileError, match=r"^timenet/hello-world@1\.0\.0: file 'control\.duckdb' has 9 bytes"): - plan_read(DEMO.dataset_id, "1.0.0", sources=sources, store=store) + plan_read(DEMO.dataset_id, "1.0.0", sources=sources, target_registry=target_registry) def test_a_payload_source_resolves_on_its_own(tmp_path): - store, _ = _child_of_the_demo(tmp_path / "store") - shutil.rmtree(release_dir(store.root, DEMO)) + target_registry, _ = _child_of_the_demo(tmp_path / "store") + shutil.rmtree(release_dir(target_registry.root, DEMO)) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - plan = plan_read(CHILD.dataset_id, None, sources=[store, origin], store=store) - assert plan.steps == (ReadStep(DEMO, origin), ReadStep(CHILD, store, depends_on=(DEMO,))) - version = read_release(CHILD.dataset_id, None, sources=[store, origin], store=store) - assert Path(version.root).is_relative_to(store.root) + plan = plan_read(CHILD.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) + assert plan.steps == (ReadStep(DEMO, origin), ReadStep(CHILD, target_registry, depends_on=(DEMO,))) + version = read_release(CHILD.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) + assert Path(version.root).is_relative_to(target_registry.root) assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(origin.root) - assert not store.exists(DEMO.dataset_id, str(DEMO.version)) + assert not target_registry.exists(DEMO.dataset_id, str(DEMO.version)) def test_a_payload_source_with_other_files_than_the_pins_fails_the_plan(tmp_path): - store, _ = _child_of_the_demo(tmp_path / "store") - shutil.rmtree(release_dir(store.root, DEMO)) + target_registry, _ = _child_of_the_demo(tmp_path / "store") + shutil.rmtree(release_dir(target_registry.root, DEMO)) origin = LocalRegistry(tmp_path / "origin") # The same release with its values in another format: CHILD pins files that it does not have. origin.put_dataset(make_dataset(), values_backend="zarr") with pytest.raises(TimeFFormatError, match=r"^payload source timenet/hello-world@1\.0\.0 differs"): - plan_read(CHILD.dataset_id, None, sources=[store, origin], store=store) + plan_read(CHILD.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) def test_a_missing_payload_source_builds_against_the_pins_of_the_release(tmp_path, monkeypatch): - store, _ = _child_of_the_demo(tmp_path / "store") - shutil.rmtree(release_dir(store.root, DEMO)) - pins = store.get_manifest(CHILD.dataset_id).payload_sources[SourceKey(str(DEMO))] + target_registry, _ = _child_of_the_demo(tmp_path / "store") + shutil.rmtree(release_dir(target_registry.root, DEMO)) + pins = target_registry.get_manifest(CHILD.dataset_id).payload_sources[SourceKey(str(DEMO))] calls = _install_demo_backend(monkeypatch) - sources = [store, InstalledConnectors()] - assert plan_read(CHILD.dataset_id, None, sources=sources, store=store).steps == ( + sources = [target_registry, InstalledConnectors()] + assert plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry).steps == ( _build(DEMO, pinned_by={CHILD: pins}), - ReadStep(CHILD, store, depends_on=(DEMO,)), + ReadStep(CHILD, target_registry, depends_on=(DEMO,)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(DEMO, False, {CHILD: pins})] - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) def test_a_read_builds_a_release_that_only_a_connector_provides(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) calls = _install_demo_backend(monkeypatch, _card(CHILD, parents={ParentAlias("parent"): DEMO})) - sources = [store, InstalledConnectors()] - # The build of CHILD reads DEMO, which the store holds, so DEMO gets no step. - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + sources = [target_registry, InstalledConnectors()] + # The build of CHILD reads DEMO, which the target registry holds, so DEMO gets no step. + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert plan.steps == (_build(CHILD),) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(CHILD, False, {})] - assert Path(version.root).is_relative_to(store.root) - # A payload source without a step opens from the store. - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + assert Path(version.root).is_relative_to(target_registry.root) + # A payload source without a step opens from the target registry. + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) @pytest.mark.parametrize(("version", "release"), [(None, "x/y"), ("1.0.0", "x/y@1.0.0")]) def test_a_release_that_no_source_provides_names_every_source(tmp_path, monkeypatch, version, release): - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") _install_demo_backend(monkeypatch) with pytest.raises(TimeNetDatasetNotFoundError) as caught: - plan_read("x/y", version, sources=[store, InstalledConnectors()], store=store) + plan_read("x/y", version, sources=[target_registry, InstalledConnectors()], target_registry=target_registry) assert type(caught.value) is TimeNetDatasetNotFoundError - assert str(caught.value) == f"{release} is in none of the sources: {store}, installed connectors." + assert str(caught.value) == f"{release} is in none of the sources: {target_registry}, installed connectors." def test_a_missing_payload_source_names_the_version_that_the_connector_builds(tmp_path, monkeypatch): @@ -257,58 +262,66 @@ def test_a_missing_payload_source_names_the_version_that_the_connector_builds(tm shutil.rmtree(release_dir(origin.root, DEMO)) _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) with pytest.raises(TimeNetMissingParentError) as caught: - plan_read(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], store=None) + plan_read(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], target_registry=None) assert str(caught.value) == ( f"payload source {DEMO} of {CHILD} is not in {origin} or installed connectors ({DEMO.dataset_id} builds 2.0.0)." ) -def test_pins_that_the_store_copy_of_a_payload_source_breaks_fail_the_read_before_any_step(tmp_path, monkeypatch): - # Origin holds CHILD, which reads OTHER and DEMO. Only the connectors provide them. The store +def test_pins_that_the_target_registry_copy_of_a_payload_source_breaks_fail_the_read_before_any_step( + tmp_path, monkeypatch +): + # Origin holds CHILD, which reads OTHER and DEMO. Only the connectors provide them. The target registry # holds DEMO with its values in another format, so the plan builds nothing that checks the pins # of CHILD on DEMO. The plan builds OTHER first. - origin, store = LocalRegistry(tmp_path / "origin"), LocalRegistry(tmp_path / "store") + origin, target_registry = LocalRegistry(tmp_path / "origin"), LocalRegistry(tmp_path / "store") _child_reading(origin, _card(OTHER), make_dataset().metadata) - store.put_dataset(make_dataset(), values_backend="zarr") + target_registry.put_dataset(make_dataset(), values_backend="zarr") calls = _install_demo_backend(monkeypatch, _card(OTHER)) with pytest.raises(TimeFFormatError, match=r"^payload source timenet/hello-world@1\.0\.0 differs"): - read_release(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], store=store) + read_release(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], target_registry=target_registry) assert calls == [] def test_a_release_that_a_source_lists_without_serving_raises_the_access_error(tmp_path): with pytest.raises(TimeNetAccessError) as caught: - read_release(DEMO.dataset_id, None, sources=[_gated_remote(tmp_path)], store=None) + read_release(DEMO.dataset_id, None, sources=[_gated_remote(tmp_path)], target_registry=None) assert caught.value.requirements == (AccessRequirement(DEMO, Access.CREDENTIALED, _GATED["access_url"]),) def test_an_exact_version_that_the_connector_does_not_declare_raises_version_unavailable(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") _install_demo_backend(monkeypatch) with pytest.raises(TimeNetVersionUnavailableError) as caught: - read_release(DEMO.dataset_id, "2.0.0", sources=[store, InstalledConnectors()], store=store) + read_release( + DEMO.dataset_id, "2.0.0", sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) assert (caught.value.release, caught.value.declared) == (DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"), DEMO.version) def test_latest_without_the_connectors_in_the_list_reads_the_older_copy_and_builds_nothing(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) # The installed connector builds a newer version, but the list does not hold the connectors. calls = _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) - assert plan_read(DEMO.dataset_id, "latest", sources=[store], store=store).steps == (ReadStep(DEMO, store),) - version = read_release(DEMO.dataset_id, "latest", sources=[store], store=store) + assert plan_read(DEMO.dataset_id, "latest", sources=[target_registry], target_registry=target_registry).steps == ( + ReadStep(DEMO, target_registry), + ) + version = read_release(DEMO.dataset_id, "latest", sources=[target_registry], target_registry=target_registry) assert version.manifest.metadata.dataset_version == DEMO.version assert calls == [] def test_a_read_without_a_version_stops_at_the_first_source_that_serves_it(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) # A later registry cannot be reached, and the installed connector builds a newer version. down = RemoteRegistry("http://down.local", transport=httpx.MockTransport(_offline)) calls = _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) - version = read_release(DEMO.dataset_id, None, sources=[store, down, InstalledConnectors()], store=store) + version = read_release( + DEMO.dataset_id, None, sources=[target_registry, down, InstalledConnectors()], target_registry=target_registry + ) assert version.manifest.metadata.dataset_version == DEMO.version assert calls == [] @@ -323,7 +336,7 @@ def card(self, dataset_id): return super().card(dataset_id) install_backends(monkeypatch, _Counted((make_dataset().metadata,), [])) - plan_read(DEMO.dataset_id, None, sources=[InstalledConnectors()], store=LocalRegistry(tmp_path / "store")) + plan_read(DEMO.dataset_id, None, sources=[InstalledConnectors()], target_registry=LocalRegistry(tmp_path / "store")) assert reads == [DEMO.dataset_id] @@ -340,55 +353,62 @@ def card(self, dataset_id): return super().card(dataset_id) install_backends(monkeypatch, _Counted((make_dataset().metadata,), [])) - store, _ = _child_of_the_demo(tmp_path / "store") - read_release(CHILD.dataset_id, None, sources=[store, InstalledConnectors()], store=store) + target_registry, _ = _child_of_the_demo(tmp_path / "store") + read_release( + CHILD.dataset_id, None, sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) # Only the error for a missing payload source names the sources, with the versions that the connectors build. assert asked == [] -def test_a_read_that_needs_a_build_without_a_store_raises(monkeypatch): +def test_a_read_that_needs_a_build_without_a_target_registry_raises(monkeypatch): _install_demo_backend(monkeypatch) with pytest.raises(TimeFValidationError, match=r"timenet/hello-world@1\.0\.0 needs a build"): - read_release(DEMO.dataset_id, None, sources=[InstalledConnectors()], store=None) + read_release(DEMO.dataset_id, None, sources=[InstalledConnectors()], target_registry=None) @pytest.mark.parametrize("parent", [PARENT, DatasetRef.parse("x/parent@1.0.0")], ids=["build-first", "read-first"]) -def test_a_release_that_a_build_copies_is_read_from_the_store_and_appears_once(tmp_path, monkeypatch, parent): +def test_a_release_that_a_build_copies_is_read_from_the_target_registry_and_appears_once(tmp_path, monkeypatch, parent): # CHILD reads the parent and DEMO. Only the connectors build the parent, and its card declares - # DEMO, which origin holds. So the build of the parent copies DEMO into the store, and the read + # DEMO, which origin holds. So the build of the parent copies DEMO into the target registry, and the read # takes it there. The read plans the payload sources in the order of their names. parent_card = _card(parent, parents={ParentAlias("demo"): DEMO}) - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - pins = _child_reading(store, parent_card, make_dataset().metadata) + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + pins = _child_reading(target_registry, parent_card, make_dataset().metadata) origin.put_dataset(make_dataset()) calls = _install_demo_backend(monkeypatch, parent_card) - sources = [store, origin, InstalledConnectors()] + sources = [target_registry, origin, InstalledConnectors()] reads = _count_reads(monkeypatch, origin) - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) # The read and the build of the parent share what the plan read. assert reads == {str(DEMO): 1} assert plan.steps == ( CopyStep(DEMO, origin), _build(parent, depends_on=(DEMO,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=tuple(pins)), + ReadStep(CHILD, target_registry, depends_on=tuple(pins)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(parent, False, {CHILD: pins[parent]})] - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) def test_a_step_that_two_builds_need_appears_once(tmp_path, monkeypatch): # CHILD reads PARENT and OTHER. Only the connectors build them, and both of their cards declare # DEMO, which origin holds. So both builds copy DEMO first. cards = [_card(reference, parents={ParentAlias("demo"): DEMO}) for reference in (PARENT, OTHER)] - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - pins = _child_reading(store, *cards) + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + pins = _child_reading(target_registry, *cards) origin.put_dataset(make_dataset()) _install_demo_backend(monkeypatch, *cards) - plan = plan_read(CHILD.dataset_id, None, sources=[store, origin, InstalledConnectors()], store=store) + plan = plan_read( + CHILD.dataset_id, + None, + sources=[target_registry, origin, InstalledConnectors()], + target_registry=target_registry, + ) assert [(step.release, step.action) for step in plan.steps] == [ (DEMO, "copy"), @@ -405,46 +425,48 @@ def test_two_builds_of_one_release_merge_into_one_build_with_the_pins_of_each(tm # the parent builds OTHER without pins, and the read builds OTHER with the pins of CHILD. The read # plans the payload sources in the order of their names. cards = [_card(parent, parents={ParentAlias("other"): OTHER}), _card(OTHER)] - store = LocalRegistry(tmp_path / "store") - pins = _child_reading(store, *cards) + target_registry = LocalRegistry(tmp_path / "store") + pins = _child_reading(target_registry, *cards) calls = _install_demo_backend(monkeypatch, *cards) - sources = [store, InstalledConnectors()] + sources = [target_registry, InstalledConnectors()] - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) # The one build of OTHER comes before the build of the parent, which needs it. assert plan.steps == ( _build(OTHER, pinned_by={CHILD: pins[OTHER]}), _build(parent, depends_on=(OTHER,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=tuple(pins)), + ReadStep(CHILD, target_registry, depends_on=tuple(pins)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(OTHER, False, {CHILD: pins[OTHER]}), (parent, False, {CHILD: pins[parent]})] - assert Path(version.payload_sources[SourceKey(str(OTHER))].root).is_relative_to(store.root) + assert Path(version.payload_sources[SourceKey(str(OTHER))].root).is_relative_to(target_registry.root) def test_two_readers_that_pin_one_file_differently_fail_the_read_before_any_step(tmp_path, monkeypatch): - # CHILD reads PARENT and OTHER. The card of PARENT declares X, which the store holds, and which + # CHILD reads PARENT and OTHER. The card of PARENT declares X, which the target registry holds, and which # reads OTHER too. So the read builds OTHER with the pins of CHILD, and the build of PARENT builds # OTHER with the pins of X. X pins one file of OTHER with another checksum, so no build matches both. x = DatasetRef.parse("test/x@1.0.0") cards = [_card(PARENT, parents={ParentAlias("x"): x}), _card(OTHER)] - store = LocalRegistry(tmp_path / "store") - _child_reading(store, cards[1], child_release=x) - _child_reading(store, *cards) - manifest = store.get_manifest(x.dataset_id) + target_registry = LocalRegistry(tmp_path / "store") + _child_reading(target_registry, cards[1], child_release=x) + _child_reading(target_registry, *cards) + manifest = target_registry.get_manifest(x.dataset_id) ((key, pinned),) = manifest.payload_sources.items() changed = pinned.files[0].model_copy(update={"checksum": "sha256:" + "0" * 64}) pinned = pinned.model_copy(update={"files": (changed, *pinned.files[1:])}) - (release_dir(store.root, x) / "manifest.json").write_text( + (release_dir(target_registry.root, x) / "manifest.json").write_text( manifest.model_copy(update={"payload_sources": {key: pinned}}).to_json() ) # A new registry, so that no manifest that it read before stays. - store = LocalRegistry(store.root) + target_registry = LocalRegistry(target_registry.root) calls = _install_demo_backend(monkeypatch, *cards) message = f"{CHILD} and {x} pin {changed.path!r} of {OTHER} as different files" with pytest.raises(TimeFFormatError, match=f"^{re.escape(message)}"): - read_release(CHILD.dataset_id, None, sources=[store, InstalledConnectors()], store=store) + read_release( + CHILD.dataset_id, None, sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) assert calls == [] @@ -454,7 +476,7 @@ def test_a_remote_read_fetches_each_manifest_once(tmp_path, monkeypatch): remote = RemoteRegistry("http://api.local", transport=transport) reads = _count_reads(monkeypatch, remote) - version = read_release(CHILD.dataset_id, None, sources=[remote], store=None) + version = read_release(CHILD.dataset_id, None, sources=[remote], target_registry=None) assert version.manifest == origin.get_manifest(CHILD.dataset_id) assert version.payload_sources[SourceKey(str(DEMO))].manifest == origin.get_manifest(DEMO.dataset_id) @@ -468,35 +490,35 @@ def test_latest_reads_the_manifest_of_the_version_that_the_answer_found(tmp_path manifest = origin.get_manifest(DEMO.dataset_id, str(DEMO.version)) reads = _count_reads(monkeypatch, origin) # A local registry answers "latest" without its manifest, so the plan reads it at the exact version. - version = read_release(DEMO.dataset_id, None, sources=[origin], store=None) + version = read_release(DEMO.dataset_id, None, sources=[origin], target_registry=None) assert version.manifest == manifest assert reads == {str(DEMO): 1} @pytest.mark.parametrize("parent", [PARENT, DatasetRef.parse("x/parent@1.0.0")], ids=["build-first", "read-first"]) def test_a_copy_that_a_build_needs_repairs_a_damaged_copy_that_the_read_found_first(tmp_path, monkeypatch, parent): - # CHILD reads the parent and DEMO. The store comes first in the list and holds DEMO damaged, and origin + # CHILD reads the parent and DEMO. The target registry comes first in the list and holds DEMO damaged, and origin # holds DEMO. Only the connectors build the parent, and its card declares DEMO. So the build copies DEMO # from origin over the damaged copy, and the read takes it there. parent_card = _card(parent, parents={ParentAlias("demo"): DEMO}) - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - pins = _child_reading(store, parent_card, make_dataset().metadata) - for registry in (store, origin): + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + pins = _child_reading(target_registry, parent_card, make_dataset().metadata) + for registry in (target_registry, origin): registry.put_dataset(make_dataset()) - (release_dir(store.root, DEMO) / "control.duckdb").write_bytes(b"cut short") + (release_dir(target_registry.root, DEMO) / "control.duckdb").write_bytes(b"cut short") _install_demo_backend(monkeypatch, parent_card) - sources = [store, origin, InstalledConnectors()] + sources = [target_registry, origin, InstalledConnectors()] - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert plan.steps == ( CopyStep(DEMO, origin), _build(parent, depends_on=(DEMO,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=tuple(pins)), + ReadStep(CHILD, target_registry, depends_on=tuple(pins)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) - assert store.damage(DEMO) is None - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) + assert target_registry.damage(DEMO) is None + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) @@ -517,36 +539,38 @@ def _store_reading(registry: LocalRegistry, release: DatasetRef, source: Dataset def test_a_payload_source_that_a_build_needs_as_a_parent_comes_with_its_own_payload_sources(tmp_path, monkeypatch): # CHILD reads the own Record of SOURCE, which reads DEMO, and a Record of the parent. Origin holds SOURCE # and DEMO. Only the connectors build the parent, and its card declares SOURCE. The read finds SOURCE in - # origin first. The build then needs SOURCE in the store, so the copy takes DEMO along. + # origin first. The build then needs SOURCE in the target registry, so the copy takes DEMO along. source, parent = DatasetRef.parse("test/source@1.0.0"), DatasetRef.parse("x/parent@1.0.0") parent_card = _card(parent, parents={ParentAlias("source"): source}) - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - store.put_dataset(make_dataset()) - _store_reading(store, source, DEMO) - store.put_dataset(_demo(parent_card)) + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + target_registry.put_dataset(make_dataset()) + _store_reading(target_registry, source, DEMO) + target_registry.put_dataset(_demo(parent_card)) child = TimeFDataset(metadata=_card(CHILD, parents={ParentAlias("source"): source, ParentAlias("parent"): parent})) for alias, reference, record in (("source", source, 2), ("parent", parent, make_dataset().records[0].id)): - with TimeFReader(open_local_version(release_dir(store.root, reference))) as reader: + with TimeFReader(open_local_version(release_dir(target_registry.root, reference))) as reader: child.import_from(reader, alias=ParentAlias(alias), records=[record]) - store.put_dataset(child) + target_registry.put_dataset(child) for reference in (DEMO, source, parent): - shutil.copytree(release_dir(store.root, reference), release_dir(origin.root, reference)) - shutil.rmtree(release_dir(store.root, reference)) + shutil.copytree(release_dir(target_registry.root, reference), release_dir(origin.root, reference)) + shutil.rmtree(release_dir(target_registry.root, reference)) shutil.rmtree(release_dir(origin.root, parent)) - pins = {pinned.release: pinned for pinned in store.get_manifest(CHILD.dataset_id).payload_sources.values()} + pins = { + pinned.release: pinned for pinned in target_registry.get_manifest(CHILD.dataset_id).payload_sources.values() + } assert list(pins) == [source, parent] _install_demo_backend(monkeypatch, parent_card) - sources = [store, origin, InstalledConnectors()] + sources = [target_registry, origin, InstalledConnectors()] - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert plan.steps == ( CopyStep(source, origin), _build(parent, depends_on=(source,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=(source, parent)), + ReadStep(CHILD, target_registry, depends_on=(source, parent)), ) - read_release(CHILD.dataset_id, None, sources=sources, store=store) - assert store.exists(DEMO.dataset_id, str(DEMO.version)) + read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) + assert target_registry.exists(DEMO.dataset_id, str(DEMO.version)) def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatch): @@ -559,7 +583,7 @@ def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatc registries.append(RemoteRegistry(f"http://{name}.local", transport=transport)) reads = _count_reads(monkeypatch, registries[0]) - version = read_release(DEMO.dataset_id, "latest", sources=registries, store=None) + version = read_release(DEMO.dataset_id, "latest", sources=registries, target_registry=None) assert version.manifest.metadata.dataset_version == newer.version # The plan takes the answer of the sources for the latest version, and asks for 2.0.0 nowhere. @@ -569,10 +593,12 @@ def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatc def test_a_cycle_through_the_requested_release_starts_at_it(tmp_path, monkeypatch): # CHILD reads PARENT, which only the connectors build, and the card of PARENT declares CHILD. parent_card = _card(PARENT, parents={ParentAlias("child"): CHILD}) - store = LocalRegistry(tmp_path / "store") - _child_reading(store, parent_card) + target_registry = LocalRegistry(tmp_path / "store") + _child_reading(target_registry, parent_card) _install_demo_backend(monkeypatch, parent_card) message = f"dataset parent cycle: {CHILD} -> {PARENT} -> {CHILD}" with pytest.raises(TimeFValidationError, match=f"^{re.escape(message)}$"): - plan_read(CHILD.dataset_id, None, sources=[store, InstalledConnectors()], store=store) + plan_read( + CHILD.dataset_id, None, sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) diff --git a/packages/timenet/tests/registry/test_origin.py b/packages/timenet/tests/registry/test_origin.py index 88d8dff54..7ba2171c5 100644 --- a/packages/timenet/tests/registry/test_origin.py +++ b/packages/timenet/tests/registry/test_origin.py @@ -74,7 +74,7 @@ def test_a_release_without_a_record_has_none(store): assert store.read_origin(DEMO) is None -def test_a_record_needs_the_release_in_the_store(tmp_path): +def test_a_record_needs_the_release_in_the_local_registry(tmp_path): empty = LocalRegistry(tmp_path / "store") with pytest.raises(TimeNetDatasetNotFoundError, match=f"the local registry holds no {DEMO}"): empty.record_origin(DEMO, OriginKind.DOWNLOAD, "https://registry.example.org") diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 467dfe285..aa62730d5 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -204,7 +204,9 @@ def test_full_download_writes_identical_bytes(version_dir, tmp_path): assert (dest / "manifest.json").exists() -def test_a_download_into_the_store_reads_the_remote_manifest_for_the_plan_and_for_the_copy(version_dir, tmp_path): +def test_a_download_into_the_local_registry_reads_the_remote_manifest_for_the_plan_and_for_the_copy( + version_dir, tmp_path +): registry, requests = _remote(version_dir, tmp_path) directory, manifest = version_dir store = LocalRegistry(tmp_path / "store") diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index 75bbca840..fa57bb601 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -98,7 +98,7 @@ def test_objects_live_under_a_datasets_prefix(s3_root, tmp_path): assert all(key.startswith("registry/datasets/") for key in keys) -def test_download_then_load_reads_the_store_copy(s3_root, tmp_path): +def test_download_then_load_reads_the_local_registry_copy(s3_root, tmp_path): registry = S3Registry(s3_root) dataset_id = make_dataset().metadata.dataset_id registry.put_dataset(make_dataset()) diff --git a/packages/timenet/tests/test_cache.py b/packages/timenet/tests/test_cache.py index 87100611e..d25c61561 100644 --- a/packages/timenet/tests/test_cache.py +++ b/packages/timenet/tests/test_cache.py @@ -196,7 +196,7 @@ def test_releases_skips_an_unfinished_write_of_a_dataset_folder(tmp_path): assert [str(release) for release in store.releases()] == ["timenet/hello-world@1.0.0"] -def test_clear_cache_keeps_a_store_inside_the_cache(tmp_path): +def test_clear_cache_keeps_a_local_registry_inside_the_cache(tmp_path): cache = tmp_path / "cache" store = LocalRegistry(cache / "registry") store.put_dataset(make_dataset()) @@ -306,7 +306,7 @@ def test_clear_cache_unlinks_a_symlinked_downloads_folder_only_if_marked_and_kee assert (target / "raw.bin").exists() -def test_clear_cache_refuses_a_store_in_a_folder_that_it_deletes(tmp_path): +def test_clear_cache_refuses_a_local_registry_in_a_folder_that_it_deletes(tmp_path): make_cache_folder(tmp_path / "cache" / "downloads") store = LocalRegistry(tmp_path / "cache" / "downloads" / "store") store.put_dataset(make_dataset()) @@ -315,7 +315,7 @@ def test_clear_cache_refuses_a_store_in_a_folder_that_it_deletes(tmp_path): assert store.exists("timenet/hello-world", "1.0.0") -def test_clear_cache_refuses_a_folder_that_it_deletes_in_the_store(tmp_path): +def test_clear_cache_refuses_a_folder_that_it_deletes_in_the_local_registry(tmp_path): # With the cache in the store, the downloads folder can be the directory of a dataset. store = LocalRegistry(tmp_path / "store") make_cache_folder(tmp_path / "store" / "acme" / "downloads") @@ -416,7 +416,7 @@ def test_clearing_the_downloads_with_a_damaged_manifest_of_a_build_deletes_nothi assert {item.release for item in stored_releases(store)} == {BUILT, PARENT, GRAND, OTHER, HAND} -def test_clearing_the_downloads_refuses_a_store_with_a_symlinked_dataset_folder(tmp_path): +def test_clearing_the_downloads_refuses_a_local_registry_with_a_symlinked_dataset_folder(tmp_path): # The listing skips a symlinked folder, so it would miss BUILT and delete the downloads that it needs. store = LocalRegistry(tmp_path / "store") _layout(store) @@ -430,7 +430,7 @@ def test_clearing_the_downloads_refuses_a_store_with_a_symlinked_dataset_folder( assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} -def test_clearing_every_kind_refuses_a_store_with_a_symlinked_release_folder(tmp_path): +def test_clearing_every_kind_refuses_a_local_registry_with_a_symlinked_release_folder(tmp_path): store = LocalRegistry(tmp_path / "store") _layout(store) _record(store) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index 87658e067..591a7fa08 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -401,7 +401,9 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): assert ["records", str(len(child.records))] in cells -def test_download_command_copies_into_the_store_and_then_has_nothing_to_do(registry_root, tmp_path, monkeypatch): +def test_download_command_copies_into_the_local_registry_and_then_has_nothing_to_do( + registry_root, tmp_path, monkeypatch +): # The last line names a long path, so the console must not wrap it. monkeypatch.setattr(_cli_module.ui, "rich", Console(stderr=True, width=1000)) monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) @@ -450,7 +452,7 @@ def test_each_command_builds_the_list_from_its_flags(registry_root, tmp_path): assert result.stdout == f"{store / 'timenet/hello-world/1.0.0'}\n" -def test_publish_copies_a_release_from_the_store(tmp_path): +def test_publish_copies_a_release_from_the_local_registry(tmp_path): store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) dest = tmp_path / "dest" @@ -491,7 +493,7 @@ def test_publish_needs_dest(tmp_path): assert "--dest" in click.unstyle(result.output) -def test_publish_of_a_release_that_the_store_lacks_names_the_fix(tmp_path): +def test_publish_of_a_release_that_the_local_registry_lacks_names_the_fix(tmp_path): store, other, dest = tmp_path / "store", tmp_path / "other", str(tmp_path / "dest") result = runner.invoke(app, ["publish", "timenet/hello-world", "--dest", dest, "--local-registry", str(store)]) assert type(result.exception) is TimeNetDatasetNotFoundError @@ -511,7 +513,7 @@ def test_publish_of_a_release_that_the_store_lacks_names_the_fix(tmp_path): assert str(result.exception) == f"timenet/hello-world is not in local registry {other}." -def test_publish_adds_no_hint_to_a_payload_source_that_the_store_lacks(tmp_path): +def test_publish_adds_no_hint_to_a_payload_source_that_the_local_registry_lacks(tmp_path): store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) child = TimeFDataset( diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 7b0850d87..f9771d96c 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -146,7 +146,7 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p assert reader.read().records[0].signals[0].to_arrow().equals(expected) -def test_a_damaged_release_outside_the_store_names_no_download(tmp_path): +def test_a_damaged_release_outside_the_local_registry_names_no_download(tmp_path): origin, _ = _child_of_the_demo(tmp_path / "origin") mirror = LocalRegistry(tmp_path / "mirror") shutil.copytree(origin.root, mirror.root) @@ -159,7 +159,7 @@ def test_a_damaged_release_outside_the_store_names_no_download(tmp_path): assert (caught.value.release, caught.value.hint) == (DatasetRef.parse("test/child@1.0.0"), None) -def test_a_damaged_build_in_the_store_names_the_build_that_replaces_it(tmp_path, monkeypatch): +def test_a_damaged_build_in_the_local_registry_names_the_build_that_replaces_it(tmp_path, monkeypatch): calls = [] install_backends(monkeypatch, _Builder(calls)) store = LocalRegistry(tmp_path / "store") @@ -202,16 +202,16 @@ def test_the_default_list_follows_the_environment(monkeypatch, tmp_path): monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(tmp_path / "remote")) monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") - store, remote = default_sources() - assert isinstance(store, LocalRegistry) and isinstance(remote, LocalRegistry) - assert (store.root, remote.root) == (tmp_path / "store", tmp_path / "remote") + local_registry, remote = default_sources() + assert isinstance(local_registry, LocalRegistry) and isinstance(remote, LocalRegistry) + assert (local_registry.root, remote.root) == (tmp_path / "store", tmp_path / "remote") # TimeNet() uses the same list. client = TimeNet() assert [_entry(source) for source in client._sources] == [ ("local", tmp_path / "store"), ("local", tmp_path / "remote"), ] - assert client._store is client._sources[0] + assert client._local_registry is client._sources[0] @pytest.mark.usefixtures("home") @@ -654,7 +654,7 @@ def test_download_copies_each_release_from_the_first_registry_that_provides_it(t @pytest.mark.parametrize("case", ["force", "damaged"]) -def test_download_refuses_a_store_copy_with_other_files(registry_root, tmp_path, case): +def test_download_refuses_a_local_registry_copy_with_other_files(registry_root, tmp_path, case): store = LocalRegistry(tmp_path / "store") other = make_dataset() other.records[0].annotate(Annotation(key="build", value="another")) @@ -674,7 +674,7 @@ def test_download_refuses_a_store_copy_with_other_files(registry_root, tmp_path, assert (directory / "manifest.json").read_text() == before -def test_download_names_the_store_when_no_other_registry_provides_its_copy(tmp_path, monkeypatch): +def test_download_names_the_local_registry_when_no_other_registry_provides_its_copy(tmp_path, monkeypatch): calls = [] install_backends(monkeypatch, _Builder(calls)) store = LocalRegistry(tmp_path / "store") @@ -691,7 +691,9 @@ def test_download_names_the_store_when_no_other_registry_provides_its_copy(tmp_p @pytest.mark.parametrize("case", ["force", "damaged"]) -def test_download_counts_a_registry_over_the_directory_of_the_store_as_the_store(tmp_path, monkeypatch, case): +def test_download_counts_a_registry_over_the_directory_of_the_local_registry_as_the_local_registry( + tmp_path, monkeypatch, case +): root = tmp_path / "reg" LocalRegistry(root).put_dataset(make_dataset()) directory = root / "timenet/hello-world/1.0.0" @@ -720,7 +722,7 @@ def test_download_counts_a_registry_over_the_directory_of_the_store_as_the_store assert {path: path.read_bytes() for path in directory.rglob("*") if path.is_file()} == before -def test_download_counts_a_symlink_to_the_store_directory_as_the_store(tmp_path): +def test_download_counts_a_symlink_to_the_local_registry_directory_as_the_local_registry(tmp_path): store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) (tmp_path / "link").symlink_to(store.root, target_is_directory=True) @@ -731,7 +733,7 @@ def test_download_counts_a_symlink_to_the_store_directory_as_the_store(tmp_path) client.download("timenet/hello-world", force=True) -def test_download_completes_the_payload_sources_of_a_complete_store_copy(tmp_path): +def test_download_completes_the_payload_sources_of_a_complete_local_registry_copy(tmp_path): origin, reference = _child_of_the_demo(tmp_path / "origin") store = LocalRegistry(tmp_path / "store") client = TimeNet(sources=[store, origin]) @@ -780,7 +782,7 @@ def test_download_copies_a_payload_source_without_the_payload_sources_of_that_so assert not store.exists(demo.dataset_id, str(demo.version)) -def test_download_repairs_a_damaged_store_copy(tmp_path): +def test_download_repairs_a_damaged_local_registry_copy(tmp_path): origin, reference = _child_of_the_demo(tmp_path / "origin") store = LocalRegistry(tmp_path / "store") client = TimeNet(sources=[store, origin.root]) @@ -797,7 +799,7 @@ def test_download_repairs_a_damaged_store_copy(tmp_path): reader.verify() -def test_download_returns_a_complete_store_copy_at_once(registry_root, tmp_path): +def test_download_returns_a_complete_local_registry_copy_at_once(registry_root, tmp_path): transport, requests = build_service_fake(registry_root) remote = RemoteRegistry("http://api.local", transport=transport) client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), remote]) diff --git a/packages/timenet/tests/test_config.py b/packages/timenet/tests/test_config.py index f5cdabb47..fdb79c966 100644 --- a/packages/timenet/tests/test_config.py +++ b/packages/timenet/tests/test_config.py @@ -13,7 +13,7 @@ def _unset(monkeypatch): @pytest.mark.usefixtures("home") -def test_the_defaults_are_the_store_the_hosted_registry_the_connectors_and_the_cache(monkeypatch): +def test_the_defaults_are_the_local_registry_the_hosted_registry_the_connectors_and_the_cache(monkeypatch): _unset(monkeypatch) cfg = settings() assert cfg.local_registry_dir == Path("~/.timenet/registry").expanduser() diff --git a/packages/timenet/tests/test_producer.py b/packages/timenet/tests/test_producer.py index e8a804b83..c79a3e964 100644 --- a/packages/timenet/tests/test_producer.py +++ b/packages/timenet/tests/test_producer.py @@ -104,7 +104,7 @@ def test_build_without_sources_copies_a_parent_from_the_remote_registry(tmp_path assert [release for release, _ in cards] == [CHILD] -def test_build_lands_in_the_store_that_the_environment_names(tmp_path, monkeypatch, cards): +def test_build_lands_in_the_local_registry_that_the_environment_names(tmp_path, monkeypatch, cards): monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "env-store")) assert timenet.build(PARENT.dataset_id) == tmp_path / "env-store/test/parent/1.0.0" assert LocalRegistry(tmp_path / "env-store").exists(PARENT.dataset_id, "1.0.0") @@ -180,7 +180,7 @@ def store(tmp_path, monkeypatch): return LocalRegistry(tmp_path / "store") -def test_publish_copies_the_latest_release_of_the_store(tmp_path, store): +def test_publish_copies_the_latest_release_of_the_local_registry(tmp_path, store): store.put_dataset(_demo(_card(DatasetRef.parse("timenet/hello-world@0.9.0")))) store.put_dataset(make_dataset()) dest = tmp_path / "dest"