feat(cache)!: record where each release in the local registry comes from

packages/timenet-connectors/src/timenet_connectors/builder/cli.py

@@ -113,7 +113,7 @@ def build(
     sources = default_sources(local_registry=local_registry, remote_registry=remote_registry, auto_build=auto_build)
     # The default list starts with the local registry, which the build writes into. The planner leaves it out of
     # the sources of the copies and builds.
-    store = cast("LocalRegistry", sources[0])
+    local = cast("LocalRegistry", sources[0])
     dest = None if publish_to is None else open_registry(publish_to, label="--publish-to")
     has_connectors = any(isinstance(source, InstalledConnectors) for source in sources)
     try:
@@ -126,7 +126,7 @@ def build(
         # plan refuses another release before any download.
         plan = plan_release_build(
             str(wanted),
-            store=store,
+            target_registry=local,
             parents_from=sources,
             force=force,
             open_access_only=dest is not None and dest.requires_open_access,
@@ -136,7 +136,7 @@ def build(
             if dest is not None:
                 lines.append(f"then publish {wanted} to {dest}")
             elif not plan.steps:
-                console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {store}")
+                console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {local}")
             if lines:
                 typer.echo("\n".join(lines))
             return
@@ -149,9 +149,9 @@ def build(
                 quiet=console.quiet,
                 isolation=None if isolation is None else ("on" if isolation else "off"),
             )
-            run_plan(plan, store, options=options)
+            run_plan(plan, local, options=options)
             if dest is not None:
-                publish(wanted, dest=dest, source=store, progress=progress)
+                publish(wanted, dest=dest, source=local, progress=progress)
     except TimeNetMissingParentError as error:
         error.hint = _missing_hint(has_connectors=has_connectors)
         raise
@@ -167,9 +167,9 @@ def build(
         error.hint = _access_hint(dest, has_connectors=has_connectors)
         raise
     if dest is None and not plan.steps:
-        console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {store}")
+        console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {local}")
     # A publish prints the version, and a local build prints the release directory.
-    typer.echo(str(wanted.version) if dest is not None else str(release_dir(store.root, wanted)))
+    typer.echo(str(wanted.version) if dest is not None else str(release_dir(local.root, wanted)))
 
 
 def _missing_hint(*, has_connectors: bool) -> str:

packages/timenet-connectors/tests/test_build_env.py

@@ -165,7 +165,7 @@ def _missing():
         uv_command(_spec(), ["timenet-build"])
 
 
-def test_run_isolated_build_runs_build_release_with_the_store_and_the_pins(monkeypatch, tmp_path):
+def test_run_isolated_build_runs_build_release_with_the_target_registry_and_the_pins(monkeypatch, tmp_path):
     release = DatasetRef.parse("timenet/hello-world@1.0.0")
     pinned = PayloadSource(release=release, files=(FilePart(path="a.bin", checksum="sha256:" + "0" * 64, size=1),))
     captured = {}

packages/timenet/src/timenet/builders.py

@@ -11,7 +11,7 @@
 from importlib.metadata import EntryPoint, entry_points
 from typing import TYPE_CHECKING, Literal, Protocol
 
-from timenet.errors import TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError
+from timenet.errors import TimeNetBuildError, TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError
 from timenet.progress import ProgressCallback
 from timenet.types import DatasetMetadata, Version
 from timenet.types.references import DatasetRef
@@ -112,10 +112,12 @@ class InstalledBuilder:
     Attributes:
         backend: The installed backend that builds the dataset.
         card: The dataset's connector card.
+        package: ``"<distribution> <version>"`` of the package that registers the backend.
     """
 
     backend: BuilderBackend
     card: DatasetMetadata
+    package: str
 
     @property
     def release(self) -> DatasetRef:
@@ -166,12 +168,13 @@ def installed_builder(dataset_id: str) -> InstalledBuilder | None:
 
     Raises:
         TimeNetInvalidCardError: If the dataset's card cannot be read.
-    """  # ruff: ignore[docstring-extraneous-exception] - raised by the backend
+        TimeNetBuildError: If no installed distribution declares the entry point of the backend.
+    """  # ruff: ignore[docstring-extraneous-exception] - raised by the backend and _package
     claim = next((claim for claim in claims() if claim.dataset_id == dataset_id), None)
     if claim is None:
         return None
     card = claim.backend.card(dataset_id)
-    return None if card is None else InstalledBuilder(claim.backend, card)
+    return None if card is None else InstalledBuilder(claim.backend, card, _package(claim))
 
 
 def declared(dataset_id: str, version: Version | str | None = None) -> InstalledBuilder:
@@ -213,3 +216,16 @@ def find_card(dataset_id: str) -> DatasetMetadata | None:
     """  # ruff: ignore[docstring-extraneous-exception] - raised by the backend
     found = installed_builder(dataset_id)
     return None if found is None else found.card
+
+
+def _package(claim: ConnectorClaim) -> str:
+    """Return ``"<distribution> <version>"`` of the package whose entry point registers the backend of a claim.
+
+    Raises:
+        TimeNetBuildError: If no installed distribution declares the entry point.
+    """
+    dist = claim.entry.dist
+    if dist is None:
+        kind = type(claim.backend)
+        raise TimeNetBuildError(f"the entry point of {kind.__module__}.{kind.__qualname__} names no installed package")
+    return f"{dist.name} {dist.version}"

packages/timenet/src/timenet/cache.py

@@ -1,9 +1,9 @@
-"""Inspect and clear the cache and the stored releases.
+"""Inspect and clear the cache and the downloads and builds in the local registry.
 
 Cache removal requires TimeNet's ``CACHEDIR.TAG`` marker.
 """
 
-from collections.abc import Iterable
+from collections.abc import Collection, Iterable
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -12,16 +12,51 @@
 from timenet.errors import TimeFValidationError
 from timenet.registry import LocalRegistry
 from timenet.registry._paths import release_dir
+from timenet.registry.origin import Origin, OriginKind
 from timenet.types.references import DatasetRef
 
 
 @dataclass(frozen=True)
 class StoredRelease:
-    """A stored release with its path and size in bytes."""
+    """A stored release with its path, provenance record, and size on disk."""
 
     release: DatasetRef
     path: Path
-    size_bytes: int
+    origin: Origin | None
+    """The record of how the release reached the local registry, or ``None`` if it has no record."""
+
+    def size(self) -> int:
+        """Measure the size of the release on disk.
+
+        Returns:
+            The size in bytes, measured on each call.
+        """
+        return _dir_size(self.path)
+
+    @property
+    def kind(self) -> OriginKind:
+        """The kind of the record, or :attr:`~OriginKind.BUILD` for a release without a record."""
+        return OriginKind.BUILD if self.origin is None else self.origin.kind
+
+
+@dataclass(frozen=True)
+class KeptRelease:
+    """A release that a clear keeps, because a release that stays needs it."""
+
+    release: DatasetRef
+    """The release that stays."""
+    needed_by: DatasetRef
+    """A release that stays and needs it, as a payload source or as the parent of a build."""
+
+
+@dataclass(frozen=True)
+class ReleasesToClear:
+    """The releases that a clear deletes, and the releases of the same kinds that it keeps."""
+
+    releases: list[StoredRelease]
+    """The releases to delete, sorted by dataset ID and version."""
+    kept: list[KeptRelease]
+    """The releases of the cleared kinds that stay, sorted by dataset ID and version."""
 
 
 @dataclass(frozen=True)
@@ -38,12 +73,14 @@ class ClearedCache:
 
 @dataclass(frozen=True)
 class ClearedReleases:
-    """The bytes freed and release folders removed by a release clear."""
+    """The bytes freed, release folders removed and releases kept by a release clear."""
 
     freed: int
     """The number of bytes freed."""
     removed: list[Path]
     """The release folders successfully removed."""
+    kept: list[KeptRelease]
+    """The releases of the cleared kinds that stay, because a release that stays needs them."""
 
 
 def human_bytes(n: int) -> str:
@@ -65,19 +102,79 @@ def human_bytes(n: int) -> str:
 
 
 def stored_releases(store: LocalRegistry) -> list[StoredRelease]:
-    """List the releases of a local registry and their sizes in bytes.
+    """List stored releases with their provenance records and sizes.
 
     Args:
-        store: The local registry to inspect.
+        store: The local registry.
 
     Returns:
         The releases, sorted by dataset ID and version.
-    """
-    found: list[StoredRelease] = []
-    for release in store.releases():
-        path = release_dir(store.root, release)
-        found.append(StoredRelease(release, path, _dir_size(path)))
-    return found
+
+    Raises:
+        TimeFFormatError: If a provenance record is invalid.
+    """  # ruff: ignore[docstring-extraneous-exception] (raised by read_origin)
+    return [
+        StoredRelease(release, release_dir(store.root, release), store.read_origin(release))
+        for release in store.releases()
+    ]
+
+
+def stored_releases_by_kind(store: LocalRegistry) -> dict[OriginKind, list[StoredRelease]]:
+    """List stored releases with their provenance records and sizes, by kind.
+
+    Args:
+        store: The local registry.
+
+    Returns:
+        The releases of each kind, sorted by dataset ID and version, and an empty list for a kind without releases.
+
+    Raises:
+        TimeFFormatError: If a provenance record is invalid.
+    """  # ruff: ignore[docstring-extraneous-exception] (raised by read_origin)
+    by_kind: dict[OriginKind, list[StoredRelease]] = {kind: [] for kind in OriginKind}
+    for item in stored_releases(store):
+        by_kind[item.kind].append(item)
+    return by_kind
+
+
+def releases_to_clear(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> ReleasesToClear:
+    """Find the releases of the given kinds that no release that stays needs.
+
+    A release that stays keeps its payload sources, and a build that stays also keeps its parents. This holds
+    for each release that stays, also for one that stays only because another one needs it.
+
+    Args:
+        store: The local registry.
+        kinds: The kinds of release to delete.
+
+    Returns:
+        The releases to delete, and the releases of those kinds that stay with a release that needs each.
+
+    Raises:
+        TimeFValidationError: If ``kinds`` is empty, or a dataset or release folder in the local registry is a symlink.
+        TimeFFormatError: If a provenance record or the manifest of a release that stays is damaged.
+        OSError: If a folder of the local registry cannot be read.
+    """  # ruff: ignore[docstring-extraneous-exception] (raised by check_no_linked_folders and stored_releases)
+    _check_kinds(kinds)
+    store.check_no_linked_folders()
+    releases = stored_releases(store)
+    held = {item.release: item for item in releases}
+    # Walk from each release that stays to the releases that it needs. If every kind goes, nothing stays.
+    staying = [item.release for item in releases if item.kind not in kinds]
+    seen = set(staying)
+    needed_by: dict[DatasetRef, DatasetRef] = {}
+    while staying:
+        release = staying.pop()
+        manifest = store.get_manifest(release.dataset_id, str(release.version))
+        for need in manifest.required_releases(with_parents=held[release].kind is OriginKind.BUILD):
+            if need in held and need not in seen:
+                seen.add(need)
+                needed_by[need] = release
+                staying.append(need)
+    return ReleasesToClear(
+        [item for item in releases if item.kind in kinds and item.release not in needed_by],
+        [KeptRelease(item.release, needed_by[item.release]) for item in releases if item.release in needed_by],
+    )
 
 
 def cache_size(cache_dir: Path) -> int:
@@ -135,27 +232,43 @@ def clear_cache(cache_dir: Path, *, local_registry: LocalRegistry) -> ClearedCac
     return ClearedCache(freed, removed, [folder for folder in folders if folder not in marked])
 
 
-def clear_releases(store: LocalRegistry) -> ClearedReleases:
-    """Delete stored releases and the dataset directories left empty.
+def clear_releases(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> ClearedReleases:
+    """Delete selected releases unless remaining releases need them.
 
-    Other files in the local registry stay.
+    Each dataset folder that this leaves empty goes too, and other files in the local registry stay.
 
     Args:
         store: The local registry.
+        kinds: The kinds of release to delete.
 
     Returns:
-        The bytes freed and the release folders removed.
+        The bytes freed, the release folders removed, and the releases kept with a release that needs each.
 
     Raises:
-        TimeNetRegistryError: If the local registry is read-only.
-        OSError: If a folder cannot be removed completely.
-    """  # ruff: ignore[docstring-extraneous-exception] (raised by remove_release)
+        TimeNetRegistryError: Before any deletion, if the local registry is read-only.
+        TimeFValidationError: Before any deletion, if ``kinds`` is empty or a dataset or release folder is a symlink.
+        TimeFFormatError: Before any deletion, if a provenance record or a kept release's manifest is damaged.
+        OSError: If a folder cannot be read or removed completely.
+    """  # ruff: ignore[docstring-extraneous-exception] (raised by releases_to_clear and remove_release)
+    _check_kinds(kinds)
+    store.check_writable("a clear of its releases")
+    found = releases_to_clear(store, kinds=kinds)
     freed = 0
     removed: list[Path] = []
-    for release in store.releases():
-        freed += store.remove_release(release)
-        removed.append(release_dir(store.root, release))
-    return ClearedReleases(freed, removed)
+    for item in found.releases:
+        freed += store.remove_release(item.release)
+        removed.append(item.path)
+    return ClearedReleases(freed, removed, found.kept)
+
+
+def _check_kinds(kinds: Collection[OriginKind]) -> None:
+    """Refuse a clear that names no kind of release.
+
+    Raises:
+        TimeFValidationError: If ``kinds`` is empty.
+    """
+    if not kinds:
+        raise TimeFValidationError("kinds names no kind of release")
 
 
 def _check_cache_deletion(cache_dir: Path, folders: Iterable[Path], *, local_registry: Path) -> None: