feat(client)!: download releases into the local registry

packages/timenet/src/timenet/_fs.py

@@ -1,4 +1,4 @@
-"""Replace a directory with a complete staging directory."""
+"""Compare directory paths and replace directories from staging."""
 
 from collections.abc import Generator
 from contextlib import contextmanager
@@ -7,6 +7,43 @@
 import uuid
 
 
+def same_directory(a: Path, b: Path) -> bool:
+    """Check whether two paths name the same directory, as the file system sees them.
+
+    Existing paths match when they identify the same directory on disk, and never match a missing path.
+    Missing paths match when their nearest existing parents match and the names below them are equal as written.
+
+    Args:
+        a: The first path, whether or not it exists.
+        b: The second path, whether or not it exists.
+
+    Returns:
+        ``True`` if the paths match.
+    """
+    if a.exists() or b.exists():
+        # On a volume that ignores case, a case variant of an existing path exists too.
+        return a.exists() and b.exists() and a.samefile(b)
+    return _below_existing_parent(a.resolve()) == _below_existing_parent(b.resolve())
+
+
+def _below_existing_parent(path: Path) -> tuple[int, int, tuple[str, ...]]:
+    """Identify a missing path by its nearest existing parent and the names below that parent.
+
+    Args:
+        path: A missing absolute path without symlinks.
+
+    Returns:
+        The parent's device and inode numbers, followed by the names below it as written.
+    """
+    for depth, parent in enumerate(path.parents, start=1):
+        try:
+            info = parent.stat()
+        except OSError:
+            continue
+        return info.st_dev, info.st_ino, path.parts[len(path.parts) - depth :]
+    return 0, 0, path.parts  # nothing exists, not even the root
+
+
 def publish_directory(staging: Path, final: Path) -> None:
     """Rename ``staging`` to ``final``, replacing any existing directory.
 

packages/timenet/src/timenet/cache.py

@@ -4,6 +4,11 @@
 download cache. All three sit under the home directory.
 """
 
+# TODO(adrian): Remove temporary glue which maps the "download storage" of this module onto the
+# storage directory, settings().storage_dir, until a later change replaces this module and its
+# view of the storage directory. download() copies into the local registry now, so the storage directory
+# holds only the copies that remote registries cache.
+
 from dataclasses import dataclass
 import os
 from pathlib import Path
@@ -43,12 +48,14 @@ def human_bytes(n: int) -> str:
 
 
 def cached_datasets() -> list[CachedDataset]:
-    """List every dataset version in the local registry and download storage, with sizes.
+    """List dataset versions and their sizes in the local registry and storage directory.
 
     Returns:
-        The cached datasets, sorted by (location, id, version).
+        The cached versions sorted by location, dataset ID, and version.
     """
     config = settings()
+    # TODO(adrian): Remove temporary glue which maps the download storage onto the storage
+    # directory until a later change replaces this module, as the note at the top says.
     found = _scan(config.registry_path, "registry") + _scan(config.storage_dir, "storage")
     return sorted(found, key=lambda dataset: (dataset.location, dataset.dataset_id, dataset.version))
 
@@ -64,20 +71,19 @@ def raw_cache_size() -> int:
 
 
 def clear_cache(*, include_registry: bool) -> tuple[int, list[Path]]:
-    """Delete cached data and return the bytes freed and the directories removed.
+    """Delete the storage directory and raw download cache.
 
-    This always clears the download storage and the raw download cache. The caller can fetch both again.
-    It removes the local registry (locally built datasets) only when the caller sets ``include_registry``.
+    Read or removal failures raise ``OSError``.
 
     Args:
-        include_registry: Also remove the local registry.
+        include_registry: Remove the local registry too.
 
     Returns:
-        A ``(bytes_freed, removed_dirs)`` pair. Every directory in ``removed_dirs`` is gone when this
-        returns. This raises an ``OSError`` if it cannot fully remove a tree. This stops the caller from
-        reporting space it never freed.
+        A ``(bytes_freed, removed_dirs)`` pair with the removed directory paths.
     """
     config = settings()
+    # TODO(adrian): Remove temporary glue which maps the download storage onto the storage
+    # directory until a later change replaces this module, as the note at the top says.
     targets = [config.storage_dir, config.cache_dir]
     if include_registry:
         targets.append(config.registry_path)

packages/timenet/src/timenet/cli/app.py

@@ -1,21 +1,21 @@
 """The consumer command-line interface, the command-line mirror of the :class:`~timenet.client.TimeNet` SDK."""
 
 from collections.abc import Callable, Iterable
-from pathlib import Path
-import sys
+from functools import partial
 from typing import TypeVar
 
 from rich.console import Console
-from rich.progress import BarColumn, DownloadColumn, Progress, TextColumn, TimeRemainingColumn, TransferSpeedColumn
 from rich.table import Table
 import typer
 
 from timenet.cache import CachedDataset, cached_datasets, clear_cache, human_bytes, raw_cache_size
 from timenet.cli.runner import run_cli
-from timenet.cli.ui import console as ui
+from timenet.cli.ui import console as ui, progress_renderer
 from timenet.client import TimeNet, default_sources
+from timenet.progress import ProgressCallback, ProgressEvent, StepStarted
 from timenet.types import DatasetMetadata, Domain, License, TaskType
 from timenet.types.access import access_requirements, effective_access
+from timenet.types.references import split_ref
 from timenet.types.tasks import TASKS
 
 
@@ -28,6 +28,15 @@
     "-r",
     help="Registry URL or path that this command reads (else $TIMENET_REGISTRY, then timenet://).",
 )
+# TODO(adrian): Remove temporary glue which maps --registry onto the list after the local registry, until the CLI
+# takes the flags of the defaults.
+_download_registry_option = typer.Option(
+    None,
+    "--registry",
+    "-r",
+    help="Registry URL or path to put after the local registry in the list of sources. The release resolves "
+    "through the whole list.",
+)
 
 T = TypeVar("T")
 
@@ -184,49 +193,41 @@ def info(dataset_id: str, registry: str | None = _registry_option) -> None:
 
 
 @app.command()
-def download(dataset_id: str, registry: str | None = _registry_option) -> None:
-    """Download a dataset version and print its local directory to stdout.
+def download(
+    dataset_id: str,
+    registry: str | None = _download_registry_option,
+    force: bool = typer.Option(
+        False, "--force", help="Copy the release again, even if the local registry holds a complete copy."
+    ),
+) -> None:
+    """Copy a release and its payload sources into the local registry.
 
-    Files go to ``$TIMENET_STORAGE`` or ``<TIMENET_HOME>/storage``.
+    Print the release directory to standard output.
     """
-    client = _client(registry)
-    manifest = client.get(dataset_id)
-    files = list(manifest.files.all_files())
-    total = sum(part.size for part in files)
-    ui.status("⬇️", f"Downloading '{dataset_id}' ({len(files)} files, {human_bytes(total)})…")
-    target = _download(client, dataset_id, total)
-    ui.success(f"Downloaded '{dataset_id}' ({target.name})")
+    sources = default_sources(remote_registry=registry)
+    started: list[StepStarted] = []
+    with progress_renderer() as show:
+        target = TimeNet(sources=sources).download(
+            dataset_id, force=force, progress=partial(_keep_steps, started, show)
+        )
+    if not started:
+        # The local registry already held the release and its payload sources.
+        ui.success(f"Nothing to do: '{split_ref(dataset_id)[0]}' {target.name} is already in {sources[0]}")
     typer.echo(str(target))
 
 
-def _download(client: TimeNet, dataset_id: str, total: int) -> Path:
-    """Download a version, rendering a live progress bar on stderr when the terminal supports it.
+def _keep_steps(started: list[StepStarted], show: ProgressCallback | None, event: ProgressEvent) -> None:
+    """Keep each step event in ``started``, and pass every event on to ``show``.
 
     Args:
-        client: The client to download through.
-        dataset_id: The dataset id, optionally followed by ``@version``.
-        total: The total file size in bytes for the progress bar.
-
-    Returns:
-        The local version directory.
+        started: Receives the step events.
+        show: The callback that shows the events, or ``None`` for a quiet console.
+        event: The progress event.
     """
-    if ui.quiet or not sys.stderr.isatty():
-        return client.download(dataset_id)
-    progress = Progress(
-        TextColumn("[bold blue]{task.description}"),
-        BarColumn(bar_width=None),
-        "[progress.percentage]{task.percentage:>3.0f}%",
-        "•",
-        DownloadColumn(),
-        "•",
-        TransferSpeedColumn(),
-        "•",
-        TimeRemainingColumn(),
-        console=ui.rich,
-    )
-    with progress:
-        task = progress.add_task("downloading", total=total)
-        return client.download(dataset_id, progress_cb=lambda n: progress.advance(task, n))
+    if isinstance(event, StepStarted):
+        started.append(event)
+    if show is not None:
+        show(event)
 
 
 cache_app = typer.Typer(help="Inspect and clear the local cache.", no_args_is_help=True)
@@ -270,14 +271,17 @@ def _cache_table(datasets: list[CachedDataset]) -> Table:
 
 @cache_app.command("clear")
 def cache_clear(
-    all_: bool = typer.Option(False, "--all", help="Also remove the local registry (built datasets)."),
+    all_: bool = typer.Option(False, "--all", help="Also remove the default local registry (builds and downloads)."),
     yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt."),
 ) -> None:
-    """Delete cached data. Clears downloads by default. The ``--all`` flag also removes built datasets."""
+    """Delete the storage directory and raw cache.
+
+    Use ``--all`` to also remove the default local registry.
+    """
     if not yes:
-        what = "downloaded datasets and the raw cache"
+        what = "the storage directory and the raw cache"
         if all_:
-            what += " and the local registry (built datasets)"
+            what += " and the local registry (builds and downloads)"
         typer.confirm(f"Remove {what}?", abort=True)
     freed, removed = clear_cache(include_registry=all_)
     typer.echo(f"Freed {human_bytes(freed)} from {len(removed)} location(s).")

packages/timenet/src/timenet/cli/ui.py

@@ -125,7 +125,7 @@ def _download_bars() -> Progress:
 
 
 def _bar_reporter(progress: Progress) -> Callable[[BytesMoved], None]:
-    """Create a callback that displays a progress bar for each transfer URL.
+    """Create a callback that displays a progress bar for each transfer.
 
     Args:
         progress: The display that holds the bars.
@@ -136,10 +136,10 @@ def _bar_reporter(progress: Progress) -> Callable[[BytesMoved], None]:
     tasks: dict[str, TaskID] = {}
 
     def report(event: BytesMoved) -> None:
-        task_id = tasks.get(event.url)
+        task_id = tasks.get(event.subject)
         if task_id is None:
-            task_id = progress.add_task("download", name=event.url.rsplit("/", 1)[-1], total=event.total)
-            tasks[event.url] = task_id
+            task_id = progress.add_task("download", name=event.subject.rsplit("/", 1)[-1], total=event.total)
+            tasks[event.subject] = task_id
         progress.update(task_id, completed=event.done, total=event.total)
 
     return report
@@ -161,10 +161,10 @@ def _text_download_reporter() -> Callable[[BytesMoved], None]:
     def report(event: BytesMoved) -> None:
         # ~20% steps when the size is known, else one line every 50 MB.
         step = event.done * 5 // event.total if event.total else event.done // _UNKNOWN_TOTAL_STEP_BYTES
-        if last.get(event.url) == step:
+        if last.get(event.subject) == step:
             return
-        last[event.url] = step
-        name = event.url.rsplit("/", 1)[-1]
+        last[event.subject] = step
+        name = event.subject.rsplit("/", 1)[-1]
         if event.total:
             console.status("⬇️", f"{name} {event.done * 100 // event.total}% ({human_bytes(event.total)})")
         else: