feat(client)!: take an ordered list of sources

examples/load_tsqa.py

@@ -1,14 +1,11 @@
-"""Load a built TimeF dataset and print a summary (no pandas or extras needed).
+"""Print a local dataset's summary and sample signal values.
 
-Build the dataset into an explicit local registry, then run this script::
+Build the dataset into a local registry, then run this script::
 
     uv run timenet-build build "chengsenwang/tsqa" --out .timenet-registry
     uv run python examples/load_tsqa.py
 
-``TimeNet(registry).open(...)`` returns a reader, and its ``read()`` returns a ``TimeFDataset``.
-``describe()`` prints its identity, counts, per-spec columns, and a record preview. Signal values
-stay in storage until you ask for them. Pass a dataset ID as the first argument and a registry path
-as the second argument.
+Pass a dataset ID and a registry path as arguments to load another dataset.
 """
 
 from pathlib import Path
@@ -18,11 +15,11 @@
 
 
 def main() -> None:
-    """Open the dataset, print its describe() summary, and show one raw series slice."""
+    """Print a local dataset's summary and the first five values of its first signal."""
     arguments = sys.argv[1:]
     dataset_id = arguments[0] if arguments else "chengsenwang/tsqa"
     registry = Path(arguments[1]) if len(arguments) > 1 else Path(".timenet-registry")
-    with TimeNet(registry).open(dataset_id) as reader:
+    with TimeNet(sources=[registry]).open(dataset_id) as reader:
         dataset = reader.read()
         dataset.describe()
 

examples/test_mean_classifier.py

@@ -24,7 +24,7 @@
 # Build and load through the same explicit local registry.
 registry = Path(".timenet-registry")
 timenet_connectors.build("timenet/test-mean", out=registry)
-dataset = TimeNet(registry).load("timenet/test-mean")
+dataset = TimeNet(sources=[registry]).load("timenet/test-mean")
 
 # A quick tour of what we loaded: identity, counts, specs, and a record preview.
 dataset.describe()

packages/timenet-connectors/src/timenet_connectors/api.py

@@ -107,18 +107,20 @@ def load(
     fetch_from: "str | BaseRegistry | None" = None,
     build_parents: bool = False,
 ) -> "TimeFDataset":
-    """Build a dataset if needed and load it into memory.
+    """Build a dataset if needed and load its values into memory.
+
+    The build writes to the default local registry.
 
     Args:
         dataset_id: The dataset ID (``org/name``).
-        version: The version to load, which the card must declare, or ``None`` for the latest.
-        fetch_from: The registry or URI that supplies missing parents and payload sources.
-        build_parents: Build missing parent releases with installed connectors.
+        version: The version declared in the connector's card, or ``None`` for the latest.
+        fetch_from: A registry or URI to copy missing dependencies from.
+        build_parents: Build missing parent releases from their installed connectors.
 
     Returns:
-        The loaded dataset.
+        The dataset with all stored values in memory.
     """
     from timenet.client import TimeNet  # ruff: ignore[import-outside-top-level]
 
     build(dataset_id, version=version, fetch_from=fetch_from, build_parents=build_parents)
-    return TimeNet().load(dataset_id, version, fetch_from=fetch_from)
+    return TimeNet().load(dataset_id, version)

packages/timenet-connectors/src/timenet_connectors/builder/cli.py

@@ -30,7 +30,6 @@
     copy_release,
     local_registry_path,
     open_registry,
-    open_writable_registry,
 )
 from timenet.registry._paths import release_dir  # ruff: ignore[import-private-name]
 from timenet.sources import InstalledConnectors
@@ -49,26 +48,23 @@ def _root(quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress stat
 
 
 def _resolve_target(out: str | None) -> WritableRegistry | Path:
-    """Resolve a build's output target: a writable registry for a remote URL, else a local directory.
+    """Resolve the registry or local directory for build output.
 
-    The selection order matches the consumer side, so the CLI that writes a dataset and the SDK that
-    reads it agree on where it lands: the ``--out`` argument, then ``$TIMENET_REGISTRY``, then the
-    default local registry.
+    It tries ``out``, then ``$TIMENET_REGISTRY``, then the default local registry.
 
     Args:
-        out: The ``--out`` value, or ``None`` to fall back to ``$TIMENET_REGISTRY`` then the default.
+        out: The output URL or path, or ``None`` to use the configured default.
 
     Returns:
-        A :class:`~timenet.registry.WritableRegistry` for a ``timenet://`` / ``http(s)://`` / ``s3://``
-        target, or the local directory ``Path`` for a plain path or ``file://`` URI.
+        A registry for a remote URL, or a ``Path`` for a local directory.
     """
     target = out if out is not None else settings().registry
     if target is None:
         return settings().registry_path
     try:
         return local_registry_path(target)
     except TimeNetRegistryError:
-        return open_writable_registry(target)
+        return open_registry(target)
 
 
 @app.command()

packages/timenet-connectors/tests/test_build_plan.py

@@ -31,7 +31,6 @@
 from timenet.types.references import DatasetRef
 from timenet.values_backends import ValuesBackend
 import timenet_connectors
-from timenet_connectors import api as api_module
 from timenet_connectors.builder import backend as backend_module, cli as cli_module, env as env_module
 from timenet_connectors.builder.backend import ConnectorBuilder
 from timenet_connectors.builder.cli import app
@@ -567,23 +566,20 @@ def test_load_obtains_a_missing_parent_before_it_builds_and_reads(tmp_path, monk
     assert calls == (builds if obtain == "fetch_from" else [(PARENT.dataset_id, False), *builds])
 
 
-def test_load_reads_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch):
+def test_load_copies_a_missing_payload_source_through_fetch_from(tmp_path, monkeypatch):
+    _install_cards(monkeypatch, parent={"parent": GRAND})
     out = LocalRegistry(tmp_path / "out")
     _parent_without_its_source(out)
     origin = LocalRegistry(tmp_path / "origin")
     origin.put_dataset(make_dataset())
     monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home"))
     monkeypatch.setenv("TIMENET_REGISTRY", str(out.root))
-    # The build has nothing to do, and no builder backend can build the payload source, so only the
-    # read can get it.
-    monkeypatch.setattr(api_module, "build", lambda dataset_id, **options: None)
-    monkeypatch.setattr(builders, "entry_points", lambda group: [])
 
     dataset = timenet_connectors.load(PARENT.dataset_id, fetch_from=origin)
 
     assert len(dataset.records) == 2
-    # The read opens the payload source in origin, where it is, and copies nothing.
-    assert not out.exists(GRAND.dataset_id, str(GRAND.version))
+    # The build copies the payload source into the store, where the read finds it.
+    assert out.exists(GRAND.dataset_id, str(GRAND.version))
 
 
 def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch):

packages/timenet-connectors/tests/test_end_to_end.py

@@ -9,7 +9,8 @@
 from timenet.engine import run_pipeline
 from timenet.errors import TimeFValidationError
 from timenet.provenance import build_env
-from timenet.registry import RemoteRegistry
+from timenet.registry import LocalRegistry, RemoteRegistry
+from timenet.sources import InstalledConnectors
 from timenet.testing import assert_datasets_equal
 from timenet_connectors import build, load
 from timenet_connectors.builder.cli import _resolve_target, app as build_app
@@ -53,7 +54,7 @@ def test_build_then_load_round_trips(converted, tmp_path):
     assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists()
 
     # CONSUME: load it back through the SDK and compare to the dataset the build wrote.
-    restored = TimeNet(registry, storage_path=tmp_path / "store").load("timenet/hello-world")
+    restored = TimeNet(sources=[str(registry)]).load("timenet/hello-world")
     [original] = converted
     assert_datasets_equal(original, restored)
 
@@ -97,7 +98,7 @@ def test_build_honors_timenet_registry(clean_env, monkeypatch, tmp_path):
     assert runner.invoke(build_app, ["build", "timenet/hello-world", "--no-isolation"]).exit_code == 0
     assert (registry / "timenet" / "hello-world" / "1.0.0" / "manifest.json").exists()
     # The SDK resolves $TIMENET_REGISTRY the same way, so it reads back what the build just wrote.
-    assert TimeNet(storage_path=tmp_path / "store").list()[0].dataset_id == "timenet/hello-world"
+    assert TimeNet().list()[0].dataset_id == "timenet/hello-world"
 
 
 def test_build_out_overrides_timenet_registry(clean_env, monkeypatch, tmp_path):
@@ -167,7 +168,7 @@ def test_build_cleans_cache_but_keep_flag_retains(tmp_path, monkeypatch):
 @pytest.mark.slow
 def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monkeypatch):
     monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home"))
-    client = TimeNet(tmp_path / "registry")
+    client = TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()])
 
     dataset = client.load("timenet/hello-world")
 
@@ -178,7 +179,7 @@ def test_load_builds_a_missing_dataset_in_an_isolated_environment(tmp_path, monk
 @pytest.mark.slow
 def test_an_isolated_build_records_its_environment(tmp_path, monkeypatch):
     monkeypatch.setenv("TIMENET_HOME", str(tmp_path / "home"))
-    TimeNet(tmp_path / "registry").load("timenet/hello-world")
+    TimeNet(sources=[LocalRegistry(tmp_path / "registry"), InstalledConnectors()]).load("timenet/hello-world")
 
     manifest = json.loads((tmp_path / "registry/timenet/hello-world/1.0.0/manifest.json").read_text())
     packages = manifest["build_env"]["packages"]

packages/timenet/src/timenet/cli/app.py

@@ -13,7 +13,7 @@
 from timenet.cache import CachedDataset, cached_datasets, clear_cache, human_bytes, raw_cache_size
 from timenet.cli.runner import run_cli
 from timenet.cli.ui import console as ui
-from timenet.client import TimeNet
+from timenet.client import TimeNet, default_sources
 from timenet.types import DatasetMetadata, Domain, License, TaskType
 from timenet.types.access import access_requirements, effective_access
 from timenet.types.tasks import TASKS
@@ -22,7 +22,12 @@
 app = typer.Typer(help="Browse a TimeNet registry and fetch datasets.", no_args_is_help=True)
 console = Console()
 
-_registry_option = typer.Option(None, "--registry", "-r", help="Registry URL or path (else $TIMENET_REGISTRY).")
+_registry_option = typer.Option(
+    None,
+    "--registry",
+    "-r",
+    help="Registry URL or path that this command reads (else $TIMENET_REGISTRY, then timenet://).",
+)
 
 T = TypeVar("T")
 
@@ -55,6 +60,20 @@ def _enum_list(values: list[str], factory: Callable[[str], T], flag: str, choice
     return parsed or None
 
 
+def _client(registry: str | None) -> TimeNet:
+    """Create a client with the default sources and an optional registry.
+
+    Args:
+        registry: The registry URL or path to read after the local registry, or ``None`` for the default.
+
+    Returns:
+        A client with the configured sources.
+    """
+    # TODO(adrian): Remove temporary glue which maps --registry onto default_sources() until the
+    # CLI takes the flags of the defaults.
+    return TimeNet(sources=default_sources(remote_registry=registry))
+
+
 def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table:
     """Build a table of dataset metadata (id, name, license, domains).
 
@@ -82,8 +101,8 @@ def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table:
 
 @app.command("list")
 def list_datasets(registry: str | None = _registry_option) -> None:
-    """List every dataset in the registry."""
-    rows = TimeNet(registry).list()
+    """Display metadata for every dataset in the selected registry."""
+    rows = _client(registry).list()
     if not rows:
         typer.echo("No datasets in the registry.")
         return
@@ -102,8 +121,11 @@ def search(  # ruff: ignore[too-many-arguments, too-many-positional-arguments]
     tag: list[str] = typer.Option([], "--tag", help="Tag(s)."),
     limit: int = typer.Option(100, "--limit", min=0, help="Maximum results."),
 ) -> None:
-    """Search datasets by filter. Repeat a flag to pass several values."""
-    results = TimeNet(registry).search(
+    """Find datasets that match the filters.
+
+    Repeat a filter flag to pass several values.
+    """
+    results = _client(registry).search(
         query=query or None,
         domain=_enum_list(domain, Domain, "--domain", [d.value for d in Domain]),
         task=_enum_list(task, lambda value: TASKS[TaskType(value)], "--task", [t.value for t in TaskType]),
@@ -120,9 +142,9 @@ def search(  # ruff: ignore[too-many-arguments, too-many-positional-arguments]
 
 
 @app.command()
-def info(dataset_id: str, version: str | None = None, registry: str | None = _registry_option) -> None:
-    """Print a dataset's manifest summary: metadata, schema, and counts."""
-    manifest = TimeNet(registry).get(dataset_id, version)
+def info(dataset_id: str, registry: str | None = _registry_option) -> None:
+    """Display a dataset's metadata, schema, and counts."""
+    manifest = _client(registry).get(dataset_id)
     counts = manifest.counts
     table = Table(title=manifest.dataset_id, title_justify="left", title_style="bold", show_header=False)
     table.add_column("Field", style="cyan")
@@ -162,37 +184,34 @@ def info(dataset_id: str, version: str | None = None, registry: str | None = _re
 
 
 @app.command()
-def download(
-    dataset_id: str,
-    version: str | None = None,
-    registry: str | None = _registry_option,
-    storage: str | None = typer.Option(None, "--storage", help="Local storage dir (else $TIMENET_STORAGE)."),
-) -> None:
-    """Fetch a complete TimeF version to local storage. Prints its directory to stdout."""
-    client = TimeNet(registry, storage_path=storage)
-    manifest = client.get(dataset_id, version)
+def download(dataset_id: str, registry: str | None = _registry_option) -> None:
+    """Download a dataset version and print its local directory to stdout.
+
+    Files go to ``$TIMENET_STORAGE`` or ``<TIMENET_HOME>/storage``.
+    """
+    client = _client(registry)
+    manifest = client.get(dataset_id)
     files = list(manifest.files.all_files())
     total = sum(part.size for part in files)
     ui.status("⬇️", f"Downloading '{dataset_id}' ({len(files)} files, {human_bytes(total)})…")
-    target = _download(client, dataset_id, version, total)
+    target = _download(client, dataset_id, total)
     ui.success(f"Downloaded '{dataset_id}' ({target.name})")
     typer.echo(str(target))
 
 
-def _download(client: TimeNet, dataset_id: str, version: str | None, total: int) -> Path:
+def _download(client: TimeNet, dataset_id: str, total: int) -> Path:
     """Download a version, rendering a live progress bar on stderr when the terminal supports it.
 
     Args:
         client: The client to download through.
-        dataset_id: The dataset id.
-        version: The version string, or ``None`` for the latest.
-        total: The total byte size of the version's files, for the bar's scale.
+        dataset_id: The dataset id, optionally followed by ``@version``.
+        total: The total file size in bytes for the progress bar.
 
     Returns:
         The local version directory.
     """
     if ui.quiet or not sys.stderr.isatty():
-        return client.download(dataset_id, version)
+        return client.download(dataset_id)
     progress = Progress(
         TextColumn("[bold blue]{task.description}"),
         BarColumn(bar_width=None),
@@ -207,7 +226,7 @@ def _download(client: TimeNet, dataset_id: str, version: str | None, total: int)
     )
     with progress:
         task = progress.add_task("downloading", total=total)
-        return client.download(dataset_id, version, progress_cb=lambda n: progress.advance(task, n))
+        return client.download(dataset_id, progress_cb=lambda n: progress.advance(task, n))
 
 
 cache_app = typer.Typer(help="Inspect and clear the local cache.", no_args_is_help=True)