feat(client)!: list and search every source

packages/timenet/src/timenet/_warnings.py

@@ -0,0 +1,24 @@
+"""Warn at the first caller outside the timenet package."""
+
+import inspect
+import warnings
+
+
+def warn_at_caller(warning: Warning) -> None:
+    """Emit a warning at the first frame of the call stack whose module is outside the ``timenet`` package.
+
+    Args:
+        warning: The warning to emit.
+    """
+    # The walk starts at this function, which is level 1 of the stack and lies in the package.
+    frame = inspect.currentframe()
+    level = 1
+    while frame is not None and _in_timenet(frame.f_globals.get("__name__", "")):
+        frame = frame.f_back
+        level += 1
+    warnings.warn(warning, stacklevel=level)
+
+
+def _in_timenet(module: str) -> bool:
+    """Return whether a module name lies in the ``timenet`` package."""
+    return module == "timenet" or module.startswith("timenet.")

packages/timenet/src/timenet/builders.py

@@ -6,9 +6,9 @@
 package plug in the same way the first-party one does.
 """
 
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from dataclasses import dataclass
-from importlib.metadata import entry_points
+from importlib.metadata import EntryPoint, entry_points
 from typing import TYPE_CHECKING, Literal, Protocol
 
 from timenet.errors import TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError
@@ -123,6 +123,38 @@ def release(self) -> DatasetRef:
         return DatasetRef(dataset_id=self.card.dataset_id, version=self.card.dataset_version)
 
 
+@dataclass(frozen=True)
+class ConnectorClaim:
+    """A dataset ID and the installed backend that owns it.
+
+    Attributes:
+        dataset_id: The dataset ID.
+        backend: The first backend that lists the ID.
+        entry: The entry point that registers the backend.
+    """
+
+    dataset_id: str
+    backend: BuilderBackend
+    entry: EntryPoint
+
+
+def claims() -> Iterator[ConnectorClaim]:
+    """Yield each dataset ID of the installed backends once, owned by the first backend that lists it.
+
+    The walk loads each backend only when it reaches its entry point.
+
+    Yields:
+        One claim per dataset ID, in the order of the entry points.
+    """
+    seen: set[str] = set()
+    for entry in entry_points(group=ENTRY_POINT_GROUP):
+        backend = entry.load()()
+        for dataset_id in backend.dataset_ids():
+            if dataset_id not in seen:
+                seen.add(dataset_id)
+                yield ConnectorClaim(dataset_id, backend, entry)
+
+
 def installed_builder(dataset_id: str) -> InstalledBuilder | None:
     """Find the first registered backend that lists a dataset, and read its card.
 
@@ -135,12 +167,11 @@ def installed_builder(dataset_id: str) -> InstalledBuilder | None:
     Raises:
         TimeNetInvalidCardError: If the dataset's card cannot be read.
     """  # ruff: ignore[docstring-extraneous-exception] - raised by the backend
-    for entry in entry_points(group=ENTRY_POINT_GROUP):
-        backend = entry.load()()
-        if dataset_id in backend.dataset_ids():
-            card = backend.card(dataset_id)
-            return None if card is None else InstalledBuilder(backend, card)
-    return None
+    claim = next((claim for claim in claims() if claim.dataset_id == dataset_id), None)
+    if claim is None:
+        return None
+    card = claim.backend.card(dataset_id)
+    return None if card is None else InstalledBuilder(claim.backend, card)
 
 
 def declared(dataset_id: str, version: Version | str | None = None) -> InstalledBuilder:

packages/timenet/src/timenet/cli/app.py

@@ -1,10 +1,12 @@
 """The consumer command-line interface, the command-line mirror of the :class:`~timenet.client.TimeNet` SDK."""
 
-from collections.abc import Callable, Iterable, Sequence
+from collections.abc import Callable, Generator, Iterable, Sequence
+from contextlib import contextmanager
 from functools import partial
 from pathlib import Path
 import shlex
 from typing import TypeVar
+import warnings
 
 from rich.console import Console
 from rich.table import Table
@@ -16,7 +18,7 @@
 from timenet.cli.ui import console as ui, progress_renderer
 from timenet.client import TimeNet, default_local_registry, default_sources
 from timenet.config import settings
-from timenet.errors import TimeNetDatasetNotFoundError, TimeNetNotBuiltError
+from timenet.errors import CatalogWarning, ConnectorCardWarning, TimeNetDatasetNotFoundError, TimeNetNotBuiltError
 from timenet.producer import publish as publish_release
 from timenet.progress import ProgressCallback, ProgressEvent, StepStarted
 from timenet.registry import open_registry
@@ -103,16 +105,35 @@ def _metadata_table(rows: list[DatasetMetadata], title: str) -> Table:
     return table
 
 
+@contextmanager
+def _warning_lines() -> Generator[None]:
+    """Show each CatalogWarning and ConnectorCardWarning of the block as a warning line, without a file and a line."""
+    caught: list[warnings.WarningMessage] = []
+    try:
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", CatalogWarning)
+            warnings.simplefilter("always", ConnectorCardWarning)
+            yield
+    finally:
+        for item in caught:
+            if issubclass(item.category, (CatalogWarning, ConnectorCardWarning)):
+                ui.warn(str(item.message))
+            else:
+                # Any other warning gets its usual output.
+                warnings.warn_explicit(item.message, item.category, item.filename, item.lineno)
+
+
 @app.command("list")
 def list_datasets(
     local_registry: str | None = LOCAL_REGISTRY,
     remote_registry: str | None = REMOTE_REGISTRY,
     auto_build: bool | None = AUTO_BUILD,
 ) -> None:
-    """List the datasets in the registry."""
-    rows = _client(local_registry, remote_registry, auto_build).list()
+    """List each dataset at its highest version across all sources."""
+    with _warning_lines():
+        rows = _client(local_registry, remote_registry, auto_build).list()
     if not rows:
-        typer.echo("No datasets in the registry.")
+        typer.echo("No datasets in the sources.")
         return
     console.print(_metadata_table(rows, "Datasets"))
 
@@ -131,20 +152,21 @@ def search(  # ruff: ignore[too-many-arguments, too-many-positional-arguments]
     remote_registry: str | None = REMOTE_REGISTRY,
     auto_build: bool | None = AUTO_BUILD,
 ) -> None:
-    """Search for datasets that match the filters.
+    """Search datasets across all sources with filters.
 
-    Repeat a filter flag to supply multiple values.
+    Repeat a filter flag to pass several values.
     """
-    results = _client(local_registry, remote_registry, auto_build).search(
-        query=query or None,
-        domain=_enum_list(domain, Domain, "--domain", [d.value for d in Domain]),
-        task=_enum_list(task, lambda value: TASKS[TaskType(value)], "--task", [t.value for t in TaskType]),
-        license=_enum_list(license, License, "--license", [lic.value for lic in License]),
-        time_series_spec=spec or None,
-        dataset_id=dataset_id or None,
-        tag=tag or None,
-        limit=limit,
-    )
+    with _warning_lines():
+        results = _client(local_registry, remote_registry, auto_build).search(
+            query=query or None,
+            domain=_enum_list(domain, Domain, "--domain", [d.value for d in Domain]),
+            task=_enum_list(task, lambda value: TASKS[TaskType(value)], "--task", [t.value for t in TaskType]),
+            license=_enum_list(license, License, "--license", [lic.value for lic in License]),
+            time_series_spec=spec or None,
+            dataset_id=dataset_id or None,
+            tag=tag or None,
+            limit=limit,
+        )
     if not results:
         typer.echo("No datasets match.")
         return

packages/timenet/src/timenet/client.py

@@ -1,7 +1,6 @@
-"""Find, inspect, download, and read datasets with :class:`TimeNet`.
+"""Find, inspect, download, and read TimeF datasets.
 
-A client checks sources in order and builds releases from installed connectors into its first local
-registry that is not read-only.
+:class:`TimeNet` checks sources in order and uses :func:`default_sources` by default.
 """
 
 # The public API has a method named ``list``. Deferred annotations keep the type hint
@@ -15,19 +14,22 @@
 import shlex
 from typing import TYPE_CHECKING, TypeAlias, TypeVar
 
+from timenet._warnings import warn_at_caller
 from timenet.builders import BuildOptions
 from timenet.config import settings
 from timenet.coordinator.build import run_plan
 from timenet.coordinator.plan import Lookups, plan_download
 from timenet.coordinator.read import raise_not_found, read_release
 from timenet.dataset import TimeFDataset
 from timenet.errors import (
+    CatalogWarning,
     TimeFValidationError,
     TimeNetAccessError,
     TimeNetDatasetNotFoundError,
     TimeNetError,
     TimeNetFileError,
     TimeNetMissingParentError,
+    TimeNetNoCatalogError,
     TimeNetNotBuiltError,
     TimeNetVersionUnavailableError,
 )
@@ -47,7 +49,7 @@
     ReleaseSource,
     resolve_release,
 )
-from timenet.types import DatasetMetadata, Domain, License, Task
+from timenet.types import DatasetFilter, DatasetMetadata, Domain, License, ListedRelease, Task
 from timenet.types.references import DatasetRef, split_ref
 
 
@@ -60,6 +62,7 @@
 # annotations.
 T = TypeVar("T")
 _Metadatas: TypeAlias = list[DatasetMetadata]
+_Listed: TypeAlias = list[ListedRelease]
 _OrList: TypeAlias = T | list[T] | None
 
 
@@ -197,14 +200,48 @@ def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None)
             )
 
     def list(self) -> _Metadatas:
-        """Return metadata for every dataset in the selected registry.
+        """List each dataset at its highest version across all sources, including connector cards.
 
-        The client uses the first registry other than the local registry, or the local registry if it is the only one.
+        A listed version can be newer than the one that :meth:`open` reads, which skips a version that a registry
+        does not serve and, without a version, takes the first source that serves the dataset.
+        A source without a catalog is left out with a :class:`~timenet.errors.CatalogWarning`. Source errors stop
+        the call.
 
         Returns:
-            One :class:`~timenet.types.DatasetMetadata` per dataset.
+            One :class:`~timenet.types.DatasetMetadata` per dataset, sorted by dataset ID.
+
+        Raises:
+            TimeNetNoCatalogError: If no source has a catalog.
+            TimeNetInvalidCardError: If an installed backend cannot list its dataset IDs.
+        """  # ruff: ignore[docstring-extraneous-exception] (raised by the backend)
+        return [release.card for release in self._listed()]
+
+    def _listed(self) -> _Listed:
+        """Return each dataset at its highest version across the sources, sorted by dataset ID.
+
+        For equal versions, the earlier source keeps the row. A connector card has no registry. A source without a
+        catalog is left out with a :class:`~timenet.errors.CatalogWarning`.
+
+        Returns:
+            One listed release per dataset.
+
+        Raises:
+            TimeNetNoCatalogError: If no source has a catalog.
         """
-        return self._catalog().list_datasets()
+        if not any(source.has_catalog for source in self._sources):
+            names = ", ".join(str(source) for source in self._sources)
+            raise TimeNetNoCatalogError(f"none of the sources has a catalog: {names}")
+        kept: dict[str, ListedRelease] = {}
+        for source in self._sources:
+            if not source.has_catalog:
+                warn_at_caller(CatalogWarning(f"{source} has no catalog, so the list leaves it out."))
+                continue
+            registry = source if isinstance(source, BaseRegistry) else None
+            for card in source.list_datasets():
+                held = kept.get(card.dataset_id)
+                if held is None or card.dataset_version > held.card.dataset_version:
+                    kept[card.dataset_id] = ListedRelease(card, registry)
+        return [kept[dataset_id] for dataset_id in sorted(kept)]
 
     def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest:
         """Return the manifest of the release that :meth:`open` reads, without a copy or a build.
@@ -269,24 +306,33 @@ def search(  # ruff: ignore[too-many-arguments]
         tag: _OrList[str] = None,
         limit: int = 100,
     ) -> _Metadatas:
-        """Search the selected registry for datasets that match the filters.
+        """Search the rows of :meth:`list`: each dataset at its highest version across all sources.
 
-        The client uses the first registry other than the local registry, or the local registry if it is the only one.
+        So the search checks only the highest version of each dataset, and an older version that matches is not found.
+        A ``task`` or ``time_series_spec`` filter reads the manifest of a row only if its card matches, and the search
+        stops at ``limit`` matches. Connector cards lack schemas, so they never match these filters.
+        A source without a catalog is left out with a :class:`~timenet.errors.CatalogWarning`. Source errors stop
+        the call.
 
         Args:
-            query: Free text terms for the name, description, and tags.
+            query: Match any term in the name, description, or tags, ignoring case.
             domain: Match any of these domains.
             task: Match any of these task classes in the schema.
             license: Match any of these licenses.
             time_series_spec: Match all of these ``spec_type`` values.
-            dataset_id: Match only these IDs.
+            dataset_id: Match any of these IDs.
             tag: Match all of these tags.
             limit: The maximum number of results.
 
         Returns:
-            The matching dataset metadata.
-        """
-        return self._catalog().search(
+            The matching dataset metadata, sorted by dataset ID.
+
+        Raises:
+            TimeFValidationError: If ``limit`` is negative.
+            TimeNetNoCatalogError: If no source has a catalog.
+            TimeNetInvalidCardError: If an installed backend cannot list its dataset IDs.
+        """  # ruff: ignore[docstring-extraneous-exception] (raised by the backend)
+        wanted = DatasetFilter.of(
             query=query,
             domain=domain,
             task=task,
@@ -296,6 +342,7 @@ def search(  # ruff: ignore[too-many-arguments]
             tag=tag,
             limit=limit,
         )
+        return wanted.select(self._listed())
 
     def download(
         self,
@@ -450,24 +497,6 @@ def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset:
         with self.open(dataset_id, version) as reader:
             return reader.read(load=LoadPlan.ALL_EAGER)
 
-    def _catalog(self) -> BaseRegistry:
-        """Choose the registry to browse.
-
-        Returns:
-            The first registry other than the local registry, or the local registry if it is the only one.
-
-        Raises:
-            TimeFValidationError: If the sources contain no registry.
-        """
-        # TODO(adrian): Remove temporary glue which maps these methods onto one registry until
-        # list() and search() ask every source. This registry gives the results of the client of
-        # one registry, from before the list of sources.
-        registries = [source for source in self._sources if isinstance(source, BaseRegistry)]
-        registry = next((registry for registry in registries if registry is not self._store), self._store)
-        if registry is None:
-            raise TimeFValidationError("list() and search() need a registry in the sources")
-        return registry
-
     @contextmanager
     def _hints(self, dataset_id: str) -> Generator[None]:
         """Add recovery hints to errors raised inside the block.

packages/timenet/src/timenet/errors.py

@@ -31,6 +31,10 @@ class TimeNetRegistryError(TimeNetError):
     """TimeNet could not load or reach a registry, or the registry could not serve a request."""
 
 
+class TimeNetNoCatalogError(TimeNetRegistryError):
+    """No source of a dataset listing has a catalog."""
+
+
 class TimeNetDatasetNotFoundError(TimeNetError):
     """A requested dataset id or version is not present in a registry or on disk."""
 
@@ -219,7 +223,11 @@ class TimeNetWarning(UserWarning):
 
 
 class ConnectorCardWarning(TimeNetWarning):
-    """An unreadable or invalid connector card excludes its dataset from listings."""
+    """A dataset listing skips a dataset with an unreadable connector card."""
+
+
+class CatalogWarning(TimeNetWarning):
+    """A dataset listing skips a source without a catalog."""
 
 
 class SpanOutsideWindowWarning(TimeNetWarning):