feat(coordinator)!: plan and run builds through build_release

packages/timenet-connectors/src/timenet_connectors/builder/backend.py

@@ -8,11 +8,19 @@
 from timenet.types import DatasetMetadata
 from timenet.types.references import DatasetRef
 from timenet_connectors.builder.cli import build_one
-from timenet_connectors.discovery import connector_dir, has_connector
+from timenet_connectors.discovery import connector_dir, has_connector, known_ids
 
 
 class ConnectorBuilder:
-    """Builds a dataset by running its connector in an environment built from its requirements."""
+    """Build a dataset with its connector, in an environment built from the connector's requirements."""
+
+    def dataset_ids(self) -> list[str]:  # ruff: ignore[no-self-use] (BuilderBackend protocol method)
+        """List this package's dataset IDs without importing connectors.
+
+        Returns:
+            The sorted dataset IDs. A card that cannot be read is skipped with a warning.
+        """
+        return known_ids()
 
     def knows(self, dataset_id: str) -> bool:  # ruff: ignore[no-self-use] (BuilderBackend protocol method)
         """Report whether a connector package exists for the id, without importing it.

packages/timenet-connectors/src/timenet_connectors/builder/cli.py

@@ -23,7 +23,7 @@
 )
 import typer
 
-from timenet.builders import BuilderBackend, find_builder, find_card
+from timenet.builders import BuilderBackend, declared, find_card, installed_builder
 from timenet.cache import human_bytes
 from timenet.cli.runner import run_cli
 from timenet.cli.ui import console
@@ -46,10 +46,10 @@
     open_registry,
     open_writable_registry,
 )
-from timenet.types.references import DatasetRef
+from timenet.types.references import DatasetRef, split_ref
 from timenet.writer.progress import ProgressStage, WriteProgressEvent
 from timenet_connectors.builder.env import run_isolated
-from timenet_connectors.discovery import connector_dir, resolve
+from timenet_connectors.discovery import known_ids, resolve
 from timenet_connectors.download import DownloadProgress, ProgressCallback, progress_sink
 
 
@@ -88,7 +88,7 @@ def _resolve_target(out: str | None) -> WritableRegistry | Path:
 @app.command()
 # ruff: ignore[too-many-arguments, too-many-positional-arguments] (a typer command: one parameter per option)
 def build(
-    dataset_id: str,
+    release: str = typer.Argument(..., metavar="DATASET_ID[@VERSION]"),
     out: str | None = typer.Option(
         None,
         "--out",
@@ -141,17 +141,20 @@ def build(
 
     Raises:
         BadParameter: If ``dataset_id`` has no known connector.
+        TimeNetVersionUnavailableError: If the connector cannot build the requested version.
         TimeNetMissingParentError: If a parent is missing. The hint of the error names the flags.
         TimeNetFileError: If a copy of a parent in the output is damaged, or a fetched file does not
             match its manifest record. The hint of the error names the flags.
         TimeNetAccessError: If the output is remote and gets a release without access declared
             ``open``.
-    """
-    try:
-        # connector_dir imports no connector. So an unknown id fails before any environment exists.
-        connector_dir(dataset_id)
-    except LookupError as exc:
-        raise typer.BadParameter(str(exc)) from exc
+    """  # ruff: ignore[docstring-extraneous-exception] - raised by declared and the build
+    # known_ids() imports no connector. So an unknown id fails before any environment exists.
+    dataset_id, version = split_ref(release)
+    known = known_ids()
+    if dataset_id not in known:
+        raise typer.BadParameter(f"no connector for {dataset_id!r}; known: {', '.join(known) or '(none)'}")
+    if version is not None:
+        declared(dataset_id, version)
     forwarded = out if out is not None else settings().registry
     if forwarded is None:
         forwarded = str(settings().registry_path)
@@ -204,7 +207,7 @@ def build_one(  # ruff: ignore[too-many-arguments]
         dataset_id: The dataset ID.
         output: A local directory or a ``timenet://``, ``http(s)://``, or ``s3://`` registry URL.
         fetch_from: The registry, or its URI, to copy missing parent releases from.
-        build_parents: Build missing parent releases from their installed connectors.
+        build_parents: Build missing parents with installed connectors.
         force: Rebuild an existing version.
         keep_cache: Keep the raw download cache after building.
         isolation: Use a separate environment if true, this interpreter if false, or ``TIMENET_ISOLATION`` if ``None``.
@@ -220,9 +223,9 @@ def build_one(  # ruff: ignore[too-many-arguments]
     isolated = settings(isolation=override).isolation == "on"
 
     def build_parent(step: Step) -> object:
-        builder = find_builder(step.release.dataset_id)
-        if builder is not None and not _in_repo(builder):
-            return _build_with(builder, step, target)
+        found = installed_builder(step.release.dataset_id)
+        if found is not None and not _in_repo(found.backend):
+            return _build_with(found.backend, step, target)
         # Each parent builds like the dataset itself. An isolated child makes the plan again. The
         # child finds every parent in the output, so it only opens the parents.
         return _build(

packages/timenet-connectors/src/timenet_connectors/discovery.py

@@ -11,10 +11,12 @@
 import importlib.util
 from pathlib import Path
 import pkgutil
+import warnings
 
 import yaml
 
 from timenet.connectors import BaseConnector
+from timenet.errors import ConnectorCardWarning
 import timenet_connectors.datasets as _datasets
 
 
@@ -47,20 +49,17 @@ def connector_dir(dataset_id: str) -> Path:
         path = Path(next(iter(spec.submodule_search_locations)))
         if (path / "dataset.yaml").is_file():
             return path
-    raise LookupError(f"no connector for {dataset_id!r}; known: {', '.join(_known_ids()) or '(none)'}")
+    raise LookupError(f"no connector for {dataset_id!r}; known: {', '.join(known_ids()) or '(none)'}")
 
 
 def has_connector(dataset_id: str) -> bool:
-    """Report whether a connector package exists for an id, without importing it.
-
-    Ask this instead of catching :func:`connector_dir`'s ``LookupError``: a miss is an ordinary
-    answer here, not an exception.
+    """Check for a dataset's connector package and card without importing the connector.
 
     Args:
-        dataset_id: A flat (``hello_world``) or ``org/name`` id.
+        dataset_id: The dataset ID, as ``org/name``.
 
     Returns:
-        Whether a connector package exists for the id.
+        Whether a connector package with a ``dataset.yaml`` card exists.
     """
     try:
         connector_dir(dataset_id)
@@ -70,41 +69,64 @@ def has_connector(dataset_id: str) -> bool:
 
 
 def requirements_for(dataset_id: str) -> Path | None:
-    """Return the connector's ``requirements.txt``, or ``None`` if it declares none.
+    """Find a connector's requirements file.
 
     Args:
-        dataset_id: A flat (``hello_world``) or ``org/name`` id.
+        dataset_id: The dataset ID, as ``org/name``.
 
     Returns:
-        The path to the connector's ``requirements.txt``, or ``None``.
+        The path to ``requirements.txt``, or ``None`` if the file is absent.
 
     Raises:
-        LookupError: If no module exists for the id.
+        LookupError: If no connector with a ``dataset.yaml`` card exists for the ID.
     """  # ruff: ignore[docstring-extraneous-exception] (raised by connector_dir, not directly here)
     path = connector_dir(dataset_id) / "requirements.txt"
     return path if path.is_file() else None
 
 
-def _known_ids() -> list[str]:
-    """List the dataset ids declared by the connector cards on disk, importing nothing.
+def known_ids() -> list[str]:
+    """List dataset IDs from connector cards without importing connectors.
 
-    :func:`available` answers the same question by importing every connector. The unknown-id error
-    path cannot afford that: those dependencies live in each connector's own environment, not this
-    one.
+    A card that cannot be read, is not a YAML mapping, or lacks ``dataset_id`` is skipped with a
+    :class:`~timenet.errors.ConnectorCardWarning`, so it fails only its own dataset.
 
     Returns:
-        The declared dataset ids, sorted.
+        The sorted dataset IDs, with duplicates removed.
     """
     ids: set[str] = set()
     for root in _datasets.__path__:
         # A flat id sits at ``datasets/<name>/``, an ``org/name`` id one level deeper.
         for card in (*Path(root).glob("*/dataset.yaml"), *Path(root).glob("*/*/dataset.yaml")):
-            declared = (yaml.safe_load(card.read_text(encoding="utf-8")) or {}).get("dataset_id")
-            if declared:
+            declared = _declared_id(card)
+            if declared is not None:
                 ids.add(declared)
     return sorted(ids)
 
 
+def _declared_id(card: Path) -> str | None:
+    """Read the dataset ID of a card, or warn and return ``None`` for a card that cannot be read.
+
+    Args:
+        card: The ``dataset.yaml`` file.
+
+    Returns:
+        The declared dataset ID, or ``None``.
+    """
+    try:
+        raw = yaml.safe_load(card.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError) as error:
+        problem = f"could not read dataset card {card}: {error}"
+    else:
+        if not isinstance(raw, dict):
+            problem = f"dataset card {card} must be a YAML mapping, got {type(raw).__name__}"
+        elif not raw.get("dataset_id"):
+            problem = f"dataset card {card} must declare a dataset_id"
+        else:
+            return raw["dataset_id"]
+    warnings.warn(f"{problem}; its dataset is skipped", ConnectorCardWarning, stacklevel=3)
+    return None
+
+
 def _connector_spec(dataset_id: str) -> importlib.machinery.ModuleSpec | None:
     """Find a connector module's spec without importing the module itself.
 

packages/timenet-connectors/tests/test_build_backend.py

@@ -1,17 +1,21 @@
 from pathlib import Path
+import re
 import sys
 
 import pytest
 
-from timenet.errors import TimeNetInvalidCardError
+from timenet.builders import installed_builder
+from timenet.errors import ConnectorCardWarning, TimeNetInvalidCardError
 from timenet.manifest import Manifest, PayloadSource
 from timenet.manifest.files import FilePart
 from timenet.reader import TimeFReader
 from timenet.registry import open_local_version
 from timenet.types.references import DatasetRef
 from timenet.values_backends import ValuesBackend
+from timenet_connectors import datasets as datasets_module
 from timenet_connectors.builder import backend as backend_module, cli as cli_module
 from timenet_connectors.builder.backend import ConnectorBuilder
+from timenet_connectors.discovery import known_ids
 
 
 def _unexpected_isolation(*args, **kwargs):
@@ -31,6 +35,17 @@ def test_knows_rejects_an_unknown_id_without_importing_a_connector():
     assert leaf not in sys.modules
 
 
+def test_connector_builder_lists_its_dataset_ids_without_importing_a_connector(monkeypatch):
+    leaf = "timenet_connectors.datasets.physionet.ecg_qa_cot"
+    monkeypatch.delitem(sys.modules, leaf, raising=False)
+
+    dataset_ids = list(ConnectorBuilder().dataset_ids())
+
+    assert {"timenet/hello-world", "physionet/ecg-qa-cot"} <= set(dataset_ids)
+    assert "nope/nothing" not in dataset_ids
+    assert leaf not in sys.modules
+
+
 def test_a_card_that_cannot_be_read_raises_and_only_a_missing_connector_gives_none(tmp_path, monkeypatch):
     assert ConnectorBuilder().card("nope/nothing") is None
     assert ConnectorBuilder().declared_version("nope/nothing") is None
@@ -43,6 +58,29 @@ def test_a_card_that_cannot_be_read_raises_and_only_a_missing_connector_gives_no
         ConnectorBuilder().declared_version("test/broken")
 
 
+@pytest.mark.parametrize(
+    ("text", "message"),
+    [
+        ("dataset_id: [unclosed\n", "could not read dataset card {}"),
+        ("- a list\n", "dataset card {} must be a YAML mapping"),
+        ("title: no id\n", "dataset card {} must declare a dataset_id"),
+    ],
+)
+def test_a_card_that_cannot_be_read_is_skipped_with_a_warning(tmp_path, monkeypatch, text, message):
+    (tmp_path / "test" / "broken").mkdir(parents=True)
+    (tmp_path / "test" / "broken" / "dataset.yaml").write_text(text)
+    monkeypatch.setattr(datasets_module, "__path__", [*datasets_module.__path__, str(tmp_path)])
+    expected = re.escape(message.format(tmp_path / "test" / "broken" / "dataset.yaml"))
+
+    with pytest.warns(ConnectorCardWarning, match=expected):
+        assert "timenet/hello-world" in known_ids()
+    # The installed entry point loads the builder backend of this package, which lists its dataset IDs.
+    with pytest.warns(ConnectorCardWarning, match=expected):
+        found = installed_builder("timenet/hello-world")
+    assert found is not None
+    assert found.card.dataset_id == "timenet/hello-world"
+
+
 def test_build_runs_in_an_isolated_environment_by_default(tmp_path, monkeypatch):
     monkeypatch.delenv("TIMENET_ISOLATION", raising=False)
     calls = []

packages/timenet-connectors/tests/test_build_cli.py

@@ -6,6 +6,7 @@
 import pytest
 from typer.testing import CliRunner
 
+from timenet.errors import TimeNetVersionUnavailableError
 from timenet.manifest import Manifest
 from timenet.reader import TimeFReader
 from timenet.registry import open_local_version
@@ -89,6 +90,23 @@ def _unexpected_pipeline(*args, **kwargs):
     assert called["isolated"] is True
 
 
+def test_build_takes_an_exact_release_and_refuses_another_version(monkeypatch, tmp_path):
+    built = []
+
+    def _fake_isolated(dataset_id, root, **kwargs):
+        built.append(dataset_id)
+        return Path(root) / "timenet/hello-world/1.0.0"
+
+    monkeypatch.setattr(cli_module, "run_isolated", _fake_isolated)
+    result = runner.invoke(app, ["build", "timenet/hello-world@1.0.0", "--out", str(tmp_path)])
+
+    assert result.exit_code == 0
+    assert built == ["timenet/hello-world"]
+    result = runner.invoke(app, ["build", "timenet/hello-world@9.9.9", "--out", str(tmp_path)])
+    assert isinstance(result.exception, TimeNetVersionUnavailableError)
+    assert built == ["timenet/hello-world"]
+
+
 def test_build_rejects_an_unknown_id_without_importing_anything(monkeypatch, tmp_path):
     # A connector's dependencies live in its own environment, not this one, so importing one here
     # to answer "which ids exist?" would raise ModuleNotFoundError instead of the unknown-id error.
@@ -106,6 +124,15 @@ def test_build_rejects_an_unknown_id_without_importing_anything(monkeypatch, tmp
     assert connectors == []
 
 
+def test_build_rejects_an_id_that_no_card_declares_and_names_the_known_ids(tmp_path):
+    # The module path of the connector accepts this spelling, but a dataset ID must match its card.
+    result = runner.invoke(app, ["build", "timenet/hello_world", "--out", str(tmp_path)])
+
+    assert result.exit_code == 2
+    assert "no connector for 'timenet/hello_world'" in result.output
+    assert "timenet/hello-world" in result.output
+
+
 def test_build_forwards_quiet_to_the_isolated_child(monkeypatch, tmp_path):
     captured = {}
     # The console is a process-wide singleton; monkeypatch restores it for the tests that follow.