Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 35 additions & 69 deletions packages/timenet-connectors/src/timenet_connectors/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,83 +24,54 @@ def build( # ruff: ignore[too-many-arguments]
fetch_from: "str | BaseRegistry | None" = None,
build_parents: bool = False,
) -> Path:
# ruff: ignore[docstring-extraneous-exception] (raised by _check_version and the parent planner, not directly here)
"""Build a dataset into a local registry by connector id.
# ruff: ignore[docstring-extraneous-exception] (raised by build_release, not directly here)
"""Build a dataset release in a local registry.

This is the producer-side one-liner over the engine. The build reuses an already-built version
unless the caller sets ``force``. The default output is the shared local registry
(:func:`timenet.registry.default_registry_path`). The SDK reads from that directory, so a build
here loads at once with ``TimeNet().load(dataset_id)``.

Like ``timenet-build build``, this runs the connector in an environment built from its
``requirements.txt``. Set ``TIMENET_ISOLATION=off`` to run it in this interpreter instead, which
is what you want while writing a connector. Missing parent releases fail the build before any
download, unless ``fetch_from`` or ``build_parents`` obtains them.
The build reuses an existing release unless you set ``force``. The connector runs in an environment
with its required packages, or in this interpreter if ``TIMENET_ISOLATION=off``.

Args:
dataset_id: The dataset id (``org/name``).
version: The expected dataset version. A connector produces only its own version, so this
value is a guard. If it does not match the connector's metadata, the build stops before it
runs. ``None`` builds the version that the connector declares.
out: Output registry directory. Defaults to the shared local registry.
force: Rebuild even if the registry already has a built version.
keep_cache: Keep the raw download cache. A successful build deletes it by default.
dataset_id: The connector's dataset ID (``org/name``).
version: The version to build, or ``None`` for the connector's declared version.
out: The output directory, or ``None`` to use the default local registry.
force: Rebuild the release even if it exists in the registry.
keep_cache: Keep downloaded source files after a successful build.
fetch_from: The registry, or its URI, to copy missing parent releases from.
build_parents: Build missing parent releases from their installed connectors.

Returns:
The committed version directory.
The release's version directory.

Raises:
TimeFValidationError: If the caller sets ``version`` and it does not match the connector's
declared version.
TimeNetMissingParentError: If a parent release is missing, with a hint that names the
keywords.
TimeNetDatasetNotFoundError: If no installed connector builds ``dataset_id``.
TimeNetVersionUnavailableError: If ``version`` differs from the connector's declared version.
TimeNetMissingParentError: If a required parent release is still missing.
"""
from timenet.builders import BuildOptions, declared # ruff: ignore[import-outside-top-level]
from timenet.coordinator.build import build_release # ruff: ignore[import-outside-top-level]
from timenet.errors import TimeNetMissingParentError # ruff: ignore[import-outside-top-level]
from timenet.registry import default_registry_path # ruff: ignore[import-outside-top-level]
from timenet_connectors.builder.cli import build_one # ruff: ignore[import-outside-top-level]

if version is not None:
_check_version(dataset_id, version)
from timenet.registry import ( # ruff: ignore[import-outside-top-level]
LocalRegistry,
default_registry_path,
open_registry,
)
from timenet.registry._paths import release_dir # ruff: ignore[import-outside-top-level, import-private-name]

release = declared(dataset_id).release
root = Path(out).expanduser() if out is not None else default_registry_path()
try:
built = build_one(
dataset_id, root, fetch_from=fetch_from, build_parents=build_parents, force=force, keep_cache=keep_cache
build_release(
f"{dataset_id}@{version}" if version else dataset_id,
store=LocalRegistry(root),
fetch_from=open_registry(fetch_from) if isinstance(fetch_from, str) else fetch_from,
build_parents=build_parents,
force=force,
options=BuildOptions(keep_cache=keep_cache),
)
except TimeNetMissingParentError as error:
error.hint = "Copy it with fetch_from=<registry> or build it with build_parents=True."
raise
return Path(built)


def _check_version(dataset_id: str, version: str) -> None:
# ruff: ignore[docstring-extraneous-exception] (LookupError and TimeNetInvalidCardError come from the calls below)
"""Reject a requested version the connector does not build.

Deliberately import-free, like :func:`timenet_connectors.discovery.requirements_for`. This guard
runs before the isolated build, so asking the connector for its metadata would need the very
dependencies that only the child environment has. It reads the card beside the connector instead.

Args:
dataset_id: The dataset id (``org/name``).
version: The requested dataset version.

Raises:
TimeFValidationError: If the connector declares a different version.
LookupError: If no connector exists for the id.
TimeNetInvalidCardError: If the connector's card is missing or invalid.
"""
from timenet.errors import TimeFValidationError # ruff: ignore[import-outside-top-level]
from timenet.types import DatasetMetadata # ruff: ignore[import-outside-top-level]
from timenet_connectors.discovery import connector_dir # ruff: ignore[import-outside-top-level]

card = DatasetMetadata.from_yaml(connector_dir(dataset_id) / "dataset.yaml")
available = str(card.dataset_version)
if version != available:
raise TimeFValidationError(
f"connector for {dataset_id!r} builds version {available}, not the requested {version}"
)
return release_dir(root, release)


def load(
Expand All @@ -110,17 +81,12 @@ def load(
fetch_from: "str | BaseRegistry | None" = None,
build_parents: bool = False,
) -> "TimeFDataset":
"""Build a dataset if needed, then load it into memory.

This is the one-call sugar over :func:`build` plus :meth:`timenet.client.TimeNet.load`. For a
clear producer and consumer split, call :func:`build` and ``TimeNet().load`` yourself.
"""Build a dataset if needed and load it into memory.

Args:
dataset_id: The dataset id (``org/name``).
version: The version string, or ``None`` for the latest. :func:`build` checks this value
against the connector's declared version and rejects a mismatch.
fetch_from: The registry, or its URI, that the build and the read copy missing parent
releases and payload sources from.
dataset_id: The dataset ID (``org/name``).
version: The version declared by the connector, or ``None`` to load the latest version.
fetch_from: The registry, or its URI, to copy missing parents and payload sources from.
build_parents: Build missing parent releases from their installed connectors.

Returns:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ def build_prepared(
"""Build a release in a local registry.

The registry must already contain all parent releases and their payload sources.
Set ``TIMENET_ISOLATION=off`` to build in this interpreter.
``options.isolation``, else ``TIMENET_ISOLATION``, set to ``"off"`` builds in this interpreter.

Args:
release: The exact dataset ID and version to build.
Expand All @@ -142,7 +142,7 @@ def build_prepared(
raise TimeNetDatasetNotFoundError(f"no connector builds {release.dataset_id!r}")
if card.dataset_version != release.version:
raise TimeNetVersionUnavailableError(release, card.dataset_version)
if settings().isolation == "on":
if settings(isolation=options.isolation).isolation == "on":
# The parent reads only the card: the connector and its dependencies exist only in the child.
run_isolated_build(
release,
Expand Down
Loading
Loading