diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py index dd6e88260..8be73fadb 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/cli.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/cli.py @@ -113,7 +113,7 @@ def build( sources = default_sources(local_registry=local_registry, remote_registry=remote_registry, auto_build=auto_build) # The default list starts with the local registry, which the build writes into. The planner leaves it out of # the sources of the copies and builds. - store = cast("LocalRegistry", sources[0]) + local = cast("LocalRegistry", sources[0]) dest = None if publish_to is None else open_registry(publish_to, label="--publish-to") has_connectors = any(isinstance(source, InstalledConnectors) for source in sources) try: @@ -126,7 +126,7 @@ def build( # plan refuses another release before any download. plan = plan_release_build( str(wanted), - store=store, + target_registry=local, parents_from=sources, force=force, open_access_only=dest is not None and dest.requires_open_access, @@ -136,7 +136,7 @@ def build( if dest is not None: lines.append(f"then publish {wanted} to {dest}") elif not plan.steps: - console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {store}") + console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {local}") if lines: typer.echo("\n".join(lines)) return @@ -149,9 +149,9 @@ def build( quiet=console.quiet, isolation=None if isolation is None else ("on" if isolation else "off"), ) - run_plan(plan, store, options=options) + run_plan(plan, local, options=options) if dest is not None: - publish(wanted, dest=dest, source=store, progress=progress) + publish(wanted, dest=dest, source=local, progress=progress) except TimeNetMissingParentError as error: error.hint = _missing_hint(has_connectors=has_connectors) raise @@ -167,9 +167,9 @@ def build( error.hint = _access_hint(dest, has_connectors=has_connectors) raise if dest is None and not plan.steps: - console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {store}") + console.success(f"Nothing to do: '{wanted.dataset_id}' {wanted.version} is already in {local}") # A publish prints the version, and a local build prints the release directory. - typer.echo(str(wanted.version) if dest is not None else str(release_dir(store.root, wanted))) + typer.echo(str(wanted.version) if dest is not None else str(release_dir(local.root, wanted))) def _missing_hint(*, has_connectors: bool) -> str: diff --git a/packages/timenet-connectors/tests/test_build_env.py b/packages/timenet-connectors/tests/test_build_env.py index 7b26a698c..f42bb2038 100644 --- a/packages/timenet-connectors/tests/test_build_env.py +++ b/packages/timenet-connectors/tests/test_build_env.py @@ -165,7 +165,7 @@ def _missing(): uv_command(_spec(), ["timenet-build"]) -def test_run_isolated_build_runs_build_release_with_the_store_and_the_pins(monkeypatch, tmp_path): +def test_run_isolated_build_runs_build_release_with_the_target_registry_and_the_pins(monkeypatch, tmp_path): release = DatasetRef.parse("timenet/hello-world@1.0.0") pinned = PayloadSource(release=release, files=(FilePart(path="a.bin", checksum="sha256:" + "0" * 64, size=1),)) captured = {} diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index c3f363a21..cfc5b3c3a 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -45,9 +45,9 @@ runner = CliRunner() -def _parent_steps(dataset_id: str, store: LocalRegistry, **options) -> list[Step]: +def _parent_steps(dataset_id: str, target_registry: LocalRegistry, **options) -> list[Step]: """Plan the build of ``dataset_id`` and return each step but its own build.""" - steps = plan_release_build(dataset_id, store=store, **options).steps + steps = plan_release_build(dataset_id, target_registry=target_registry, **options).steps return [step for step in steps if not (step.action == "build" and step.release.dataset_id == dataset_id)] @@ -110,7 +110,7 @@ def _install_cards(monkeypatch, **parents: dict[str, DatasetRef]) -> dict: @pytest.mark.parametrize( ("grand_in", "expected"), [ - ("output", lambda origin: [_build(PARENT)]), + ("target", lambda origin: [_build(PARENT)]), ( "origin", lambda origin: [ @@ -129,12 +129,12 @@ def _install_cards(monkeypatch, **parents: dict[str, DatasetRef]) -> dict: ) def test_plan_copies_or_builds_each_parent_once_before_its_children(tmp_path, monkeypatch, grand_in, expected): _install_cards(monkeypatch, child={"parent": PARENT, "grand": GRAND}, parent={"grand": GRAND}) - registries = {name: LocalRegistry(tmp_path / name) for name in ("output", "origin")} + registries = {name: LocalRegistry(tmp_path / name) for name in ("target", "origin")} if grand_in in registries: registries[grand_in].put_dataset(make_dataset()) steps = _parent_steps( - CHILD.dataset_id, registries["output"], parents_from=[registries["origin"], InstalledConnectors()] + CHILD.dataset_id, registries["target"], parents_from=[registries["origin"], InstalledConnectors()] ) assert steps == expected(registries["origin"]) @@ -161,10 +161,10 @@ def test_plan_copies_or_builds_each_parent_once_before_its_children(tmp_path, mo # ruff: ignore[too-many-arguments, too-many-positional-arguments] def test_plan_rejects_unobtainable_parents_before_anything_runs(tmp_path, monkeypatch, cards, options, error, match): _install_cards(monkeypatch, **cards) - output = LocalRegistry(tmp_path / "output") + target_registry = LocalRegistry(tmp_path / "output") with pytest.raises(error, match=match): - _parent_steps(CHILD.dataset_id, output, **options) + _parent_steps(CHILD.dataset_id, target_registry, **options) _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) @@ -202,27 +202,29 @@ def _declared_without_its_sources_source(registry: LocalRegistry) -> None: shutil.rmtree(release_dir(registry.root, GRAND)) -@pytest.mark.parametrize("where", ["output", "origin", "source-in-origin"]) +@pytest.mark.parametrize("where", ["target", "origin", "source-in-origin"]) def test_a_declared_parent_needs_only_its_direct_payload_sources(tmp_path, monkeypatch, child_builds, where): _install_cards(monkeypatch, child={"declared": DECLARED}) - output, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") - _declared_without_its_sources_source(origin if where == "origin" else output) + target_registry, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") + _declared_without_its_sources_source(origin if where == "origin" else target_registry) if where == "source-in-origin": release_dir(origin.root, SOURCE).parent.mkdir(parents=True) - shutil.move(release_dir(output.root, SOURCE), release_dir(origin.root, SOURCE)) + shutil.move(release_dir(target_registry.root, SOURCE), release_dir(origin.root, SOURCE)) # DECLARED opens with SOURCE, its direct payload source. So the plan needs neither GRAND, which - # SOURCE reads and no registry holds, nor a step for SOURCE where the output holds it. + # SOURCE reads and no registry holds, nor a step for SOURCE where the target registry holds it. steps = { - "output": [], + "target": [], "origin": [CopyStep(DECLARED, origin)], "source-in-origin": [CopyStep(SOURCE, origin, with_payload_sources=False)], } - assert _parent_steps(CHILD.dataset_id, output, parents_from=[origin]) == steps[where] - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[origin]), output) + assert _parent_steps(CHILD.dataset_id, target_registry, parents_from=[origin]) == steps[where] + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[origin]), target_registry + ) for reference in (DECLARED, SOURCE): - assert output.exists(reference.dataset_id, str(reference.version)) - assert not output.exists(GRAND.dataset_id, str(GRAND.version)) + assert target_registry.exists(reference.dataset_id, str(reference.version)) + assert not target_registry.exists(GRAND.dataset_id, str(GRAND.version)) # Only the requested release builds. assert child_builds == [(CHILD.dataset_id, False)] @@ -231,11 +233,11 @@ def test_a_payload_source_that_is_later_a_declared_parent_brings_its_own_payload # The child declares DECLARED first, then SOURCE. The walk finds SOURCE first as the payload source # of DECLARED. As a declared parent, SOURCE also needs GRAND, its own payload source, which is missing. _install_cards(monkeypatch, child={"declared": DECLARED, "source": SOURCE}) - output = LocalRegistry(tmp_path / "output") - _declared_without_its_sources_source(output) - pins = output.get_manifest(SOURCE.dataset_id).payload_sources[SourceKey(str(GRAND))] + target_registry = LocalRegistry(tmp_path / "output") + _declared_without_its_sources_source(target_registry) + pins = target_registry.get_manifest(SOURCE.dataset_id).payload_sources[SourceKey(str(GRAND))] - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert steps == [_build(GRAND, pinned_by={SOURCE: pins})] @@ -275,22 +277,22 @@ def convert(self, raw_refs): return make_dataset() -def _parent_without_its_source(output: LocalRegistry) -> None: +def _parent_without_its_source(target_registry: LocalRegistry) -> None: """Store GRAND and PARENT, which reads a GRAND Record, then remove GRAND.""" - output.put_dataset(make_dataset()) - _child_of(output, PARENT, GRAND, make_dataset().records[0].id) - shutil.rmtree(release_dir(output.root, GRAND)) + target_registry.put_dataset(make_dataset()) + _child_of(target_registry, PARENT, GRAND, make_dataset().records[0].id) + shutil.rmtree(release_dir(target_registry.root, GRAND)) @pytest.mark.parametrize("backend", [ValuesBackend.PARQUET, ValuesBackend.ZARR]) def test_a_pinned_source_is_rebuilt_in_staging_and_published_only_if_it_matches(tmp_path, monkeypatch, backend): _install_cards(monkeypatch, child={"parent": PARENT}, parent={"parent": GRAND}) monkeypatch.setattr(_Grand, "values_backend", backend) - output = LocalRegistry(tmp_path / "output") - _parent_without_its_source(output) - pins = output.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] + target_registry = LocalRegistry(tmp_path / "output") + _parent_without_its_source(target_registry) + pins = target_registry.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert steps == [_build(GRAND, pinned_by={PARENT: pins})] built = [] @@ -305,33 +307,39 @@ def child(release, store_root, *, pinned_by, **_): monkeypatch.setattr(backend_module, "run_isolated_build", child) if backend == ValuesBackend.PARQUET: - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[InstalledConnectors()]), output) - assert output.exists(GRAND.dataset_id, str(GRAND.version)) + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()]), + target_registry, + ) + assert target_registry.exists(GRAND.dataset_id, str(GRAND.version)) assert built == [GRAND, CHILD] return # A Zarr rebuild writes other files than the Parquet files PARENT reads, so nothing is published. with pytest.raises(TimeFFormatError, match=r"Nothing was published\. Fetch the exact release .* or rebuild"): - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[InstalledConnectors()]), output) + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()]), + target_registry, + ) assert built == [GRAND] - assert not output.exists(GRAND.dataset_id, str(GRAND.version)) - assert not list(release_dir(output.root, GRAND).parent.glob("*")) + assert not target_registry.exists(GRAND.dataset_id, str(GRAND.version)) + assert not list(release_dir(target_registry.root, GRAND).parent.glob("*")) def test_a_payload_source_that_two_releases_read_is_built_against_the_pins_of_both(tmp_path, monkeypatch): sibling = DatasetRef.parse("test/sibling@1.0.0") _install_cards(monkeypatch, child={"parent": PARENT, "sibling": sibling}) - output = LocalRegistry(tmp_path / "output") - output.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "output") + target_registry.put_dataset(make_dataset()) # PARENT and the sibling each read another Record of GRAND, so their pins differ. - _child_of(output, PARENT, GRAND, make_dataset().records[0].id) - _child_of(output, sibling, GRAND, make_dataset().records[1].id) - shutil.rmtree(release_dir(output.root, GRAND)) + _child_of(target_registry, PARENT, GRAND, make_dataset().records[0].id) + _child_of(target_registry, sibling, GRAND, make_dataset().records[1].id) + shutil.rmtree(release_dir(target_registry.root, GRAND)) pins = { - reader: output.get_manifest(reader.dataset_id).payload_sources[SourceKey(str(GRAND))] + reader: target_registry.get_manifest(reader.dataset_id).payload_sources[SourceKey(str(GRAND))] for reader in (PARENT, sibling) } - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert steps == [_build(GRAND, pinned_by=pins)] @@ -350,70 +358,72 @@ def _repin(registry: LocalRegistry, reader: DatasetRef) -> None: def test_two_readers_that_pin_one_file_differently_fail_the_plan_of_its_build(tmp_path, monkeypatch): sibling = DatasetRef.parse("test/sibling@1.0.0") _install_cards(monkeypatch, child={"parent": PARENT, "sibling": sibling}) - output = LocalRegistry(tmp_path / "output") - output.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "output") + target_registry.put_dataset(make_dataset()) for reader in (PARENT, sibling): - _child_of(output, reader, GRAND, make_dataset().records[0].id) - _repin(output, sibling) - shutil.rmtree(release_dir(output.root, GRAND)) + _child_of(target_registry, reader, GRAND, make_dataset().records[0].id) + _repin(target_registry, sibling) + shutil.rmtree(release_dir(target_registry.root, GRAND)) # No build of GRAND can match both readers, so the plan fails before any step runs. with pytest.raises(TimeFFormatError, match=f"pin .* of {GRAND} as different files, so no build"): - plan_release_build(CHILD.dataset_id, store=output, parents_from=[InstalledConnectors()]) + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()]) def test_a_held_payload_source_that_breaks_a_pin_fails_the_plan(tmp_path, monkeypatch): _install_cards(monkeypatch, child={"parent": PARENT}) - output = LocalRegistry(tmp_path / "output") - output.put_dataset(make_dataset()) - _child_of(output, PARENT, GRAND, make_dataset().records[0].id) - _repin(output, PARENT) + target_registry = LocalRegistry(tmp_path / "output") + target_registry.put_dataset(make_dataset()) + _child_of(target_registry, PARENT, GRAND, make_dataset().records[0].id) + _repin(target_registry, PARENT) with pytest.raises(TimeFFormatError, match=f"payload source {GRAND} differs from the release {PARENT}"): - plan_release_build(CHILD.dataset_id, store=output) + plan_release_build(CHILD.dataset_id, target_registry=target_registry) def test_a_built_release_needs_only_its_payload_sources(tmp_path, monkeypatch): _install_cards(monkeypatch, parent={"parent": GRAND}) - output = LocalRegistry(tmp_path / "output") - _parent_without_its_source(output) - pins = output.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] + target_registry = LocalRegistry(tmp_path / "output") + _parent_without_its_source(target_registry) + pins = target_registry.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] with pytest.raises(TimeNetMissingParentError, match=r"^payload source timenet/hello-world@1\.0\.0 of test/parent"): - _parent_steps(PARENT.dataset_id, output) + _parent_steps(PARENT.dataset_id, target_registry) connectors = [InstalledConnectors()] # The source that PARENT reads builds against its pins. - assert _parent_steps(PARENT.dataset_id, output, parents_from=connectors) == [ + assert _parent_steps(PARENT.dataset_id, target_registry, parents_from=connectors) == [ _build(GRAND, pinned_by={PARENT: pins}) ] # A rebuild of PARENT needs its declared parents instead, and pins the new files. - assert _parent_steps(PARENT.dataset_id, output, parents_from=connectors, force=True) == [_build(GRAND)] + assert _parent_steps(PARENT.dataset_id, target_registry, parents_from=connectors, force=True) == [_build(GRAND)] # A registry that holds the source fills the gap: the local PARENT is reused and GRAND is copied. origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - assert _parent_steps(PARENT.dataset_id, output, parents_from=[origin]) == [ + assert _parent_steps(PARENT.dataset_id, target_registry, parents_from=[origin]) == [ CopyStep(GRAND, origin, with_payload_sources=False) ] def test_a_damaged_parent_is_copied_again_but_never_built_over(tmp_path, monkeypatch, child_builds): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - output, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") + target_registry, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] - for registry in (output, origin): + for registry in (target_registry, origin): registry.put_dataset(parent) - (release_dir(output.root, PARENT) / "control.duckdb").write_bytes(b"cut short") + (release_dir(target_registry.root, PARENT) / "control.duckdb").write_bytes(b"cut short") with pytest.raises(TimeNetFileError, match=r"'control\.duckdb' has 9 bytes, expected [\d,]+ bytes\.$") as caught: - _parent_steps(CHILD.dataset_id, output, parents_from=[InstalledConnectors()]) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()]) assert caught.value.release == PARENT - assert _parent_steps(CHILD.dataset_id, output, parents_from=[origin, InstalledConnectors()]) == [ + assert _parent_steps(CHILD.dataset_id, target_registry, parents_from=[origin, InstalledConnectors()]) == [ CopyStep(PARENT, origin) ] - run_plan(plan_release_build(CHILD.dataset_id, store=output, parents_from=[origin]), output) - assert _parent_steps(CHILD.dataset_id, output) == [] + run_plan( + plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[origin]), target_registry + ) + assert _parent_steps(CHILD.dataset_id, target_registry) == [] assert child_builds == [(CHILD.dataset_id, False)] @@ -427,6 +437,10 @@ def __init__(self, root): def local_root(self): return self._local.root + @property + def location(self): + return self._local.location + def list_datasets(self): return self._local.list_datasets() @@ -452,10 +466,10 @@ def put_release(self, from_directory): def test_open_access_only_refuses_a_parent_to_build_whose_card_is_not_open(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) cards[PARENT.dataset_id] = cards[PARENT.dataset_id].model_copy(update=_GATED) - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") with pytest.raises(TimeNetAccessError) as caught: - _parent_steps(CHILD.dataset_id, store, parents_from=[InstalledConnectors()], open_access_only=True) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()], open_access_only=True) error = caught.value requirement = AccessRequirement(PARENT, Access.CREDENTIALED, _GATED["access_url"]) @@ -464,15 +478,15 @@ def test_open_access_only_refuses_a_parent_to_build_whose_card_is_not_open(tmp_p def test_open_access_only_names_the_access_page_of_a_held_parent_from_its_lineage(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") # The card of the held parent stays open. Its lineage holds GRAND, which is credentialed. held = TimeFDataset(metadata=cards[PARENT.dataset_id]) grand = make_dataset().metadata.model_copy(update=_GATED) held._import_state.lineage[str(GRAND)] = ParentDatasetCard(metadata=grand) - store.put_dataset(held) + target_registry.put_dataset(held) with pytest.raises(TimeNetAccessError) as caught: - _parent_steps(CHILD.dataset_id, store, parents_from=[InstalledConnectors()], open_access_only=True) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[InstalledConnectors()], open_access_only=True) assert (caught.value.dataset, caught.value.access) == (PARENT, Access.CREDENTIALED) assert caught.value.requirements == (AccessRequirement(GRAND, Access.CREDENTIALED, _GATED["access_url"]),) @@ -544,9 +558,12 @@ def test_every_build_entry_point_prepares_parents(tmp_path, monkeypatch, child_b def build(origin=None): # The client and timenet.build() build through the coordinator. - store = LocalRegistry(out) + target_registry = LocalRegistry(out) run_plan( - plan_release_build(CHILD.dataset_id, store=store, parents_from=[] if origin is None else [origin]), store + plan_release_build( + CHILD.dataset_id, target_registry=target_registry, parents_from=[] if origin is None else [origin] + ), + target_registry, ) with pytest.raises(TimeNetMissingParentError, match=r"parent release test/parent@1\.0\.0 of test/child@1\.0\.0"): @@ -565,14 +582,14 @@ def build(origin=None): def test_a_pinned_parent_build_hands_its_pins_to_the_isolated_child(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, parent={"parent": GRAND}) - output = LocalRegistry(tmp_path / "output") - _parent_without_its_source(output) - pins = output.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] + local_registry = LocalRegistry(tmp_path / "output") + _parent_without_its_source(local_registry) + pins = local_registry.get_manifest(PARENT.dataset_id).payload_sources[SourceKey(str(GRAND))] received, pin_files = [], [] def run_pipeline(connector, root, *, pinned_by, **_): received.append(pinned_by) - output.put_dataset(make_dataset(), pinned_by=pinned_by) + local_registry.put_dataset(make_dataset(), pinned_by=pinned_by) def child(command, env): # Stands in for the uv child: run the forwarded command line through the CLI in this process. @@ -586,15 +603,15 @@ def child(command, env): monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) monkeypatch.setattr(backend_module, "run_pipeline", run_pipeline) - # The output holds PARENT, so the build gets only GRAND, the payload source of PARENT. - result = runner.invoke(app, ["build", PARENT.dataset_id, "--local-registry", str(output.root)]) + # The local registry holds PARENT, so the build gets only GRAND, the payload source of PARENT. + result = runner.invoke(app, ["build", PARENT.dataset_id, "--local-registry", str(local_registry.root)]) assert result.exit_code == 0, result.output # The rebuild of GRAND gets the pins of PARENT, through a file that is gone afterwards. assert received == [{PARENT: pins}] assert len(pin_files) == 1 assert not pin_files[0].exists() - assert output.exists(GRAND.dataset_id, str(GRAND.version)) + assert local_registry.exists(GRAND.dataset_id, str(GRAND.version)) _COPY_HINT = ( @@ -607,7 +624,7 @@ def child(command, env): def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - store = LocalRegistry(tmp_path / "store") + local_registry = LocalRegistry(tmp_path / "store") options = [] if case == "missing": # No registry holds the parent, and no installed connector builds it. @@ -615,8 +632,8 @@ def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): if case == "missing-without-auto-build": options = ["--no-auto-build"] if case == "damaged": - store.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) - (release_dir(store.root, PARENT) / "control.duckdb").write_bytes(b"cut short") + local_registry.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) + (release_dir(local_registry.root, PARENT) / "control.duckdb").write_bytes(b"cut short") options = ["--remote-registry", str(tmp_path / "remote registry")] if case == "not-open": cards[CHILD.dataset_id] = cards[CHILD.dataset_id].model_copy(update=_GATED) @@ -629,11 +646,11 @@ def test_the_cli_hint_names_its_flags(tmp_path, monkeypatch, case): monkeypatch.setattr(client_module, "open_registry", lambda uri, **_: hosted) options = ["--no-auto-build"] - result = runner.invoke(app, ["build", CHILD.dataset_id, "--local-registry", str(store.root), *options]) + result = runner.invoke(app, ["build", CHILD.dataset_id, "--local-registry", str(local_registry.root), *options]) # The hint of a damaged copy repeats the registry flags of the command, quoted for the shell. remote = str(tmp_path / "remote registry") - flags = f" --local-registry {shlex.quote(str(store.root))} --remote-registry {shlex.quote(remote)}" + flags = f" --local-registry {shlex.quote(str(local_registry.root))} --remote-registry {shlex.quote(remote)}" hints = { "missing": (TimeNetMissingParentError, f". {_COPY_HINT[0].upper()}{_COPY_HINT[1:]}"), "missing-without-auto-build": ( @@ -663,7 +680,7 @@ def _publish_to(tmp_path, monkeypatch, cards) -> tuple[_Hosted, LocalRegistry, l """Make each ``--publish-to`` of the CLI open a registry that is not local, and record each isolated build. Returns: - The registry that ``--publish-to`` opens, the store, and the list of the builds. + The registry that ``--publish-to`` opens, the local registry, and the list of the builds. """ remote = _Hosted(tmp_path / "hosted") monkeypatch.setattr(cli_module, "open_registry", lambda url, **_: remote) @@ -681,25 +698,25 @@ def test_publish_to_refuses_a_credentialed_release_before_any_download(tmp_path, cards[gated_ref.dataset_id] = cards[gated_ref.dataset_id].model_copy(update=_GATED) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id])) - _, store, calls = _publish_to(tmp_path, monkeypatch, cards) + _, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) command = ["build", CHILD.dataset_id, "--publish-to", "s3://bucket/registry"] result = runner.invoke(app, [*command, "--remote-registry", str(origin.root), "--no-auto-build"]) - # The build into the store gets the access checks of the destination. + # The build into the local registry gets the access checks of the destination. assert isinstance(result.exception, TimeNetAccessError), result.output assert result.exception.dataset == gated_ref assert calls == [] - assert not store.exists(PARENT.dataset_id, str(PARENT.version)) + assert not local_registry.exists(PARENT.dataset_id, str(PARENT.version)) -def test_publish_to_builds_into_the_store_and_publishes(tmp_path, monkeypatch): +def test_publish_to_builds_into_the_local_registry_and_publishes(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) origin = LocalRegistry(tmp_path / "origin") parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] origin.put_dataset(parent) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) command = ["build", CHILD.dataset_id, "--publish-to", "s3://bucket/registry"] command += ["--remote-registry", str(origin.root), "--no-auto-build"] @@ -707,24 +724,24 @@ def test_publish_to_builds_into_the_store_and_publishes(tmp_path, monkeypatch): assert result.exit_code == 0, result.output assert result.stdout == f"{CHILD.version}\n" - # The parent lands in the store first, and the child builds there. Then the publish takes the + # The parent lands in the local registry first, and the child builds there. Then the publish takes the # child and its payload source, the parent, to the destination. assert calls == [(CHILD.dataset_id, False)] - for registry in (store, remote): + for registry in (local_registry, remote): assert registry.exists(CHILD.dataset_id, str(CHILD.version)) assert registry.exists(PARENT.dataset_id, str(PARENT.version)) # A destination that holds the version gets nothing, and nothing builds. - shutil.rmtree(store.root) + shutil.rmtree(local_registry.root) result = runner.invoke(app, command) assert (result.exit_code, result.stdout) == (0, f"{CHILD.version}\n") assert result.stderr.splitlines() == [f"✅ Nothing to do: 'test/child' 1.0.0 is already in {remote}"] assert calls == [(CHILD.dataset_id, False)] - assert not store.root.exists() + assert not local_registry.root.exists() def test_publish_to_copies_parents_only_from_the_sources(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] remote.put_dataset(parent) @@ -741,7 +758,7 @@ def test_publish_to_copies_parents_only_from_the_sources(tmp_path, monkeypatch): result = runner.invoke(app, [*command, "--remote-registry", str(remote.local_root)]) assert result.exit_code == 0, result.output assert calls == [(CHILD.dataset_id, False)] - assert store.exists(PARENT.dataset_id, str(PARENT.version)) + assert local_registry.exists(PARENT.dataset_id, str(PARENT.version)) assert remote.exists(CHILD.dataset_id, str(CHILD.version)) @@ -757,7 +774,7 @@ def test_publish_to_a_local_directory_publishes_there(tmp_path, monkeypatch, chi assert result.exit_code == 0, result.output assert result.stdout == f"{CHILD.version}\n" assert child_builds == [(PARENT.dataset_id, False), (CHILD.dataset_id, False)] - # The build lands in the store, and the publish copies the child with its payload source. + # The build lands in the local registry, and the publish copies the child with its payload source. for registry in (default_local_registry(), published): assert registry.exists(CHILD.dataset_id, str(CHILD.version)) assert registry.exists(PARENT.dataset_id, str(PARENT.version)) @@ -813,7 +830,7 @@ def test_force_with_publish_to_does_nothing_for_the_same_files(tmp_path, monkeyp _store_from_card(CHILD.dataset_id, saved.root, force=False) def copy_the_saved_release(release, store_root, *, force=False, **_): - # Each build puts the same release directory into the store, so both builds write the same files. + # Each build puts the same release directory into the local registry, so both builds write the same files. calls.append((release.dataset_id, force)) LocalRegistry(store_root).put_release(release_dir(saved.root, CHILD)) @@ -831,61 +848,65 @@ def copy_the_saved_release(release, store_root, *, force=False, **_): assert _files(remote.local_root) == published -def test_the_plan_drops_the_store_from_parents_from(tmp_path, monkeypatch): +def test_the_plan_drops_the_target_registry_from_parents_from(tmp_path, monkeypatch): _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") - # The list holds the store itself and another registry over its directory. The plan drops both, - # so the error names the store once. + # The list holds the target registry itself and another registry over its directory. The plan drops both, + # so the error names the target registry once. with pytest.raises(TimeNetMissingParentError) as missing: - _parent_steps(CHILD.dataset_id, store, parents_from=[store, LocalRegistry(store.root)]) + _parent_steps( + CHILD.dataset_id, target_registry, parents_from=[target_registry, LocalRegistry(target_registry.root)] + ) - assert missing.value.searched == str(store) + assert missing.value.searched == str(target_registry) def test_a_parent_that_a_registry_lists_without_serving_is_built_by_the_connectors(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) hosted = _Hosted(tmp_path / "hosted") hosted.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id].model_copy(update=_GATED))) - output = LocalRegistry(tmp_path / "out") + target_registry = LocalRegistry(tmp_path / "out") - steps = _parent_steps(CHILD.dataset_id, output, parents_from=[hosted, InstalledConnectors()]) + steps = _parent_steps(CHILD.dataset_id, target_registry, parents_from=[hosted, InstalledConnectors()]) assert steps == [_build(PARENT)] # Without the connectors, the plan names the access page of the release. with pytest.raises(TimeNetAccessError) as caught: - _parent_steps(CHILD.dataset_id, output, parents_from=[hosted]) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[hosted]) assert caught.value.requirements == (AccessRequirement(PARENT, Access.CREDENTIALED, _GATED["access_url"]),) @pytest.mark.parametrize("answer", ["not-found", "not-served"]) -def test_a_damaged_copy_in_the_output_stops_the_plan_if_no_registry_serves_the_release(tmp_path, monkeypatch, answer): +def test_a_damaged_copy_in_the_target_registry_stops_the_plan_if_no_registry_serves_the_release( + tmp_path, monkeypatch, answer +): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - output, hosted = LocalRegistry(tmp_path / "output"), _Hosted(tmp_path / "hosted") + target_registry, hosted = LocalRegistry(tmp_path / "output"), _Hosted(tmp_path / "hosted") parent = make_dataset() parent._metadata = cards[PARENT.dataset_id] - output.put_dataset(parent) - (release_dir(output.root, PARENT) / "control.duckdb").write_bytes(b"cut short") + target_registry.put_dataset(parent) + (release_dir(target_registry.root, PARENT) / "control.duckdb").write_bytes(b"cut short") if answer == "not-served": hosted.put_dataset(TimeFDataset(metadata=cards[PARENT.dataset_id].model_copy(update=_GATED))) # The damage, not the missing release or its access page, is what the caller must fix. with pytest.raises(TimeNetFileError) as caught: - _parent_steps(CHILD.dataset_id, output, parents_from=[hosted]) + _parent_steps(CHILD.dataset_id, target_registry, parents_from=[hosted]) assert caught.value.release == PARENT def test_the_requested_build_waits_for_a_payload_source_of_a_held_parent(tmp_path, monkeypatch): _install_cards(monkeypatch, child={"declared": DECLARED}) - output, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") + target_registry, origin = LocalRegistry(tmp_path / "output"), LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) _child_of(origin, SOURCE, GRAND, make_dataset().records[0].id) _child_of(origin, DECLARED, SOURCE, 2) - # The output holds DECLARED, but not SOURCE, whose files DECLARED reads. - shutil.copytree(release_dir(origin.root, DECLARED), release_dir(output.root, DECLARED)) + # The target registry holds DECLARED, but not SOURCE, whose files DECLARED reads. + shutil.copytree(release_dir(origin.root, DECLARED), release_dir(target_registry.root, DECLARED)) - plan = plan_release_build(CHILD.dataset_id, store=output, parents_from=[origin]) + plan = plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[origin]) # DECLARED has no step, so the build of CHILD waits for the step of SOURCE behind it. assert plan.steps == ( @@ -915,19 +936,19 @@ def test_a_plan_reads_each_manifest_once_from_each_registry(tmp_path, monkeypatc layout.put_dataset(make_dataset()) _child_of(layout, SOURCE, GRAND, make_dataset().records[0].id) _child_of(layout, DECLARED, SOURCE, 2) - hosted, output = _Hosted(layout.root), LocalRegistry(tmp_path / "output") - hosted_reads, output_reads = _count_reads(monkeypatch, hosted), _count_reads(monkeypatch, output) + hosted, target_registry = _Hosted(layout.root), LocalRegistry(tmp_path / "output") + hosted_reads, target_reads = _count_reads(monkeypatch, hosted), _count_reads(monkeypatch, target_registry) - plan = plan_release_build(CHILD.dataset_id, store=output, parents_from=[hosted]) + plan = plan_release_build(CHILD.dataset_id, target_registry=target_registry, parents_from=[hosted]) assert plan.steps[:-1] == tuple(CopyStep(reference, hosted) for reference in parents.values()) # The registry that is not local answers through its manifest, and the plan reads that manifest # only once. assert hosted_reads == {str(reference): 1 for reference in (DECLARED, SOURCE, GRAND)} - # The output is read for the requested release too, which it does not hold. - assert output_reads == {str(reference): 1 for reference in (CHILD, DECLARED, SOURCE)} + # The target registry is read for the requested release too, which it does not hold. + assert target_reads == {str(reference): 1 for reference in (CHILD, DECLARED, SOURCE)} hosted_reads.clear() - run_plan(plan, output) + run_plan(plan, target_registry) # Each copy reads the manifests of its release and of the payload sources that it brings once # more, from the registry as it is when the copy runs. SOURCE is in both copies. assert hosted_reads == {str(DECLARED): 1, str(SOURCE): 2, str(GRAND): 1} @@ -966,7 +987,7 @@ def test_plan_lines_name_read_steps(tmp_path): def test_dry_run_prints_the_plan_and_changes_nothing(tmp_path, monkeypatch): cards = _install_cards(monkeypatch, child={"parent": PARENT}, parent={}) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) result = runner.invoke(app, ["build", CHILD.dataset_id, "--dry-run", "--publish-to", "s3://bucket/registry"]) assert result.exit_code == 0, result.output assert result.stdout.splitlines() == [ @@ -974,7 +995,7 @@ def test_dry_run_prints_the_plan_and_changes_nothing(tmp_path, monkeypatch): "└── test/parent@1.0.0: build with installed connectors", f"then publish test/child@1.0.0 to {remote}", ] - assert calls == [] and not store.root.exists() and not remote.exists(CHILD.dataset_id, "1.0.0") + assert calls == [] and not local_registry.root.exists() and not remote.exists(CHILD.dataset_id, "1.0.0") def test_a_build_prints_the_plan_before_the_first_step(tmp_path, monkeypatch, child_builds): @@ -1009,21 +1030,21 @@ def test_quiet_hides_the_plan(tmp_path, monkeypatch, child_builds): assert child_builds == [(PARENT.dataset_id, False), (CHILD.dataset_id, False)] -@pytest.mark.parametrize("held_in", ["store", "destination", "store-only"]) +@pytest.mark.parametrize("held_in", ["local-registry", "destination", "local-registry-only"]) def test_a_dry_run_of_a_held_release_says_nothing_to_do(tmp_path, monkeypatch, held_in): cards = _install_cards(monkeypatch, child={}, parent={}) - remote, store, calls = _publish_to(tmp_path, monkeypatch, cards) + remote, local_registry, calls = _publish_to(tmp_path, monkeypatch, cards) # The line names a long path, so the console must not wrap it. monkeypatch.setattr(cli_module.console, "rich", Console(stderr=True, width=1000)) - held_by = remote if held_in == "destination" else store - _store_from_card(CHILD.dataset_id, held_by.root if held_by is store else remote.local_root, force=False) - options = [] if held_in == "store" else ["--publish-to", "s3://bucket/registry"] + held_by = remote if held_in == "destination" else local_registry + _store_from_card(CHILD.dataset_id, held_by.root if held_by is local_registry else remote.local_root, force=False) + options = [] if held_in == "local-registry" else ["--publish-to", "s3://bucket/registry"] result = runner.invoke(app, ["build", CHILD.dataset_id, "--dry-run", *options]) assert result.exit_code == 0, result.output - if held_in == "store-only": - # The store holds the release and the destination does not, so only the publish is left to do. + if held_in == "local-registry-only": + # The local registry holds the release and the destination does not, so only the publish is left to do. assert result.stdout.splitlines() == [f"then publish test/child@1.0.0 to {remote}"] assert result.stderr == "" else: diff --git a/packages/timenet/src/timenet/builders.py b/packages/timenet/src/timenet/builders.py index 8c0ad7de9..ae9984b1a 100644 --- a/packages/timenet/src/timenet/builders.py +++ b/packages/timenet/src/timenet/builders.py @@ -11,7 +11,7 @@ from importlib.metadata import EntryPoint, entry_points from typing import TYPE_CHECKING, Literal, Protocol -from timenet.errors import TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError +from timenet.errors import TimeNetBuildError, TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError from timenet.progress import ProgressCallback from timenet.types import DatasetMetadata, Version from timenet.types.references import DatasetRef @@ -112,10 +112,12 @@ class InstalledBuilder: Attributes: backend: The installed backend that builds the dataset. card: The dataset's connector card. + package: ``" "`` of the package that registers the backend. """ backend: BuilderBackend card: DatasetMetadata + package: str @property def release(self) -> DatasetRef: @@ -166,12 +168,13 @@ def installed_builder(dataset_id: str) -> InstalledBuilder | None: Raises: TimeNetInvalidCardError: If the dataset's card cannot be read. - """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend + TimeNetBuildError: If no installed distribution declares the entry point of the backend. + """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend and _package claim = next((claim for claim in claims() if claim.dataset_id == dataset_id), None) if claim is None: return None card = claim.backend.card(dataset_id) - return None if card is None else InstalledBuilder(claim.backend, card) + return None if card is None else InstalledBuilder(claim.backend, card, _package(claim)) def declared(dataset_id: str, version: Version | str | None = None) -> InstalledBuilder: @@ -213,3 +216,16 @@ def find_card(dataset_id: str) -> DatasetMetadata | None: """ # ruff: ignore[docstring-extraneous-exception] - raised by the backend found = installed_builder(dataset_id) return None if found is None else found.card + + +def _package(claim: ConnectorClaim) -> str: + """Return ``" "`` of the package whose entry point registers the backend of a claim. + + Raises: + TimeNetBuildError: If no installed distribution declares the entry point. + """ + dist = claim.entry.dist + if dist is None: + kind = type(claim.backend) + raise TimeNetBuildError(f"the entry point of {kind.__module__}.{kind.__qualname__} names no installed package") + return f"{dist.name} {dist.version}" diff --git a/packages/timenet/src/timenet/cache.py b/packages/timenet/src/timenet/cache.py index 71fcacd7e..3452b1e54 100644 --- a/packages/timenet/src/timenet/cache.py +++ b/packages/timenet/src/timenet/cache.py @@ -1,9 +1,9 @@ -"""Inspect and clear the cache and the stored releases. +"""Inspect and clear the cache and the downloads and builds in the local registry. Cache removal requires TimeNet's ``CACHEDIR.TAG`` marker. """ -from collections.abc import Iterable +from collections.abc import Collection, Iterable from dataclasses import dataclass from pathlib import Path @@ -12,16 +12,51 @@ from timenet.errors import TimeFValidationError from timenet.registry import LocalRegistry from timenet.registry._paths import release_dir +from timenet.registry.origin import Origin, OriginKind from timenet.types.references import DatasetRef @dataclass(frozen=True) class StoredRelease: - """A stored release with its path and size in bytes.""" + """A stored release with its path, provenance record, and size on disk.""" release: DatasetRef path: Path - size_bytes: int + origin: Origin | None + """The record of how the release reached the local registry, or ``None`` if it has no record.""" + + def size(self) -> int: + """Measure the size of the release on disk. + + Returns: + The size in bytes, measured on each call. + """ + return _dir_size(self.path) + + @property + def kind(self) -> OriginKind: + """The kind of the record, or :attr:`~OriginKind.BUILD` for a release without a record.""" + return OriginKind.BUILD if self.origin is None else self.origin.kind + + +@dataclass(frozen=True) +class KeptRelease: + """A release that a clear keeps, because a release that stays needs it.""" + + release: DatasetRef + """The release that stays.""" + needed_by: DatasetRef + """A release that stays and needs it, as a payload source or as the parent of a build.""" + + +@dataclass(frozen=True) +class ReleasesToClear: + """The releases that a clear deletes, and the releases of the same kinds that it keeps.""" + + releases: list[StoredRelease] + """The releases to delete, sorted by dataset ID and version.""" + kept: list[KeptRelease] + """The releases of the cleared kinds that stay, sorted by dataset ID and version.""" @dataclass(frozen=True) @@ -38,12 +73,14 @@ class ClearedCache: @dataclass(frozen=True) class ClearedReleases: - """The bytes freed and release folders removed by a release clear.""" + """The bytes freed, release folders removed and releases kept by a release clear.""" freed: int """The number of bytes freed.""" removed: list[Path] """The release folders successfully removed.""" + kept: list[KeptRelease] + """The releases of the cleared kinds that stay, because a release that stays needs them.""" def human_bytes(n: int) -> str: @@ -65,19 +102,79 @@ def human_bytes(n: int) -> str: def stored_releases(store: LocalRegistry) -> list[StoredRelease]: - """List the releases of a local registry and their sizes in bytes. + """List stored releases with their provenance records and sizes. Args: - store: The local registry to inspect. + store: The local registry. Returns: The releases, sorted by dataset ID and version. - """ - found: list[StoredRelease] = [] - for release in store.releases(): - path = release_dir(store.root, release) - found.append(StoredRelease(release, path, _dir_size(path))) - return found + + Raises: + TimeFFormatError: If a provenance record is invalid. + """ # ruff: ignore[docstring-extraneous-exception] (raised by read_origin) + return [ + StoredRelease(release, release_dir(store.root, release), store.read_origin(release)) + for release in store.releases() + ] + + +def stored_releases_by_kind(store: LocalRegistry) -> dict[OriginKind, list[StoredRelease]]: + """List stored releases with their provenance records and sizes, by kind. + + Args: + store: The local registry. + + Returns: + The releases of each kind, sorted by dataset ID and version, and an empty list for a kind without releases. + + Raises: + TimeFFormatError: If a provenance record is invalid. + """ # ruff: ignore[docstring-extraneous-exception] (raised by read_origin) + by_kind: dict[OriginKind, list[StoredRelease]] = {kind: [] for kind in OriginKind} + for item in stored_releases(store): + by_kind[item.kind].append(item) + return by_kind + + +def releases_to_clear(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> ReleasesToClear: + """Find the releases of the given kinds that no release that stays needs. + + A release that stays keeps its payload sources, and a build that stays also keeps its parents. This holds + for each release that stays, also for one that stays only because another one needs it. + + Args: + store: The local registry. + kinds: The kinds of release to delete. + + Returns: + The releases to delete, and the releases of those kinds that stay with a release that needs each. + + Raises: + TimeFValidationError: If ``kinds`` is empty, or a dataset or release folder in the local registry is a symlink. + TimeFFormatError: If a provenance record or the manifest of a release that stays is damaged. + OSError: If a folder of the local registry cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by check_no_linked_folders and stored_releases) + _check_kinds(kinds) + store.check_no_linked_folders() + releases = stored_releases(store) + held = {item.release: item for item in releases} + # Walk from each release that stays to the releases that it needs. If every kind goes, nothing stays. + staying = [item.release for item in releases if item.kind not in kinds] + seen = set(staying) + needed_by: dict[DatasetRef, DatasetRef] = {} + while staying: + release = staying.pop() + manifest = store.get_manifest(release.dataset_id, str(release.version)) + for need in manifest.required_releases(with_parents=held[release].kind is OriginKind.BUILD): + if need in held and need not in seen: + seen.add(need) + needed_by[need] = release + staying.append(need) + return ReleasesToClear( + [item for item in releases if item.kind in kinds and item.release not in needed_by], + [KeptRelease(item.release, needed_by[item.release]) for item in releases if item.release in needed_by], + ) def cache_size(cache_dir: Path) -> int: @@ -135,27 +232,43 @@ def clear_cache(cache_dir: Path, *, local_registry: LocalRegistry) -> ClearedCac return ClearedCache(freed, removed, [folder for folder in folders if folder not in marked]) -def clear_releases(store: LocalRegistry) -> ClearedReleases: - """Delete stored releases and the dataset directories left empty. +def clear_releases(store: LocalRegistry, *, kinds: Collection[OriginKind]) -> ClearedReleases: + """Delete selected releases unless remaining releases need them. - Other files in the local registry stay. + Each dataset folder that this leaves empty goes too, and other files in the local registry stay. Args: store: The local registry. + kinds: The kinds of release to delete. Returns: - The bytes freed and the release folders removed. + The bytes freed, the release folders removed, and the releases kept with a release that needs each. Raises: - TimeNetRegistryError: If the local registry is read-only. - OSError: If a folder cannot be removed completely. - """ # ruff: ignore[docstring-extraneous-exception] (raised by remove_release) + TimeNetRegistryError: Before any deletion, if the local registry is read-only. + TimeFValidationError: Before any deletion, if ``kinds`` is empty or a dataset or release folder is a symlink. + TimeFFormatError: Before any deletion, if a provenance record or a kept release's manifest is damaged. + OSError: If a folder cannot be read or removed completely. + """ # ruff: ignore[docstring-extraneous-exception] (raised by releases_to_clear and remove_release) + _check_kinds(kinds) + store.check_writable("a clear of its releases") + found = releases_to_clear(store, kinds=kinds) freed = 0 removed: list[Path] = [] - for release in store.releases(): - freed += store.remove_release(release) - removed.append(release_dir(store.root, release)) - return ClearedReleases(freed, removed) + for item in found.releases: + freed += store.remove_release(item.release) + removed.append(item.path) + return ClearedReleases(freed, removed, found.kept) + + +def _check_kinds(kinds: Collection[OriginKind]) -> None: + """Refuse a clear that names no kind of release. + + Raises: + TimeFValidationError: If ``kinds`` is empty. + """ + if not kinds: + raise TimeFValidationError("kinds names no kind of release") def _check_cache_deletion(cache_dir: Path, folders: Iterable[Path], *, local_registry: Path) -> None: diff --git a/packages/timenet/src/timenet/cli/app.py b/packages/timenet/src/timenet/cli/app.py index 4d31eaca9..3d9af8c59 100644 --- a/packages/timenet/src/timenet/cli/app.py +++ b/packages/timenet/src/timenet/cli/app.py @@ -12,7 +12,14 @@ from rich.table import Table import typer -from timenet.cache import cache_size, clear_cache, clear_releases, human_bytes, stored_releases +from timenet.cache import ( + cache_size, + clear_cache, + clear_releases, + human_bytes, + releases_to_clear, + stored_releases_by_kind, +) from timenet.cli.options import AUTO_BUILD, LOCAL_REGISTRY, REMOTE_REGISTRY from timenet.cli.runner import run_cli from timenet.cli.ui import console as ui, progress_renderer @@ -22,6 +29,7 @@ from timenet.producer import publish as publish_release from timenet.progress import ProgressCallback, ProgressEvent, StepStarted from timenet.registry import open_registry +from timenet.registry.origin import OriginKind from timenet.types import DatasetMetadata, Domain, License, TaskType from timenet.types.access import access_requirements, effective_access from timenet.types.parent_dataset_card import ParentDatasetCard @@ -344,30 +352,44 @@ def publish( no_args_is_help=True, ) local_app.add_typer(cache_app, name="cache") +downloads_app = typer.Typer(help="The releases in the local registry that came from a registry.", no_args_is_help=True) +local_app.add_typer(downloads_app, name="downloads") +builds_app = typer.Typer( + help="The releases in the local registry that a connector built. A release without a record counts as a build.", + no_args_is_help=True, +) +local_app.add_typer(builds_app, name="builds") _YES = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt.") +# The words of each kind in the output: the noun, and the title of its list. +_KIND_WORDS = {OriginKind.DOWNLOAD: ("download", "Downloads"), OriginKind.BUILD: ("build", "Builds")} @local_app.command("info") def local_info(local_registry: str | None = LOCAL_REGISTRY) -> None: - """Show cache and local registry paths and sizes.""" + """Show paths and sizes for the cache, downloads, and builds.""" store, cache_dir = default_local_registry(local_registry=local_registry), settings().cache_dir - releases = stored_releases(store) - size = human_bytes(sum(item.size_bytes for item in releases)) + by_kind = stored_releases_by_kind(store) # Plain lines, so that a long path does not wrap. typer.echo(_cache_line(cache_dir)) - typer.echo(f"Local registry {store.root}: {_count(len(releases), 'release')}, {size}") + for kind, of_kind in by_kind.items(): + title = _KIND_WORDS[kind][1] + size = human_bytes(sum(item.size() for item in of_kind)) + typer.echo(f"{title} in {store.root}: {_count(len(of_kind), 'release')}, {size}") @local_app.command("clear") def local_clear(yes: bool = _YES, local_registry: str | None = LOCAL_REGISTRY) -> None: - """Delete the cache and every release in the local registry.""" + """Delete the cache, every download and every build.""" store, cache_dir = default_local_registry(local_registry=local_registry), settings().cache_dir if not yes: - typer.confirm(f"Delete the cache in {cache_dir} and every release in {store.root}?", abort=True) + builds = _count(len(stored_releases_by_kind(store)[OriginKind.BUILD]), "build") + typer.confirm( + f"Delete the cache in {cache_dir} and every release in {store.root}, {builds} included?", abort=True + ) # The cache clear checks the folders before it deletes anything, so it runs first. cache = clear_cache(cache_dir, local_registry=store) - releases = clear_releases(store) + releases = clear_releases(store, kinds=tuple(OriginKind)) _warn_unmarked(cache.kept) _freed(releases.freed + cache.freed, len(releases.removed) + len(cache.removed)) @@ -389,6 +411,65 @@ def cache_clear(yes: bool = _YES) -> None: _freed(cleared.freed, len(cleared.removed)) +@downloads_app.command("info") +def downloads_info(local_registry: str | None = LOCAL_REGISTRY) -> None: + """Show each download in the local registry with its registry and size.""" + _show_releases(OriginKind.DOWNLOAD, local_registry) + + +@downloads_app.command("clear") +def downloads_clear(yes: bool = _YES, local_registry: str | None = LOCAL_REGISTRY) -> None: + """Delete the downloads in the local registry, except those that a remaining release needs.""" + _clear_releases(OriginKind.DOWNLOAD, yes, local_registry) + + +@builds_app.command("info") +def builds_info(local_registry: str | None = LOCAL_REGISTRY) -> None: + """Show each build in the local registry with its connector package and size.""" + _show_releases(OriginKind.BUILD, local_registry) + + +@builds_app.command("clear") +def builds_clear(yes: bool = _YES, local_registry: str | None = LOCAL_REGISTRY) -> None: + """Delete the builds in the local registry, except those that a remaining release needs.""" + _clear_releases(OriginKind.BUILD, yes, local_registry) + + +def _show_releases(kind: OriginKind, local_registry: str | None) -> None: + """Show releases of one kind in a table, or a message if none exist.""" + store = default_local_registry(local_registry=local_registry) + releases = stored_releases_by_kind(store)[kind] + noun, title = _KIND_WORDS[kind] + if not releases: + typer.echo(f"No {noun}s in {store.root}.") + return + table = Table(title=f"{title} in {store.root}", title_justify="left", title_style="bold") + table.add_column("Dataset", style="cyan") + table.add_column("Version") + table.add_column("Source", overflow="fold") + table.add_column("Size", justify="right") + sizes = [item.size() for item in releases] # each release folder is measured once + for item, size in zip(releases, sizes, strict=True): + # A build without a record has no source. + source = "-" if item.origin is None else item.origin.source + table.add_row(item.release.dataset_id, str(item.release.version), source, human_bytes(size)) + table.add_section() + table.add_row("total", "", "", human_bytes(sum(sizes)), style="bold") + console.print(table) + + +def _clear_releases(kind: OriginKind, yes: bool, local_registry: str | None) -> None: + """Delete the releases of one kind after a prompt that counts them, and name each release that stays.""" + store = default_local_registry(local_registry=local_registry) + if not yes: + releases = releases_to_clear(store, kinds=[kind]).releases + typer.confirm(f"Delete {_count(len(releases), _KIND_WORDS[kind][0])} from {store.root}?", abort=True) + cleared = clear_releases(store, kinds=[kind]) + for item in cleared.kept: + ui.warn(f"Kept {item.release}: {item.needed_by} needs it.") + _freed(cleared.freed, len(cleared.removed)) + + def _cache_line(cache_dir: Path) -> str: """Return the path and the size of the cache.""" return f"Cache {cache_dir}: {human_bytes(cache_size(cache_dir))}" diff --git a/packages/timenet/src/timenet/client.py b/packages/timenet/src/timenet/client.py index 4818af2ab..56c94244c 100644 --- a/packages/timenet/src/timenet/client.py +++ b/packages/timenet/src/timenet/client.py @@ -105,9 +105,9 @@ def default_sources( TimeNetRegistryError: If ``local_registry`` names a remote registry, or the remote registry cannot be opened. """ # ruff: ignore[docstring-extraneous-exception] (raised by settings and open_registry) cfg = settings(local_registry=local_registry, remote_registry=remote_registry, auto_build=auto_build) - store = LocalRegistry(cfg.local_registry_dir) + local = LocalRegistry(cfg.local_registry_dir) remote = open_registry(cfg.remote_registry, label="TIMENET_REMOTE_REGISTRY") - return [store, remote, *([InstalledConnectors()] if cfg.auto_build else [])] + return [local, remote, *([InstalledConnectors()] if cfg.auto_build else [])] def _resolve_ref(dataset_id: str, version: str | None) -> tuple[str, str | None]: @@ -190,10 +190,10 @@ def __init__(self, sources: Sequence[ReleaseSource | str | Path] | None = None) self._sources = tuple(as_release_sources(sources, keyword="sources")) if not self._sources: raise TimeFValidationError("sources is empty. Pass at least one source, or None for the default list") - self._store = next( + self._local_registry = next( (source for source in self._sources if isinstance(source, LocalRegistry) and not source.read_only), None ) - if self._store is None and any(isinstance(source, InstalledConnectors) for source in self._sources): + if self._local_registry is None and any(isinstance(source, InstalledConnectors) for source in self._sources): raise TimeFValidationError( "InstalledConnectors() needs a writable local registry to build into. " "Add a LocalRegistry without read_only=True to the sources" @@ -352,35 +352,34 @@ def download( force: bool = False, progress: ProgressCallback | None = None, ) -> Path: - """Copy a dataset release and its payload sources into the local registry. + """Copy a release and its payload sources into the local registry. - The release resolves as in :meth:`open`, without a build. The release and each payload source that the - local registry lacks or holds damaged come from the first registry other than the local registry that - provides them. + The release resolves as in :meth:`open`, without a build. Missing or damaged releases come from the first + other registry that provides them, with a provenance record that names it. Args: - dataset_id: The dataset ID, optionally followed by ``@version``. + dataset_id: The dataset ID. version: An exact version, ``"latest"``, or ``None``, as in :meth:`open`. force: Copy the release again, even if the local registry holds a complete copy. - progress: A callback for copy events and byte counts. + progress: Receives step events and byte counts for each copied release. Returns: The release directory in the local registry. Raises: - TimeFValidationError: If no source is a writable local registry. - TimeNetAccessError: If a required release is only listed or requires access in a remote registry. - TimeNetNotBuiltError: If only an installed connector provides the release. + TimeFValidationError: If the sources have no writable local registry. + TimeNetAccessError: If a required release is only listed or lacks open access in a remote registry. + TimeNetNotBuiltError: If only installed connectors provide the release. TimeNetDatasetNotFoundError: If no registry other than the local registry provides the release. TimeNetVersionUnavailableError: If the version is missing and its connector builds another version. TimeNetMissingParentError: If no registry provides a missing payload source. TimeNetVersionConflictError: If the local registry holds the same release with different files. - TimeNetFileError: If no registry can replace a damaged copy, or a copied file is damaged. - TimeFFormatError: If a payload source differs from its recorded files or the source links form a cycle. + TimeNetFileError: If a stored copy cannot be repaired or a copied file is damaged. + TimeFFormatError: If a payload source breaks a pin or the payload sources form a cycle. """ # ruff: ignore[docstring-extraneous-exception] (raised by the plan and the copies) dataset_id, version = _resolve_ref(dataset_id, version) - store = self._store - if store is None: + local_registry = self._local_registry + if local_registry is None: # Without a local registry, a download has no place to go. raise TimeFValidationError( "download() needs a writable local registry to copy into. " @@ -404,13 +403,13 @@ def download( raise TimeNetDatasetNotFoundError(f"{error} Add a registry that holds it to the sources.") from error release = DatasetRef(dataset_id=dataset_id, version=answer.version) try: - plan = plan_download(release, store, sources=self._sources, force=force, lookups=lookups) + plan = plan_download(release, local_registry, sources=self._sources, force=force, lookups=lookups) except TimeNetFileError as error: # The local registry holds a damaged copy that no other registry provides. error.hint = self._file_hint(error) raise - run_plan(plan, store, options=BuildOptions(progress=progress)) - return release_dir(store.root, release) + run_plan(plan, local_registry, options=BuildOptions(progress=progress)) + return release_dir(local_registry.root, release) def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: """Open a dataset version for lazy reads, where it is. @@ -465,7 +464,9 @@ def open(self, dataset_id: str, version: str | None = None) -> TimeFReader: """ # ruff: ignore[docstring-extraneous-exception] (raised by read_release and the builder) dataset_id, version = _resolve_ref(dataset_id, version) with self._hints(dataset_id): - return TimeFReader(read_release(dataset_id, version, sources=self._sources, store=self._store)) + return TimeFReader( + read_release(dataset_id, version, sources=self._sources, target_registry=self._local_registry) + ) def load(self, dataset_id: str, version: str | None = None) -> TimeFDataset: """Read a whole dataset version into memory, with every stored value loaded. @@ -570,20 +571,20 @@ def _file_hint(self, error: TimeNetFileError) -> str | None: Returns: A repair hint, or ``None`` if the local registry has no damaged copy or the lookup raises a TimeNet error. """ - store = self._store - if store is None: + local_registry = self._local_registry + if local_registry is None: return None damaged = error.release try: - held = store.find_manifest(damaged) - if held is None or store.damage(damaged, held) is None: + held = local_registry.find_manifest(damaged) + if held is None or local_registry.damage(damaged, held) is None: # The local registry holds no damaged copy, so the error is about another registry. return None - found = _copy_source(self._sources, store, damaged) + found = _copy_source(self._sources, local_registry, damaged) except TimeNetError: # The hint must not hide the error that it explains, for example if a registry is offline. return None - return _repair_hint(damaged, found, store) + return _repair_hint(damaged, found, local_registry) def open_torch( self, dataset_id: str, version: str | None = None, *, load: LoadPlan = LoadPlan.ALL_EAGER @@ -643,39 +644,41 @@ def _access_hint(dataset_id: str, sources: Sequence[ReleaseSource]) -> str: return f"No source serves its files. Add InstalledConnectors() to the sources, or build it locally with {build}." -def _copy_source(sources: Sequence[ReleaseSource], store: LocalRegistry, release: DatasetRef) -> Found | None: +def _copy_source(sources: Sequence[ReleaseSource], local_registry: LocalRegistry, release: DatasetRef) -> Found | None: """Find the answer of a release source outside the local registry. Args: sources: The sources to search, in order. - store: The local registry to exclude. + local_registry: The local registry to exclude. release: The dataset ID and exact version. Returns: The answer of the source that provides or builds the release, or ``None`` if none exists. """ - others = [source for source in sources if not is_same_registry(source, store)] + others = [source for source in sources if not is_same_registry(source, local_registry)] answer = resolve_release(others, release.dataset_id, str(release.version)) return answer if isinstance(answer, Found) else None -def _repair_hint(release: DatasetRef, found: Found | None, store: LocalRegistry) -> str: +def _repair_hint(release: DatasetRef, found: Found | None, local_registry: LocalRegistry) -> str: """Suggest how to replace a damaged local release. Args: release: The damaged release. found: The answer of a source for the release, or ``None``. - store: The local registry that holds the damaged release. + local_registry: The local registry that holds the damaged release. Returns: A download or rebuild hint, or an explanation that no replacement is available. """ - if isinstance(found, FoundInRegistry) and not is_same_registry(found.registry, store): + if isinstance(found, FoundInRegistry) and not is_same_registry(found.registry, local_registry): # A registry other than the local registry provides the release. return f"Copy an intact release into the local registry with download('{release}')." if isinstance(found, FoundInConnector): # Only an installed connector provides the release. - command = f"timenet-build build {release.dataset_id} --local-registry {shlex.quote(str(store.root))} --force" + command = ( + f"timenet-build build {release.dataset_id} --local-registry {shlex.quote(str(local_registry.root))} --force" + ) return f"Build it again with `{command}`." # Only the local registry holds the release. return "No registry other than the local registry provides it, so download() cannot replace it." diff --git a/packages/timenet/src/timenet/coordinator/build.py b/packages/timenet/src/timenet/coordinator/build.py index d13243ef1..43a2df98d 100644 --- a/packages/timenet/src/timenet/coordinator/build.py +++ b/packages/timenet/src/timenet/coordinator/build.py @@ -9,6 +9,7 @@ from timenet.registry import copy_release from timenet.registry.dependencies import prepare_local_copy from timenet.registry.local import LocalRegistry +from timenet.registry.origin import OriginKind from timenet.sources import ReleaseSource from timenet.types.references import is_latest, split_ref @@ -16,7 +17,7 @@ def plan_release_build( dataset_id: str, *, - store: LocalRegistry, + target_registry: LocalRegistry, parents_from: Sequence[ReleaseSource] = (), force: bool = False, open_access_only: bool = False, @@ -28,7 +29,7 @@ def plan_release_build( Args: dataset_id: A dataset ID such as ``org/name``, or an exact release such as ``org/name@1.2.0``. - store: The local registry to write to. + target_registry: The local registry to write to. parents_from: Sources to copy or build missing dependencies from, in preference order. force: Plan a forced rebuild and its parents even if the requested release already exists. open_access_only: Require ``access: open`` for every needed release. @@ -45,7 +46,7 @@ def plan_release_build( _, steps = plan_steps( # A connector builds only the version of its card, so "latest" asks for that version. declared(dataset_id, None if is_latest(version) else version), - store, + target_registry, parents_from=parents_from, force_build=force, open_access_only=open_access_only, @@ -53,24 +54,24 @@ def plan_release_build( return Plan(tuple(steps)) -def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = None) -> None: - """Run the copy and build steps of a plan in order. +def run_plan(plan: Plan, target_registry: LocalRegistry, *, options: BuildOptions | None = None) -> None: + """Run a plan's copy and build steps in order. - Skip read steps. + Each copied or built release gets a record of its source. Args: - plan: The steps to run. - store: The local registry to copy and build into. - options: Build settings, including the progress callback. + plan: The plan to run. + target_registry: The local registry to copy and build into. + options: Build settings, such as the progress callback. Raises: - TimeNetRegistryError: If the plan copies or builds a release and ``store`` is read-only. - TimeNetVersionConflictError: If a copied release conflicts with the files in ``store``. - TimeFFormatError: If a build changes files pinned by other releases. + TimeNetRegistryError: If the plan copies or builds a release and the local registry is read-only. + TimeNetVersionConflictError: If the local registry holds the same release with different files. + TimeFFormatError: If a build changes files that other releases pin. TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the copy and the backends if any(not isinstance(step, ReadStep) for step in plan.steps): - store.check_writable("the copies and builds of a plan") + target_registry.check_writable("the copies and builds of a plan") options = options or BuildOptions() for step in plan.steps: match step: @@ -78,35 +79,40 @@ def run_plan(plan: Plan, store: LocalRegistry, *, options: BuildOptions | None = # A read opens the release where it is, so it has nothing to run. pass case CopyStep(): - copy_release( + copies = copy_release( step.release, step.registry, - store, + target_registry, with_payload_sources=step.with_payload_sources, force=step.force, progress=options.progress, ) + for release in copies: + target_registry.record_origin(release, OriginKind.DOWNLOAD, step.registry.location) case BuildStep(): - _build(step, store, options=options) + _build(step, target_registry, options=options) case _: assert_never(step) -def _build(step: BuildStep, store: LocalRegistry, *, options: BuildOptions) -> None: - """Build an exact release with the builder of its step and check its required files. +def _build(step: BuildStep, target_registry: LocalRegistry, *, options: BuildOptions) -> None: + """Build the requested release with the builder of its step and record the builder's package. Args: step: The build step, with ``force`` set to rebuild an existing release. - store: The local registry to build into. + target_registry: The local registry to build into. options: The build settings. Raises: - TimeNetDatasetNotFoundError: If the result lacks the release or a payload source. + TimeNetDatasetNotFoundError: If the build omits the release or a payload source. """ # ruff: ignore[docstring-extraneous-exception] - raised by prepare_local_copy if options.progress is not None: options.progress(StepStarted(step.release, "build")) - step.builder.backend.build(step.release, store, force=step.force, pinned_by=step.pinned_by, options=options) + step.builder.backend.build( + step.release, target_registry, force=step.force, pinned_by=step.pinned_by, options=options + ) # After a successful build, the local registry must hold the exact release with its payload sources. - prepare_local_copy(store.root, step.release) + prepare_local_copy(target_registry.root, step.release) + target_registry.record_origin(step.release, OriginKind.BUILD, step.builder.package) if options.progress is not None: options.progress(StepFinished(step.release, "build")) diff --git a/packages/timenet/src/timenet/coordinator/plan.py b/packages/timenet/src/timenet/coordinator/plan.py index 69ef4bd56..f6918d353 100644 --- a/packages/timenet/src/timenet/coordinator/plan.py +++ b/packages/timenet/src/timenet/coordinator/plan.py @@ -117,10 +117,11 @@ class _Release: role: Role """The release's role as a dataset or a source of payload files.""" - needed_in_local_registry: bool - """True if the plan needs a complete copy in the output, False if a source that serves the release is enough.""" + needed_in_target: bool + """True if the plan needs a complete copy in the target registry, False if a source that serves the release is + enough.""" step: Step | None - """The step before dependencies are set, or ``None`` for a release that the output holds.""" + """The step before dependencies are set, or ``None`` for a release that the target registry holds.""" dependencies: tuple[DatasetRef, ...] """The parent or payload source releases that must be ready first.""" @@ -201,10 +202,10 @@ def _ask(self, source: ReleaseSource, dataset_id: str, version: Version | None) class _Graph: """A graph of required releases and their read, copy or build steps.""" - output: LocalRegistry | None + target_registry: LocalRegistry | None """The local registry for copies and builds, or ``None`` for a read that copies and builds nothing.""" sources: Sequence[ReleaseSource] - """The sources in preference order. A read reads the output like any other source.""" + """The sources in preference order. A read reads the target registry like any other source.""" open_only: bool """True if each release that the walk visits must declare ``access: open``.""" builds: bool = True @@ -220,9 +221,13 @@ class _Graph: @cached_property def build_sources(self) -> list[ReleaseSource]: - """The sources without the output, which is never a source of its own copies and builds.""" - output = self.output - return [source for source in self.sources if output is None or not is_same_registry(source, output)] + """The sources without the target registry, which is never a source of its own copies and builds.""" + target_registry = self.target_registry + return [ + source + for source in self.sources + if target_registry is None or not is_same_registry(source, target_registry) + ] def visit( # ruff: ignore[too-many-arguments] self, @@ -230,22 +235,22 @@ def visit( # ruff: ignore[too-many-arguments] needed_by: DatasetRef | None, pinned: PayloadSource | None = None, *, - needed_in_local_registry: bool, + needed_in_target: bool, answer: ReleaseLookup | None = None, force: bool = False, ) -> None: """Add a required release and its dependencies to the graph. A node is planned again when a later visit needs more of it: a dataset instead of a payload source, or a - copy in the output instead of a read. + copy in the target registry instead of a read. Args: reference: The release to add. needed_by: The release that requires ``reference``, or ``None`` for the requested release. pinned: Required file records, or ``None`` for a build parent. - needed_in_local_registry: True if the plan needs a complete copy in the output. + needed_in_target: True if the plan needs a complete copy in the target registry. answer: The answer of the sources for ``reference``, or ``None`` to ask them. - force: Copy the release again, even if the output holds a complete copy. + force: Copy the release again, even if the target registry holds a complete copy. """ role: Role = "dataset" if pinned is None else "payload-source" if pinned is not None and needed_by is not None: @@ -253,10 +258,10 @@ def visit( # ruff: ignore[too-many-arguments] found = self.releases.get(reference) if found is not None: role = "dataset" if "dataset" in {found.role, role} else "payload-source" - needed_in_local_registry = found.needed_in_local_registry or needed_in_local_registry - if (role, needed_in_local_registry) == (found.role, found.needed_in_local_registry): + needed_in_target = found.needed_in_target or needed_in_target + if (role, needed_in_target) == (found.role, found.needed_in_target): return - if needed_in_local_registry: + if needed_in_target: self.local(reference, needed_by, role, force=force) else: self.read(reference, needed_by, role, answer) @@ -273,9 +278,9 @@ def read( answer: The answer of the sources for ``reference``, or ``None`` to ask them. Raises: - TimeNetFileError: If a source other than a registry provides the release and the output holds a + TimeNetFileError: If a source other than a registry provides the release and the target registry holds a damaged copy. - TimeFValidationError: If only installed connectors provide the release and the plan has no output. + TimeFValidationError: If only installed connectors provide the release and the plan has no target registry. TimeNetDatasetNotFoundError: If the registry of the answer no longer holds the release. TimeNetAccessError: If a source lists the release but no source serves it. TimeNetMissingParentError: If no source provides the payload source. @@ -291,14 +296,14 @@ def read( step = ReadStep(reference, answer.registry) self.releases[reference] = _Release(role, False, step, tuple(source.release for source in sources)) for source in sources: - self.visit(source.release, reference, source, needed_in_local_registry=False) + self.visit(source.release, reference, source, needed_in_target=False) return - # Nothing but a registry replaces a damaged copy in the output. - damage = self.in_output(reference)[1] + # Nothing but a registry replaces a damaged copy in the target registry. + damage = self.in_target(reference)[1] if damage is not None: raise damage if isinstance(answer, FoundInConnector): - if self.output is None: + if self.target_registry is None: raise TimeFValidationError( f"{reference} needs a build by {InstalledConnectors()}, but the read has no local registry to " "build into" @@ -310,34 +315,34 @@ def read( raise self.missing(reference, needed_by, role, self.sources) def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, *, force: bool = False) -> None: - """Plan a complete copy of a release in the output, from the output itself, a copy or a build. + """Plan a complete copy of a release in the target registry, from the target registry itself, a copy or a build. Args: reference: The exact release. needed_by: The release that needs ``reference``, or ``None`` for the requested release. role: Whether the release is a dataset or a payload source. - force: Copy the release again, even if the output holds a complete copy. + force: Copy the release again, even if the target registry holds a complete copy. Raises: - TimeFValidationError: If the plan has no output. + TimeFValidationError: If the plan has no target registry. TimeNetNotBuiltError: If the plan builds nothing and only installed connectors provide the release. TimeNetDatasetNotFoundError: If no source provides the requested release, or ``force`` is set and only - the output holds it. - TimeNetFileError: If the output holds a damaged copy that no other registry provides. + the target registry holds it. + TimeNetFileError: If the target registry holds a damaged copy that no other registry provides. TimeNetAccessError: If a source lists the release but no source serves it. TimeNetMissingParentError: If no source provides the release. """ # ruff: ignore[docstring-extraneous-exception] - the file, access and missing errors are raised as values - output = self.output - if output is None: + target_registry = self.target_registry + if target_registry is None: raise TimeFValidationError(f"{reference} must be in a local registry, but the read has none") - held, damage = self.in_output(reference) + held, damage = self.in_target(reference) if held is not None and not force: if self.open_only: require_open_access(held.metadata, held.lineage) sources = list(held.payload_sources.values()) if role == "dataset" else [] self.releases[reference] = _Release(role, True, None, tuple(source.release for source in sources)) for source in sources: - self.visit(source.release, reference, source, needed_in_local_registry=True) + self.visit(source.release, reference, source, needed_in_target=True) return answer = self.lookups.resolve(self.build_sources, reference.dataset_id, str(reference.version)) if isinstance(answer, FoundInRegistry): @@ -346,7 +351,7 @@ def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, # A copy from another registry repairs a damaged copy. Nothing else replaces it. raise damage elif held is not None: - # With force, the copy needs a registry other than the output, and none provides the release. + # With force, the copy needs a registry other than the target registry, and none provides the release. raise TimeNetDatasetNotFoundError( f"No registry other than the local registry provides {reference}, so download(force=True) cannot copy " "it again." @@ -359,20 +364,20 @@ def local(self, reference: DatasetRef, needed_by: DatasetRef | None, role: Role, elif isinstance(answer, NotServed): raise answer.access_error(reference.dataset_id) else: - # No source provides the release, or only the output does. - raise self.missing(reference, needed_by, role, (output, *self.build_sources)) + # No source provides the release, or only the target registry does. + raise self.missing(reference, needed_by, role, (target_registry, *self.build_sources)) def copy(self, reference: DatasetRef, role: Role, registry: BaseRegistry, *, force: bool = False) -> None: """Add a copy step and plan the payload sources that the copy does not bring. - A dataset copy brings each payload source that ``registry`` serves. Each other payload source that the output - lacks gets its own step first. + A dataset copy brings each payload source that ``registry`` serves. Each other payload source that the target + registry lacks gets its own step first. Args: reference: The exact release to copy. role: Whether the release is a dataset or a payload source. registry: The registry to copy from. - force: Copy the release again, even if the output holds a complete copy. + force: Copy the release again, even if the target registry holds a complete copy. Raises: TimeNetDatasetNotFoundError: If the registry lacks the release. @@ -390,7 +395,7 @@ def copy(self, reference: DatasetRef, role: Role, registry: BaseRegistry, *, for manifest, registry, find=lambda source: self.listed(registry, source), - held=lambda source: self.in_output(source)[0], + held=lambda source: self.in_target(source)[0], ) if self.open_only: for found in (*sources.fetched.values(), *sources.held.values()): @@ -400,7 +405,7 @@ def copy(self, reference: DatasetRef, role: Role, registry: BaseRegistry, *, for # that depends on it closes a cycle and does not look it up again. self.releases[reference] = _Release(role, True, step, tuple(sources.missing)) for source, pinned in sources.missing.items(): - self.visit(source, reference, pinned, needed_in_local_registry=True) + self.visit(source, reference, pinned, needed_in_target=True) def build(self, reference: DatasetRef, role: Role, builder: InstalledBuilder) -> None: """Add a build step and plan the parents declared in its card. @@ -417,7 +422,7 @@ def build(self, reference: DatasetRef, role: Role, builder: InstalledBuilder) -> self.releases[reference] = _Release(role, True, BuildStep(reference, builder), parents) # A build opens each of its parents with the payload sources of that parent. for parent in parents: - self.visit(parent, reference, needed_in_local_registry=True) + self.visit(parent, reference, needed_in_target=True) @staticmethod def missing( @@ -450,22 +455,22 @@ def listed(self, registry: BaseRegistry, reference: DatasetRef) -> Manifest | No return None return self.lookups.manifest(registry, reference) - def in_output(self, reference: DatasetRef) -> tuple[Manifest | None, TimeNetFileError | None]: - """Find a release in the output and check local files for damage. + def in_target(self, reference: DatasetRef) -> tuple[Manifest | None, TimeNetFileError | None]: + """Find a release in the target registry and check local files for damage. Missing files and incorrect file sizes count as damage in a local registry. Returns: ``(manifest, None)`` if found, ``(None, error)`` if damaged, or ``(None, None)`` if absent or if the - plan has no output. + plan has no target registry. """ - if self.output is None: + if self.target_registry is None: return None, None - manifest = self.lookups.manifest(self.output, reference) + manifest = self.lookups.manifest(self.target_registry, reference) if manifest is None: return manifest, None if reference not in self.damage: - self.damage[reference] = self.output.damage(reference, manifest) + self.damage[reference] = self.target_registry.damage(reference, manifest) damage = self.damage[reference] return (manifest, None) if damage is None else (None, damage) @@ -484,8 +489,9 @@ def check(self) -> None: # One build must match every reader, so the readers must pin each file alike. _check_agreement(reference, readers) continue - # A read or a copy uses the release in its registry. A release without a step is intact in the output. - registry = self.output if step is None else step.registry + # A read or a copy uses the release in its registry. A release without a step is intact in the target + # registry. + registry = self.target_registry if step is None else step.registry manifest = None if registry is None else self.lookups.manifest(registry, reference) if manifest is None: raise TimeNetDatasetNotFoundError(f"{registry} no longer holds {reference}") @@ -501,19 +507,19 @@ def open(self, reference: DatasetRef) -> DatasetVersion: """Open a release of the plan without its payload sources, after the steps of the plan ran. Returns: - The release in the registry of its read step, else in the output. + The release in the registry of its read step, else in the target registry. Raises: - TimeFValidationError: If the release has no read step and the plan has no output. + TimeFValidationError: If the release has no read step and the plan has no target registry. """ step = self.releases[reference].step if isinstance(step, ReadStep): return step.registry.open_release(reference, manifest=self.lookups.manifest(step.registry, reference)) - if self.output is None: - # Without an output, the planner plans only read steps. So this is a bug of the planner. - raise TimeFValidationError(f"{reference} has no read step and the plan of the read has no output") - # The steps changed the output, so the release opens with a manifest that is read again. - return self.output.open_release(reference) + if self.target_registry is None: + # Without a target registry, the planner plans only read steps. So this is a bug of the planner. + raise TimeFValidationError(f"{reference} has no read step and the plan of the read has no target registry") + # The steps changed the target registry, so the release opens with a manifest that is read again. + return self.target_registry.open_release(reference) def order(self) -> list[DatasetRef]: """Order the releases so that each release comes after the releases that it depends on. @@ -589,7 +595,7 @@ def _check_agreement(release: DatasetRef, readers: Mapping[DatasetRef, PayloadSo def plan_steps( builder: InstalledBuilder, - output: LocalRegistry, + target_registry: LocalRegistry, *, parents_from: Sequence[ReleaseSource], force_build: bool, @@ -597,13 +603,13 @@ def plan_steps( ) -> tuple[DatasetRef, list[Step]]: """Plan the steps to build a dataset and its dependencies. - A damaged copy of the requested release in the output gets a forced build. + A damaged copy of the requested release in the target registry gets a forced build. Args: builder: The installed connector that builds the dataset. - output: The destination registry. + target_registry: The destination registry. parents_from: The sources to search for missing releases, in order. - force_build: Rebuild the requested release, even if the output holds it. + force_build: Rebuild the requested release, even if the target registry holds it. open_access_only: Require open access for every release. Returns: @@ -613,11 +619,11 @@ def plan_steps( if open_access_only: require_open_access(card) root = builder.release - # The graph leaves the output out of the sources of its copies and builds. - graph = _Graph(output, parents_from, open_only=open_access_only) - # If the output already holds the requested release, a reader needs only its payload sources to + # The graph leaves the target registry out of the sources of its copies and builds. + graph = _Graph(target_registry, parents_from, open_only=open_access_only) + # If the target registry already holds the requested release, a reader needs only its payload sources to # open it. Its declared parents matter only to a rebuild. - held, damage = (None, None) if force_build else graph.in_output(root) + held, damage = (None, None) if force_build else graph.in_target(root) if held is not None and open_access_only: # The installed card has no lineage, so the check above does not see an ancestor that is not open. require_open_access(held.metadata, held.lineage) @@ -627,19 +633,19 @@ def plan_steps( step = BuildStep(root, builder, force=force_build or damage is not None) graph.releases[root] = _Release("dataset", True, step, tuple(card.parents.values())) for parent in card.parents.values(): - graph.visit(parent, root, needed_in_local_registry=True) + graph.visit(parent, root, needed_in_target=True) else: sources = list(held.payload_sources.values()) graph.releases[root] = _Release("dataset", True, None, tuple(source.release for source in sources)) for source in sources: - graph.visit(source.release, root, source, needed_in_local_registry=True) + graph.visit(source.release, root, source, needed_in_target=True) graph.check() return root, graph.create_ordered_steps() def plan_download( release: DatasetRef, - store: LocalRegistry, + target_registry: LocalRegistry, *, sources: Sequence[ReleaseSource], force: bool = False, @@ -652,8 +658,8 @@ def plan_download( Args: release: The exact release. - store: The local registry to copy into. - sources: The sources to search in order, excluding registries at the directory of ``store``. + target_registry: The local registry to copy into. + sources: The sources to search in order, excluding registries at the directory of ``target_registry``. force: Copy the release again, even if the local registry holds a complete copy. lookups: Cached source answers and manifests, or ``None`` to query sources. @@ -668,9 +674,9 @@ def plan_download( TimeNetMissingParentError: If no registry provides a payload source that the local registry lacks. TimeFFormatError: If a payload source differs from the files that a release pins. """ # ruff: ignore[docstring-extraneous-exception] - raised by the walk and as values - others = [source for source in sources if not is_same_registry(source, store)] - graph = _Graph(store, others, open_only=False, builds=False, lookups=lookups or Lookups()) - graph.visit(release, None, needed_in_local_registry=True, force=force) + others = [source for source in sources if not is_same_registry(source, target_registry)] + graph = _Graph(target_registry, others, open_only=False, builds=False, lookups=lookups or Lookups()) + graph.visit(release, None, needed_in_target=True, force=force) graph.check() return Plan(tuple(graph.create_ordered_steps())) diff --git a/packages/timenet/src/timenet/coordinator/read.py b/packages/timenet/src/timenet/coordinator/read.py index f5dc30a11..ae55bf2ad 100644 --- a/packages/timenet/src/timenet/coordinator/read.py +++ b/packages/timenet/src/timenet/coordinator/read.py @@ -19,19 +19,19 @@ def plan_read( - dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], store: LocalRegistry | None + dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], target_registry: LocalRegistry | None ) -> Plan: """Plan and validate the steps needed to read a release. A release that a registry provides gets a ``"read"`` step, and a release that only installed connectors - provide gets the steps of its build into ``store``. This checks local files and payload source pins before + provide gets the steps of its build into ``target_registry``. This checks local files and payload source pins before any step runs. Args: dataset_id: The dataset ID. version: An exact version, ``"latest"``, or ``None``, as in :func:`~timenet.sources.resolve_release`. sources: Registries and ``InstalledConnectors()`` to search, in order. - store: The local registry for copies and builds, or ``None`` to copy and build nothing. + target_registry: The local registry for copies and builds, or ``None`` to copy and build nothing. Returns: The ordered plan, ending with the requested release's step if it has one. @@ -45,7 +45,7 @@ def plan_read( TimeFFormatError: If a payload source conflicts with a pin or two pins conflict. TimeFValidationError: If a build has no local registry to build into, or the releases form a cycle. """ # ruff: ignore[docstring-extraneous-exception] - raised by the planner and the checks - graph, _ = _walk(dataset_id, version, sources=sources, store=store) + graph, _ = _walk(dataset_id, version, sources=sources, target_registry=target_registry) return Plan(tuple(graph.create_ordered_steps())) @@ -54,7 +54,7 @@ def read_release( version: str | None, *, sources: Sequence[ReleaseSource], - store: LocalRegistry | None, + target_registry: LocalRegistry | None, ) -> DatasetVersion: """Open a release with its payload sources after completing required copies and builds. @@ -65,22 +65,22 @@ def read_release( dataset_id: The dataset ID. version: An exact version, ``"latest"``, or ``None``, as in :func:`~timenet.sources.resolve_release`. sources: Registries and ``InstalledConnectors()`` to search, in order. - store: The local registry for copies and builds, or ``None`` to copy and build nothing. + target_registry: The local registry for copies and builds, or ``None`` to copy and build nothing. Returns: The release with attached payload sources that match its file pins. Raises: TimeNetDatasetNotFoundError: If no source provides the release. - TimeNetVersionConflictError: If a copied release conflicts with the files in ``store``. + TimeNetVersionConflictError: If a copied release conflicts with the files in ``target_registry``. TimeFFormatError: If a build changes files pinned by other releases. TimeNetBuildError: If a build fails. """ # ruff: ignore[docstring-extraneous-exception] - raised by the plan, the run and the open - graph, release = _walk(dataset_id, version, sources=sources, store=store) + graph, release = _walk(dataset_id, version, sources=sources, target_registry=target_registry) plan = Plan(tuple(graph.create_ordered_steps())) if any(not isinstance(step, ReadStep) for step in plan.steps): # The plan copies and builds only into a local registry. - run_plan(plan, cast("LocalRegistry", store)) + run_plan(plan, cast("LocalRegistry", target_registry)) return attach_payload_sources( graph.open(release), graph.open, searched=lambda: describe_searched_sources(sources, release) ) @@ -112,7 +112,7 @@ def raise_not_found(dataset_id: str, version: str | None, sources: Sequence[Rele def _walk( - dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], store: LocalRegistry | None + dataset_id: str, version: str | None, *, sources: Sequence[ReleaseSource], target_registry: LocalRegistry | None ) -> tuple[_Graph, DatasetRef]: """Plan a read in one walk of the release graph, and check the plan. @@ -122,17 +122,17 @@ def _walk( Raises: TimeNetDatasetNotFoundError: If no source provides the release. """ # ruff: ignore[docstring-extraneous-exception] - raised by raise_not_found - graph = _Graph(store, sources, open_only=False) + graph = _Graph(target_registry, sources, open_only=False) answer = graph.lookups.resolve(sources, dataset_id, version) if isinstance(answer, NotFound): if not is_latest(version): # Nothing but a registry replaces a damaged copy in the local registry. - damage = graph.in_output(DatasetRef.parse(f"{dataset_id}@{version}"))[1] + damage = graph.in_target(DatasetRef.parse(f"{dataset_id}@{version}"))[1] if damage is not None: raise damage raise_not_found(dataset_id, version, sources) release = DatasetRef(dataset_id=dataset_id, version=answer.version) # The walk takes the answer, so that a read without an exact version does not ask again for the exact version. - graph.visit(release, None, needed_in_local_registry=False, answer=answer) + graph.visit(release, None, needed_in_target=False, answer=answer) graph.check() return graph, release diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index fb1137ff9..0d48886f9 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -11,10 +11,11 @@ from timenet.manifest.files import ManifestFiles from timenet.manifest.payload_source import PayloadSource from timenet.types import DatasetMetadata, DatasetSchema -from timenet.types._model import TimeFModel +from timenet.types._model import TimeFModel, _first_problem from timenet.types.metadata_fields import DatasetId from timenet.types.parent_dataset_card import ParentDatasetCard from timenet.types.reference_keys import ParentAlias, SourceKeyField +from timenet.types.references import DatasetRef from timenet.types.wire import ValueEncoding from timenet.values_backends import ValuesBackend @@ -85,6 +86,20 @@ def _one_parent_card_per_release(self) -> Self: ) return self + def required_releases(self, *, with_parents: bool) -> list[DatasetRef]: + """List the releases that this release needs: its payload sources, and its parents if asked. + + Args: + with_parents: Also list the declared parents. + + Returns: + The releases of the payload sources, then the parents. + """ + releases = [source.release for source in self.payload_sources.values()] + if with_parents: + releases.extend(self.metadata.parents.values()) + return releases + def to_dict(self) -> dict[str, Any]: """Serialize the manifest to a validated JSON-compatible dict with all keys present. @@ -135,11 +150,7 @@ def parse_manifest(text: str | bytes, where: str) -> Manifest: try: return Manifest.model_validate_json(text) except ValidationError as error: - first = error.errors()[0] - location = ".".join(str(part) for part in first["loc"]) - message = first["msg"].removeprefix("Value error, ") - detail = f"{location}: {message}" if location else message - raise TimeFFormatError(f"{where} is not a valid manifest: {detail}") from error + raise TimeFFormatError(f"{where} is not a valid manifest: {_first_problem(error)}") from error def read_manifest(path: Path) -> Manifest: diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index 1d57b575d..a1b060ed4 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -28,25 +28,28 @@ def copy_release( # ruff: ignore[too-many-arguments] with_payload_sources: bool = True, force: bool = False, progress: ProgressCallback | None = None, -) -> None: - """Copy a release and its required payload sources to a registry. +) -> tuple[DatasetRef, ...]: + """Copy a release and its payload sources to another registry. - Payload sources go first, a complete copy in ``dest`` is reused, and a damaged local copy is fetched again. - All checks run before the first copy, so a failed check copies nothing. + Copy payload sources first, reuse complete copies, and repair damaged local copies. + Check all releases before copying any files. Args: reference: The exact release to copy. source: The registry to copy from. dest: The registry to copy to. with_payload_sources: Also copy the payload sources that the manifest of the release lists. - force: Fetch ``reference`` again, even if ``dest`` has a complete local copy. - progress: Receives step events for each copied release and byte counts when ``dest`` is local. + force: Fetch ``reference`` again even if the local destination has a complete copy. + progress: Receives step events for each copy and byte counts for local destinations. + + Returns: + The copied or repaired releases, in copy order. Raises: TimeNetRegistryError: If ``dest`` is read-only. TimeFValidationError: If ``force`` is set and ``dest`` is not local. - TimeNetVersionConflictError: If ``dest`` holds a release of the copy with other files or another card. - TimeNetAccessError: If a release is not open and ``dest`` or ``source`` requires open access. + TimeNetVersionConflictError: If ``dest`` holds the same release with other files or another card. + TimeNetAccessError: If a release lacks open access and either registry requires it. TimeNetDatasetNotFoundError: If ``source`` lacks the release. TimeNetMissingParentError: If neither registry holds a complete payload source. TimeFFormatError: If a manifest names the wrong release or a pin fails. @@ -65,7 +68,7 @@ def copy_release( # ruff: ignore[too-many-arguments] ] _check_access(manifests, set(pending), source, dest) if not pending: - return + return () # Only a copy from a non-local source to a non-local dest downloads into this directory, each # pending release on its own. with tempfile.TemporaryDirectory(prefix="timenet-copy-", ignore_cleanup_errors=True) as staging: @@ -76,6 +79,7 @@ def copy_release( # ruff: ignore[too-many-arguments] _copy_one(ref, manifests[ref], source, dest, Path(staging), force=again, progress=progress) if progress is not None: progress(StepFinished(ref, "copy")) + return tuple(pending) def _copy_one( # ruff: ignore[too-many-arguments] diff --git a/packages/timenet/src/timenet/registry/base.py b/packages/timenet/src/timenet/registry/base.py index df1da31a6..870a89128 100644 --- a/packages/timenet/src/timenet/registry/base.py +++ b/packages/timenet/src/timenet/registry/base.py @@ -29,7 +29,7 @@ # Reports download progress: called with each file (or chunk) of bytes as it lands. A ``download_release`` # implementation calls it so a caller can render a progress bar; ``None`` downloads silently. -ProgressCallback = Callable[[int], None] +ByteCountCallback = Callable[[int], None] class BaseRegistry(ABC): @@ -51,6 +51,11 @@ def read_only(self) -> bool: """Whether this registry refuses every write, set with ``read_only=True``.""" return self._read_only + @property + @abstractmethod + def location(self) -> str: + """The registry's URL, path, or name.""" + def check_writable(self, write: str) -> None: """Refuse a write if this registry is read-only. @@ -178,7 +183,7 @@ def download_release( *, force: bool = False, manifest: Manifest | None = None, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> None: """Download one release's files to a local directory. diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index b84d62616..3f752bcdb 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -1,11 +1,15 @@ """A registry backed by a local ``///`` directory tree.""" from collections.abc import Callable, Mapping +from datetime import UTC, datetime +import os from pathlib import Path import shutil from typing import BinaryIO +import uuid import pyarrow.fs as pafs +from pydantic import ValidationError from timenet._fs import delete_and_measure, same_directory, staged_directory from timenet.dataset import TimeFDataset @@ -15,15 +19,21 @@ from timenet.registry._paths import release_dir from timenet.registry.base import BaseRegistry from timenet.registry.dependencies import check_file_sizes, validate_manifest_reference +from timenet.registry.origin import Origin, OriginKind, without_credentials from timenet.registry.version import DatasetVersion from timenet.registry.writable import WritableRegistry from timenet.sources import FoundInRegistry, NotFound, ReleaseLookup, ReleaseSource from timenet.types import DatasetMetadata, Version +from timenet.types._model import _first_problem from timenet.types.metadata_fields import validate_dataset_id from timenet.types.references import DatasetRef, is_latest from timenet.writer import TimeFWriter, WriteProgressEvent +_ORIGIN_FILE = ".origin.json" +"""The provenance record of a release, in the folder of the release.""" + + class LocalRegistry(WritableRegistry): """Read and write dataset versions in a local directory.""" @@ -43,6 +53,11 @@ def __init__(self, root: str | Path, *, read_only: bool = False) -> None: def __str__(self) -> str: return f"local registry {self._root}" + @property + def location(self) -> str: + """The registry's absolute root path.""" + return str(self._root.resolve()) + @property def root(self) -> Path: """The directory this registry reads from and writes to.""" @@ -70,26 +85,72 @@ def list_datasets(self) -> list[DatasetMetadata]: return [self.get_manifest(dataset_id, str(version)).metadata for dataset_id, version in latest.items()] def releases(self) -> list[DatasetRef]: - """List the committed releases at any depth below the root. + """List the committed releases below the root. - A release is a ``//manifest.json`` outside hidden folders and unfinished writes. + A release is a ``///manifest.json`` outside hidden folders and unfinished writes. Returns: The releases, sorted by dataset ID and version. - """ - if not self._root.is_dir(): - return [] + + Raises: + OSError: If a folder cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by _walk) + return self._walk()[0] + + def check_no_linked_folders(self) -> None: + """Check that no dataset or release folder is a symlink, which :meth:`releases` would skip. + + Raises: + TimeFValidationError: If a dataset or release folder is a symlink. + OSError: If a folder cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by _walk) + linked = self._walk()[1] + if linked: + raise TimeFValidationError( + f"{linked[0]} is a symlink, so the clear cannot see the releases behind it or the " + "releases that they need. Replace the link with its folder, then clear again." + ) + + def _walk(self) -> tuple[list[DatasetRef], list[Path]]: + """Walk the dataset and release folders once, without following a symlink. + + Returns: + The releases, sorted by dataset ID and version, and the linked dataset or release folders in walk order. + + Raises: + OSError: If a folder cannot be read. + """ # ruff: ignore[docstring-extraneous-exception] (raised by os.walk) found: list[DatasetRef] = [] - for manifest_path in self._root.rglob(MANIFEST_FILE): - parts = manifest_path.parent.relative_to(self._root).parts - # A release needs a dataset ID before its version, so a manifest at the root or one level below is none. - if not parts[:-1] or any(part.startswith(".") or ".tmp-" in part for part in parts): + linked: list[Path] = [] + if not self._root.is_dir(): + return found, linked + release_depth = 3 # // + + def fail(error: OSError) -> None: + raise error # os.walk otherwise skips a folder that it cannot read + + for parent, dirs, files in os.walk(self._root, onerror=fail): + folder = Path(parent) + parts = folder.relative_to(self._root).parts + if len(parts) == release_depth: + dirs.clear() # the walk stops at the releases + release = self._release_at_parts(parts) if MANIFEST_FILE in files else None + if release is not None: + found.append(release) continue - try: - found.append(DatasetRef.parse(f"{'/'.join(parts[:-1])}@{parts[-1]}")) - except TimeFValidationError: - continue # for example a directory whose name is not a version - return sorted(found, key=lambda release: (release.dataset_id, release.version)) + dirs[:] = sorted(name for name in dirs if not _is_hidden_or_staging(name)) + linked.extend(folder / name for name in dirs if (folder / name).is_symlink()) + return sorted(found, key=lambda release: (release.dataset_id, release.version)), linked + + def _release_at_parts(self, parts: tuple[str, ...]) -> DatasetRef | None: + """Return the release of a ``//`` folder, or ``None`` if the names are not one.""" + try: + release = DatasetRef.parse(f"{parts[0]}/{parts[1]}@{parts[2]}") + except TimeFValidationError: + return None # for example a directory whose name is not a version + if release_dir(self._root, release) != self._root.joinpath(*parts): + return None # for example 01.0.0, which parses as 1.0.0 + return release def remove_release(self, release: DatasetRef) -> int: """Delete a release and the dataset folders that this leaves empty. @@ -113,6 +174,54 @@ def remove_release(self, release: DatasetRef) -> int: parent = parent.parent return freed + def record_origin(self, release: DatasetRef, kind: OriginKind, source: str) -> None: + """Record how a stored release reached this registry, with the current time. + + The record replaces an earlier one. A URL in ``source`` loses its user info, query and fragment, so no + credential lands in the record. + + Args: + release: The release. + kind: How the release reached this registry. + source: The registry location, or the name and version of the backend package. + + Raises: + TimeNetDatasetNotFoundError: If this registry does not hold the release. + """ + if not self.exists(release.dataset_id, str(release.version)): + raise TimeNetDatasetNotFoundError(f"the local registry holds no {release}, so it gets no provenance record") + origin = Origin(kind=kind, source=without_credentials(source), at=datetime.now(UTC).replace(microsecond=0)) + directory = release_dir(self._root, release) + # Write next to the record, then rename, so a reader never sees half a record. + staging = directory / f"{_ORIGIN_FILE}.tmp-{uuid.uuid4().hex}" + try: + staging.write_text(origin.model_dump_json() + "\n") + staging.replace(directory / _ORIGIN_FILE) + finally: + staging.unlink(missing_ok=True) + + def read_origin(self, release: DatasetRef) -> Origin | None: + """Read the provenance record of a stored release. + + Args: + release: The release. + + Returns: + The record, or ``None`` if the release has none. + + Raises: + TimeFFormatError: If the record is invalid. + """ + path = release_dir(self._root, release) / _ORIGIN_FILE + try: + text = path.read_text() + except FileNotFoundError: + return None + try: + return Origin.model_validate_json(text) + except ValidationError as error: + raise TimeFFormatError(f"{path} is not a valid provenance record: {_first_problem(error)}.") from error + def get_manifest(self, dataset_id: str, version: str | None = None) -> Manifest: """Return a dataset's manifest (latest version if unspecified). @@ -368,6 +477,18 @@ def is_same_registry(source: ReleaseSource, registry: BaseRegistry) -> bool: ) +def _is_hidden_or_staging(name: str) -> bool: + """Check whether a folder name is hidden or names an unfinished write. + + Args: + name: The folder name. + + Returns: + ``True`` if the name starts with ``.`` or holds ``.tmp-``. + """ + return name.startswith(".") or ".tmp-" in name + + def _is_version(name: str) -> bool: """Return whether a directory name is a parseable version. diff --git a/packages/timenet/src/timenet/registry/origin.py b/packages/timenet/src/timenet/registry/origin.py new file mode 100644 index 000000000..315714c57 --- /dev/null +++ b/packages/timenet/src/timenet/registry/origin.py @@ -0,0 +1,57 @@ +"""The provenance record: where and when each release in the local registry was downloaded or built. + +:class:`~timenet.registry.LocalRegistry` writes and reads the record of each release. The record stays in the +local registry when the release is copied elsewhere. +""" + +from datetime import UTC, datetime +from enum import Enum +from urllib.parse import urlsplit, urlunsplit + +from pydantic import AwareDatetime, BaseModel, ConfigDict, field_validator + + +class OriginKind(Enum): + """How a release reached the local registry.""" + + DOWNLOAD = "download" + """A copy from a registry.""" + BUILD = "build" + """A build by an installed connector.""" + + +class Origin(BaseModel): + """The kind, source, and time of a stored release's download or build.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: OriginKind + """How the release reached the local registry.""" + source: str + """The registry location, or the name and version of the backend package.""" + at: AwareDatetime + """The time that the release reached the local registry, in UTC.""" + + @field_validator("at") + @classmethod + def _in_utc(cls, value: datetime) -> datetime: + """Return the time in UTC.""" + return value.astimezone(UTC) + + +def without_credentials(location: str) -> str: + """Remove user information, the query, and the fragment from a URL. + + Return strings unchanged unless they have both a URL scheme and a host. + + Args: + location: A URL, path, or name. + + Returns: + The URL without user information, query, or fragment, or the original non-URL string. + """ + parts = urlsplit(location) + if not (parts.scheme and parts.netloc): + return location + host = parts.netloc.rpartition("@")[2] + return urlunsplit((parts.scheme, host, parts.path, "", "")) diff --git a/packages/timenet/src/timenet/registry/remote/_download.py b/packages/timenet/src/timenet/registry/remote/_download.py index 093a7089b..572b5601b 100644 --- a/packages/timenet/src/timenet/registry/remote/_download.py +++ b/packages/timenet/src/timenet/registry/remote/_download.py @@ -21,7 +21,7 @@ from timenet.manifest import Manifest from timenet.manifest.files import FilePart from timenet.registry._paths import safe_version_path -from timenet.registry.base import ProgressCallback +from timenet.registry.base import ByteCountCallback from timenet.registry.remote._http import ( RegistryHttpClient, _download_path, @@ -45,7 +45,7 @@ def download_version_files( # ruff: ignore[too-many-arguments] *, force: bool = False, max_concurrency: int = 8, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> Path: """Download every file of a version into ``dest_dir`` atomically. @@ -79,7 +79,7 @@ async def _download_all( # ruff: ignore[too-many-arguments, too-many-positional files: list[FilePart], staging: Path, max_concurrency: int, - progress_cb: ProgressCallback | None, + progress_cb: ByteCountCallback | None, ) -> None: """Download all files concurrently into the staging directory. @@ -143,7 +143,7 @@ async def _stream_one( # ruff: ignore[too-many-arguments] *, release: DatasetRef, record: FilePart, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> None: """Stream one presigned URL to ``dest`` via a ``.part`` temp file, renamed on success. diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index e439268e3..74143b5eb 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -17,7 +17,7 @@ from timenet.format.constants import MANIFEST_FILE from timenet.manifest import Manifest, parse_manifest from timenet.registry.lazy_read import LazyReadRegistry -from timenet.registry.remote._download import ProgressCallback, download_version_files +from timenet.registry.remote._download import ByteCountCallback, download_version_files from timenet.registry.remote._http import RegistryHttpClient, _reaching from timenet.registry.remote.filesystem import RegistryFileSystem from timenet.registry.remote.models import ( @@ -58,6 +58,11 @@ def __init__( def __str__(self) -> str: return f"registry {self._base_url}" + @property + def location(self) -> str: + """The registry service's base URL.""" + return self._base_url + def list_datasets(self) -> list[DatasetMetadata]: """Return the latest-version metadata of every dataset, sorted by id. @@ -137,7 +142,7 @@ def download_release( *, force: bool = False, manifest: Manifest | None = None, - progress_cb: ProgressCallback | None = None, + progress_cb: ByteCountCallback | None = None, ) -> None: """Download one release's files to a local directory. diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index b9b227a14..41fc3bce4 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -62,6 +62,11 @@ def __init__(self, uri: str, *, read_only: bool = False) -> None: def __str__(self) -> str: return f"S3 registry {self._display_root()}" + @property + def location(self) -> str: + """The registry's root URL as ``s3://bucket/prefix``.""" + return self._display_root() + def list_datasets(self) -> list[DatasetMetadata]: """Unsupported: the S3 backend has no catalog. diff --git a/packages/timenet/src/timenet/testing/backends.py b/packages/timenet/src/timenet/testing/backends.py index 71f4ece70..a421b2b5a 100644 --- a/packages/timenet/src/timenet/testing/backends.py +++ b/packages/timenet/src/timenet/testing/backends.py @@ -1,7 +1,7 @@ """A fake set of installed builder backends.""" from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING from timenet import builders @@ -11,12 +11,20 @@ import pytest +@dataclass(frozen=True) +class _FakeDistribution: + """The distribution that declares each fake entry point.""" + + name: str = "timenet-testing" + version: str = "0" + + @dataclass(frozen=True) class _FakeEntryPoint: - """An entry point that no installed distribution declares.""" + """An entry point of the fake distribution ``timenet-testing 0``.""" backend: object - dist: None = None + dist: _FakeDistribution = field(default_factory=_FakeDistribution) def load(self) -> Callable[[], object]: """Return the factory of the backend.""" diff --git a/packages/timenet/src/timenet/types/_model.py b/packages/timenet/src/timenet/types/_model.py index 6157bec07..1963529da 100644 --- a/packages/timenet/src/timenet/types/_model.py +++ b/packages/timenet/src/timenet/types/_model.py @@ -1,8 +1,8 @@ -"""Shared configuration for immutable TimeF data models.""" +"""Shared configuration for immutable TimeF data models, and the text of their validation errors.""" from typing import Any -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, ValidationError class TimeFModel(BaseModel): @@ -24,3 +24,18 @@ def __reduce__(self) -> tuple[Any, tuple[Any, ...]]: The rebuild callable and its arguments: this class and the serialized fields. """ return type(self).model_validate, (self.model_dump(),) + + +def _first_problem(error: ValidationError) -> str: + """Describe the first problem of a validation error as ``: ``. + + Args: + error: The validation error. + + Returns: + The location and message of the first problem, or the message alone if it has no location. + """ + first = error.errors()[0] + location = ".".join(str(part) for part in first["loc"]) + message = first["msg"].removeprefix("Value error, ") + return f"{location}: {message}" if location else message diff --git a/packages/timenet/tests/_origin_record.py b/packages/timenet/tests/_origin_record.py new file mode 100644 index 000000000..f01dd783c --- /dev/null +++ b/packages/timenet/tests/_origin_record.py @@ -0,0 +1,13 @@ +"""The file of a release's provenance record, for tests that damage, delete or read it.""" + +from pathlib import Path + +from timenet.registry import LocalRegistry +from timenet.registry._paths import release_dir +from timenet.registry.local import _ORIGIN_FILE +from timenet.types.references import DatasetRef + + +def origin_file(store: LocalRegistry, release: DatasetRef) -> Path: + """Return the file of the provenance record of a release in a local registry.""" + return release_dir(store.root, release) / _ORIGIN_FILE diff --git a/packages/timenet/tests/coordinator/test_build.py b/packages/timenet/tests/coordinator/test_build.py index 4236a1a74..188b688f2 100644 --- a/packages/timenet/tests/coordinator/test_build.py +++ b/packages/timenet/tests/coordinator/test_build.py @@ -13,6 +13,7 @@ from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, open_local_version from timenet.registry._paths import release_dir +from timenet.registry.origin import OriginKind from timenet.sources import InstalledConnectors from timenet.testing import install_backends, make_dataset from timenet.types import Access @@ -79,7 +80,7 @@ def backends(monkeypatch): @pytest.fixture -def store(tmp_path): +def target_registry(tmp_path): return LocalRegistry(tmp_path / "store") @@ -120,26 +121,26 @@ def _store_child_without_parent(registry): return registry.get_manifest(CHILD_REF.dataset_id).payload_sources[SourceKey(str(PARENT_REF))] -def test_plan_release_build_rejects_a_version_the_connector_does_not_declare(store, fake): +def test_plan_release_build_rejects_a_version_the_connector_does_not_declare(target_registry, fake): with pytest.raises(TimeNetVersionUnavailableError) as caught: - plan_release_build("a/b@2.0.0", store=store) + plan_release_build("a/b@2.0.0", target_registry=target_registry) assert (caught.value.release, caught.value.declared) == (DatasetRef.parse("a/b@2.0.0"), REF_AB.version) def test_a_plan_that_builds_refuses_a_read_only_local_registry_before_any_step(tmp_path, fake): - store = LocalRegistry(tmp_path / "store", read_only=True) + target_registry = LocalRegistry(tmp_path / "store", read_only=True) with pytest.raises(TimeNetRegistryError, match=r"read_only=True, so it refuses the copies and builds of a plan\."): - run_plan(Plan((_build(REF_AB),)), store) + run_plan(Plan((_build(REF_AB),)), target_registry) assert fake.calls == [] -def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, store, fake_parent, fake_child): +def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, target_registry, fake_parent, fake_child): origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) events = [] - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[origin]) - run_plan(plan, store, options=BuildOptions(progress=events.append)) + plan = plan_release_build(CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin]) + run_plan(plan, target_registry, options=BuildOptions(progress=events.append)) assert plan.steps == ( CopyStep(PARENT_REF, origin), @@ -153,21 +154,21 @@ def test_a_build_copies_a_missing_parent_and_reports_each_step_once(tmp_path, st StepFinished(CHILD_REF, "build"), ] assert fake_parent.calls == [] - assert store.exists(PARENT_REF.dataset_id, str(PARENT_REF.version)) + assert target_registry.exists(PARENT_REF.dataset_id, str(PARENT_REF.version)) -def test_a_build_copies_a_release_whose_payload_source_only_the_store_holds( - tmp_path, store, fake_parent, fake_child, backends +def test_a_build_copies_a_release_whose_payload_source_only_the_target_registry_holds( + tmp_path, target_registry, fake_parent, fake_child, backends ): # The registry to copy from holds the child, but not the parent whose files the child reads. origin = LocalRegistry(tmp_path / "origin") _store_child_without_parent(origin) - store.put_dataset(make_dataset()) + target_registry.put_dataset(make_dataset()) fake_grandchild = _Backend(_grandchild_card()) backends.append(fake_grandchild) - plan = plan_release_build(GRANDCHILD_REF.dataset_id, store=store, parents_from=[origin]) - run_plan(plan, store) + plan = plan_release_build(GRANDCHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin]) + run_plan(plan, target_registry) assert plan.steps == ( CopyStep(CHILD_REF, origin), @@ -175,11 +176,11 @@ def test_a_build_copies_a_release_whose_payload_source_only_the_store_holds( ) assert fake_parent.calls == fake_child.calls == [] assert fake_grandchild.calls == [(GRANDCHILD_REF, False, {})] - assert store.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) + assert target_registry.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( - tmp_path, store, fake_parent, fake_child, backends + tmp_path, target_registry, fake_parent, fake_child, backends ): # The registry to copy from holds the child, but not the parent whose files the child reads. origin = LocalRegistry(tmp_path / "origin") @@ -187,8 +188,10 @@ def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( fake_grandchild = _Backend(_grandchild_card()) backends.append(fake_grandchild) - plan = plan_release_build(GRANDCHILD_REF.dataset_id, store=store, parents_from=[origin, InstalledConnectors()]) - run_plan(plan, store) + plan = plan_release_build( + GRANDCHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin, InstalledConnectors()] + ) + run_plan(plan, target_registry) assert plan.steps == ( _build(PARENT_REF, pinned_by={CHILD_REF: pins}), @@ -198,111 +201,146 @@ def test_a_build_builds_a_payload_source_of_a_release_that_it_copies( assert fake_parent.calls == [(PARENT_REF, False, {CHILD_REF: pins})] assert fake_child.calls == [] assert fake_grandchild.calls == [(GRANDCHILD_REF, False, {})] - assert store.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) - + assert target_registry.get_manifest(CHILD_REF.dataset_id) == origin.get_manifest(CHILD_REF.dataset_id) -def test_a_forced_build_forces_only_the_requested_release(store, fake_parent, fake_child): - store.put_dataset(_demo(_child_card())) - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()], force=True) - run_plan(plan, store) +def test_run_plan_records_each_copy_and_each_build(tmp_path, target_registry, backends): + # The registry to copy from holds the child and its payload source, the parent. + origin = LocalRegistry(tmp_path / "origin") + _store_child_without_parent(origin) + origin.put_dataset(make_dataset()) + backends.extend([_Backend(_child_card()), _Backend(_grandchild_card())]) + plan = plan_release_build(GRANDCHILD_REF.dataset_id, target_registry=target_registry, parents_from=[origin]) + assert [(step.action, step.release) for step in plan.steps] == [("copy", CHILD_REF), ("build", GRANDCHILD_REF)] + + run_plan(plan, target_registry) + + records = {} + for release in (PARENT_REF, CHILD_REF, GRANDCHILD_REF): + record = target_registry.read_origin(release) + records[release] = None if record is None else (record.kind, record.source) + # The copy of the child brings its payload source along, and both name the registry. The build + # names the fake distribution of the fake backend. + assert records == { + PARENT_REF: (OriginKind.DOWNLOAD, str(origin.root.resolve())), + CHILD_REF: (OriginKind.DOWNLOAD, str(origin.root.resolve())), + GRANDCHILD_REF: (OriginKind.BUILD, "timenet-testing 0"), + } + + +def test_a_forced_build_forces_only_the_requested_release(target_registry, fake_parent, fake_child): + target_registry.put_dataset(_demo(_child_card())) + + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()], force=True + ) + run_plan(plan, target_registry) assert plan.steps == (_build(PARENT_REF), _build(CHILD_REF, depends_on=(PARENT_REF,), force=True)) assert fake_parent.calls == [(PARENT_REF, False, {})] assert fake_child.calls == [(CHILD_REF, True, {})] -def test_a_build_takes_a_parent_that_the_store_holds_and_asks_a_later_registry_nothing( - tmp_path, store, fake_parent, fake_child +def test_a_build_takes_a_parent_that_the_target_registry_holds_and_asks_a_later_registry_nothing( + tmp_path, target_registry, fake_parent, fake_child ): - store.put_dataset(make_dataset()) + target_registry.put_dataset(make_dataset()) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) transport, requests = build_service_fake(origin.root) remote = RemoteRegistry("http://api.local", transport=transport) - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[store, remote]) - run_plan(plan, store) + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[target_registry, remote] + ) + run_plan(plan, target_registry) assert plan.steps == (_build(CHILD_REF),) assert requests == [] -def test_a_build_skips_a_release_the_store_holds(store, fake): - store.put_dataset(_demo(_card(REF_AB))) - assert plan_release_build("a/b", store=store).steps == () +def test_a_build_skips_a_release_the_target_registry_holds(target_registry, fake): + target_registry.put_dataset(_demo(_card(REF_AB))) + assert plan_release_build("a/b", target_registry=target_registry).steps == () -def test_a_damaged_copy_of_the_requested_release_gets_a_forced_build(store, fake): - store.put_dataset(_demo(_card(REF_AB))) - (release_dir(store.root, REF_AB) / "control.duckdb").write_bytes(b"cut short") +def test_a_damaged_copy_of_the_requested_release_gets_a_forced_build(target_registry, fake): + target_registry.put_dataset(_demo(_card(REF_AB))) + (release_dir(target_registry.root, REF_AB) / "control.duckdb").write_bytes(b"cut short") - plan = plan_release_build("a/b", store=store) + plan = plan_release_build("a/b", target_registry=target_registry) assert plan.steps == (_build(REF_AB, force=True),) - run_plan(plan, store) + run_plan(plan, target_registry) assert fake.calls == [(REF_AB, True, {})] - assert store.damage(REF_AB) is None + assert target_registry.damage(REF_AB) is None -def test_plan_release_build_with_open_access_only_refuses_a_credentialed_card(store, backends): +def test_plan_release_build_with_open_access_only_refuses_a_credentialed_card(target_registry, backends): fake_credentialed = _Backend(_card(REF_AB, **_GATED)) backends.append(fake_credentialed) with pytest.raises(TimeNetAccessError) as caught: - plan_release_build("a/b", store=store, open_access_only=True) + plan_release_build("a/b", target_registry=target_registry, open_access_only=True) assert caught.value.dataset == REF_AB - # Without the flag, the local store takes a release that is not open. - run_plan(plan_release_build("a/b", store=store), store) + # Without the flag, the target registry takes a release that is not open. + run_plan(plan_release_build("a/b", target_registry=target_registry), target_registry) assert fake_credentialed.calls == [(REF_AB, False, {})] -def test_plan_release_build_with_open_access_only_refuses_a_held_release_with_a_credentialed_ancestor(store, backends): - # The card of the child is open. The child keeps a Record of the open release a/b, which the store +def test_plan_release_build_with_open_access_only_refuses_a_held_release_with_a_credentialed_ancestor( + target_registry, backends +): + # The card of the child is open. The child keeps a Record of the open release a/b, which the target registry # lacks. It reads nothing from the credentialed parent, which is only in its lineage. child_card = _card(CHILD_REF, parents={ParentAlias("parent"): PARENT_REF, ParentAlias("open"): REF_AB}) other, child = _Backend(_card(REF_AB)), _Backend(child_card) backends.extend([other, child]) - store.put_dataset(_demo(make_dataset().metadata.model_copy(update=_GATED))) - store.put_dataset(_demo(_card(REF_AB))) + target_registry.put_dataset(_demo(make_dataset().metadata.model_copy(update=_GATED))) + target_registry.put_dataset(_demo(_card(REF_AB))) dataset = TimeFDataset(metadata=child_card) with ( - TimeFReader(open_local_version(release_dir(store.root, PARENT_REF))) as gated_reader, - TimeFReader(open_local_version(release_dir(store.root, REF_AB))) as open_reader, + TimeFReader(open_local_version(release_dir(target_registry.root, PARENT_REF))) as gated_reader, + TimeFReader(open_local_version(release_dir(target_registry.root, REF_AB))) as open_reader, ): dataset.import_from(gated_reader, alias=ParentAlias("parent")) dataset.import_from(open_reader, alias=ParentAlias("open"), records=[make_dataset().records[0].id]) - store.put_dataset(dataset) - shutil.rmtree(store.root / PARENT_REF.dataset_id) - shutil.rmtree(release_dir(store.root, REF_AB)) - held = store.get_manifest(CHILD_REF.dataset_id) + target_registry.put_dataset(dataset) + shutil.rmtree(target_registry.root / PARENT_REF.dataset_id) + shutil.rmtree(release_dir(target_registry.root, REF_AB)) + held = target_registry.get_manifest(CHILD_REF.dataset_id) assert [card.source for card in held.lineage] == [PARENT_REF, REF_AB] assert list(held.payload_sources) == [SourceKey(str(REF_AB))] with pytest.raises(TimeNetAccessError) as caught: plan_release_build( - CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()], open_access_only=True + CHILD_REF.dataset_id, + target_registry=target_registry, + parents_from=[InstalledConnectors()], + open_access_only=True, ) assert caught.value.dataset == CHILD_REF -def test_a_plan_runs_only_through_run_plan(store, fake_parent, fake_child): - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()]) +def test_a_plan_runs_only_through_run_plan(target_registry, fake_parent, fake_child): + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()] + ) assert plan.steps == ( _build(PARENT_REF), _build(CHILD_REF, depends_on=(PARENT_REF,)), ) - assert not store.root.exists() + assert not target_registry.root.exists() assert fake_parent.calls == fake_child.calls == [] - run_plan(plan, store) + run_plan(plan, target_registry) assert fake_parent.calls == [(PARENT_REF, False, {})] assert fake_child.calls == [(CHILD_REF, False, {})] -def test_a_plan_reads_the_card_of_each_connector_once(store, backends): +def test_a_plan_reads_the_card_of_each_connector_once(target_registry, backends): reads = [] class _Counted(_Backend): @@ -311,7 +349,9 @@ def card(self, dataset_id): return super().card(dataset_id) backends.extend([_Counted(make_dataset().metadata), _Counted(_child_card())]) - plan = plan_release_build(CHILD_REF.dataset_id, store=store, parents_from=[InstalledConnectors()]) - run_plan(plan, store) + plan = plan_release_build( + CHILD_REF.dataset_id, target_registry=target_registry, parents_from=[InstalledConnectors()] + ) + run_plan(plan, target_registry) assert reads == [CHILD_REF.dataset_id, PARENT_REF.dataset_id] - assert store.exists(CHILD_REF.dataset_id, str(CHILD_REF.version)) + assert target_registry.exists(CHILD_REF.dataset_id, str(CHILD_REF.version)) diff --git a/packages/timenet/tests/coordinator/test_read.py b/packages/timenet/tests/coordinator/test_read.py index fb3f8ef89..7da5b4bf0 100644 --- a/packages/timenet/tests/coordinator/test_read.py +++ b/packages/timenet/tests/coordinator/test_read.py @@ -95,11 +95,11 @@ def _install_demo_backend(monkeypatch, *cards) -> list: return calls -def _child_reading(store, *cards, child_release=CHILD): +def _child_reading(target_registry, *cards, child_release=CHILD): """Store a child, which reads one Record of the demo dataset under each card, without those releases. Args: - store: The registry to store the child in. + target_registry: The registry to store the child in. *cards: The cards of the releases that the child reads. child_release: The release of the child. @@ -112,13 +112,13 @@ def _child_reading(store, *cards, child_release=CHILD): } child = TimeFDataset(metadata=_card(child_release, parents=parents)) for (alias, parent), card in zip(parents.items(), cards, strict=True): - store.put_dataset(_demo(card)) - with TimeFReader(open_local_version(release_dir(store.root, parent))) as reader: + target_registry.put_dataset(_demo(card)) + with TimeFReader(open_local_version(release_dir(target_registry.root, parent))) as reader: child.import_from(reader, alias=alias, records=[make_dataset().records[0].id]) - store.put_dataset(child) + target_registry.put_dataset(child) for parent in parents.values(): - shutil.rmtree(release_dir(store.root, parent)) - manifest = store.get_manifest(child_release.dataset_id) + shutil.rmtree(release_dir(target_registry.root, parent)) + manifest = target_registry.get_manifest(child_release.dataset_id) return {pinned.release: pinned for pinned in manifest.payload_sources.values()} @@ -147,35 +147,40 @@ def counted(dataset_id, version=None): def test_a_read_reads_in_place_from_the_first_source_that_holds_the_release(tmp_path): - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - plan = plan_read(DEMO.dataset_id, None, sources=[store, origin], store=store) + plan = plan_read(DEMO.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) assert plan.steps == (ReadStep(DEMO, origin),) - version = read_release(DEMO.dataset_id, None, sources=[store, origin], store=store) + version = read_release(DEMO.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) assert Path(version.root).is_relative_to(origin.root) - assert not store.root.exists() + assert not target_registry.root.exists() def test_a_damaged_copy_fails_the_read_before_any_step_runs(tmp_path, monkeypatch): - store, _ = _child_of_the_demo(tmp_path / "store") + target_registry, _ = _child_of_the_demo(tmp_path / "store") # No registry holds DEMO, so the plan builds it before the read opens CHILD. - shutil.rmtree(release_dir(store.root, DEMO)) + shutil.rmtree(release_dir(target_registry.root, DEMO)) # Origin and the connector also provide CHILD, but the read stops at the damaged copy, which comes first. origin = LocalRegistry(tmp_path / "origin") - shutil.copytree(store.root, origin.root) - (release_dir(store.root, CHILD) / "control.duckdb").write_bytes(b"cut short") + shutil.copytree(target_registry.root, origin.root) + (release_dir(target_registry.root, CHILD) / "control.duckdb").write_bytes(b"cut short") calls = _install_demo_backend(monkeypatch, _card(CHILD, parents={ParentAlias("parent"): DEMO})) with pytest.raises(TimeNetFileError, match=r"^test/child@1\.0\.0: file 'control\.duckdb' has 9 bytes"): - read_release(CHILD.dataset_id, None, sources=[store, origin, InstalledConnectors()], store=store) + read_release( + CHILD.dataset_id, + None, + sources=[target_registry, origin, InstalledConnectors()], + target_registry=target_registry, + ) assert calls == [] @pytest.mark.parametrize("answer", ["build", "not-found", "not-served"]) -def test_a_damaged_copy_in_the_store_fails_a_read_that_no_registry_serves(tmp_path, monkeypatch, answer): - # The store is not in the list, so no registry answers for its damaged copy. - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) - (release_dir(store.root, DEMO) / "control.duckdb").write_bytes(b"cut short") +def test_a_damaged_copy_in_the_target_registry_fails_a_read_that_no_registry_serves(tmp_path, monkeypatch, answer): + # The target registry is not in the list, so no registry answers for its damaged copy. + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) + (release_dir(target_registry.root, DEMO) / "control.duckdb").write_bytes(b"cut short") _install_demo_backend(monkeypatch) sources: list[ReleaseSource] if answer == "build": @@ -186,70 +191,70 @@ def test_a_damaged_copy_in_the_store_fails_a_read_that_no_registry_serves(tmp_pa # The damage error comes before the access error of the registry that lists DEMO. sources = [_gated_remote(tmp_path)] with pytest.raises(TimeNetFileError, match=r"^timenet/hello-world@1\.0\.0: file 'control\.duckdb' has 9 bytes"): - plan_read(DEMO.dataset_id, "1.0.0", sources=sources, store=store) + plan_read(DEMO.dataset_id, "1.0.0", sources=sources, target_registry=target_registry) def test_a_payload_source_resolves_on_its_own(tmp_path): - store, _ = _child_of_the_demo(tmp_path / "store") - shutil.rmtree(release_dir(store.root, DEMO)) + target_registry, _ = _child_of_the_demo(tmp_path / "store") + shutil.rmtree(release_dir(target_registry.root, DEMO)) origin = LocalRegistry(tmp_path / "origin") origin.put_dataset(make_dataset()) - plan = plan_read(CHILD.dataset_id, None, sources=[store, origin], store=store) - assert plan.steps == (ReadStep(DEMO, origin), ReadStep(CHILD, store, depends_on=(DEMO,))) - version = read_release(CHILD.dataset_id, None, sources=[store, origin], store=store) - assert Path(version.root).is_relative_to(store.root) + plan = plan_read(CHILD.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) + assert plan.steps == (ReadStep(DEMO, origin), ReadStep(CHILD, target_registry, depends_on=(DEMO,))) + version = read_release(CHILD.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) + assert Path(version.root).is_relative_to(target_registry.root) assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(origin.root) - assert not store.exists(DEMO.dataset_id, str(DEMO.version)) + assert not target_registry.exists(DEMO.dataset_id, str(DEMO.version)) def test_a_payload_source_with_other_files_than_the_pins_fails_the_plan(tmp_path): - store, _ = _child_of_the_demo(tmp_path / "store") - shutil.rmtree(release_dir(store.root, DEMO)) + target_registry, _ = _child_of_the_demo(tmp_path / "store") + shutil.rmtree(release_dir(target_registry.root, DEMO)) origin = LocalRegistry(tmp_path / "origin") # The same release with its values in another format: CHILD pins files that it does not have. origin.put_dataset(make_dataset(), values_backend="zarr") with pytest.raises(TimeFFormatError, match=r"^payload source timenet/hello-world@1\.0\.0 differs"): - plan_read(CHILD.dataset_id, None, sources=[store, origin], store=store) + plan_read(CHILD.dataset_id, None, sources=[target_registry, origin], target_registry=target_registry) def test_a_missing_payload_source_builds_against_the_pins_of_the_release(tmp_path, monkeypatch): - store, _ = _child_of_the_demo(tmp_path / "store") - shutil.rmtree(release_dir(store.root, DEMO)) - pins = store.get_manifest(CHILD.dataset_id).payload_sources[SourceKey(str(DEMO))] + target_registry, _ = _child_of_the_demo(tmp_path / "store") + shutil.rmtree(release_dir(target_registry.root, DEMO)) + pins = target_registry.get_manifest(CHILD.dataset_id).payload_sources[SourceKey(str(DEMO))] calls = _install_demo_backend(monkeypatch) - sources = [store, InstalledConnectors()] - assert plan_read(CHILD.dataset_id, None, sources=sources, store=store).steps == ( + sources = [target_registry, InstalledConnectors()] + assert plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry).steps == ( _build(DEMO, pinned_by={CHILD: pins}), - ReadStep(CHILD, store, depends_on=(DEMO,)), + ReadStep(CHILD, target_registry, depends_on=(DEMO,)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(DEMO, False, {CHILD: pins})] - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) def test_a_read_builds_a_release_that_only_a_connector_provides(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) calls = _install_demo_backend(monkeypatch, _card(CHILD, parents={ParentAlias("parent"): DEMO})) - sources = [store, InstalledConnectors()] - # The build of CHILD reads DEMO, which the store holds, so DEMO gets no step. - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + sources = [target_registry, InstalledConnectors()] + # The build of CHILD reads DEMO, which the target registry holds, so DEMO gets no step. + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert plan.steps == (_build(CHILD),) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(CHILD, False, {})] - assert Path(version.root).is_relative_to(store.root) - # A payload source without a step opens from the store. - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + assert Path(version.root).is_relative_to(target_registry.root) + # A payload source without a step opens from the target registry. + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) @pytest.mark.parametrize(("version", "release"), [(None, "x/y"), ("1.0.0", "x/y@1.0.0")]) def test_a_release_that_no_source_provides_names_every_source(tmp_path, monkeypatch, version, release): - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") _install_demo_backend(monkeypatch) with pytest.raises(TimeNetDatasetNotFoundError) as caught: - plan_read("x/y", version, sources=[store, InstalledConnectors()], store=store) + plan_read("x/y", version, sources=[target_registry, InstalledConnectors()], target_registry=target_registry) assert type(caught.value) is TimeNetDatasetNotFoundError - assert str(caught.value) == f"{release} is in none of the sources: {store}, installed connectors." + assert str(caught.value) == f"{release} is in none of the sources: {target_registry}, installed connectors." def test_a_missing_payload_source_names_the_version_that_the_connector_builds(tmp_path, monkeypatch): @@ -257,58 +262,66 @@ def test_a_missing_payload_source_names_the_version_that_the_connector_builds(tm shutil.rmtree(release_dir(origin.root, DEMO)) _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) with pytest.raises(TimeNetMissingParentError) as caught: - plan_read(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], store=None) + plan_read(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], target_registry=None) assert str(caught.value) == ( f"payload source {DEMO} of {CHILD} is not in {origin} or installed connectors ({DEMO.dataset_id} builds 2.0.0)." ) -def test_pins_that_the_store_copy_of_a_payload_source_breaks_fail_the_read_before_any_step(tmp_path, monkeypatch): - # Origin holds CHILD, which reads OTHER and DEMO. Only the connectors provide them. The store +def test_pins_that_the_target_registry_copy_of_a_payload_source_breaks_fail_the_read_before_any_step( + tmp_path, monkeypatch +): + # Origin holds CHILD, which reads OTHER and DEMO. Only the connectors provide them. The target registry # holds DEMO with its values in another format, so the plan builds nothing that checks the pins # of CHILD on DEMO. The plan builds OTHER first. - origin, store = LocalRegistry(tmp_path / "origin"), LocalRegistry(tmp_path / "store") + origin, target_registry = LocalRegistry(tmp_path / "origin"), LocalRegistry(tmp_path / "store") _child_reading(origin, _card(OTHER), make_dataset().metadata) - store.put_dataset(make_dataset(), values_backend="zarr") + target_registry.put_dataset(make_dataset(), values_backend="zarr") calls = _install_demo_backend(monkeypatch, _card(OTHER)) with pytest.raises(TimeFFormatError, match=r"^payload source timenet/hello-world@1\.0\.0 differs"): - read_release(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], store=store) + read_release(CHILD.dataset_id, None, sources=[origin, InstalledConnectors()], target_registry=target_registry) assert calls == [] def test_a_release_that_a_source_lists_without_serving_raises_the_access_error(tmp_path): with pytest.raises(TimeNetAccessError) as caught: - read_release(DEMO.dataset_id, None, sources=[_gated_remote(tmp_path)], store=None) + read_release(DEMO.dataset_id, None, sources=[_gated_remote(tmp_path)], target_registry=None) assert caught.value.requirements == (AccessRequirement(DEMO, Access.CREDENTIALED, _GATED["access_url"]),) def test_an_exact_version_that_the_connector_does_not_declare_raises_version_unavailable(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") + target_registry = LocalRegistry(tmp_path / "store") _install_demo_backend(monkeypatch) with pytest.raises(TimeNetVersionUnavailableError) as caught: - read_release(DEMO.dataset_id, "2.0.0", sources=[store, InstalledConnectors()], store=store) + read_release( + DEMO.dataset_id, "2.0.0", sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) assert (caught.value.release, caught.value.declared) == (DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"), DEMO.version) def test_latest_without_the_connectors_in_the_list_reads_the_older_copy_and_builds_nothing(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) # The installed connector builds a newer version, but the list does not hold the connectors. calls = _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) - assert plan_read(DEMO.dataset_id, "latest", sources=[store], store=store).steps == (ReadStep(DEMO, store),) - version = read_release(DEMO.dataset_id, "latest", sources=[store], store=store) + assert plan_read(DEMO.dataset_id, "latest", sources=[target_registry], target_registry=target_registry).steps == ( + ReadStep(DEMO, target_registry), + ) + version = read_release(DEMO.dataset_id, "latest", sources=[target_registry], target_registry=target_registry) assert version.manifest.metadata.dataset_version == DEMO.version assert calls == [] def test_a_read_without_a_version_stops_at_the_first_source_that_serves_it(tmp_path, monkeypatch): - store = LocalRegistry(tmp_path / "store") - store.put_dataset(make_dataset()) + target_registry = LocalRegistry(tmp_path / "store") + target_registry.put_dataset(make_dataset()) # A later registry cannot be reached, and the installed connector builds a newer version. down = RemoteRegistry("http://down.local", transport=httpx.MockTransport(_offline)) calls = _install_demo_backend(monkeypatch, _card(DatasetRef.parse(f"{DEMO.dataset_id}@2.0.0"))) - version = read_release(DEMO.dataset_id, None, sources=[store, down, InstalledConnectors()], store=store) + version = read_release( + DEMO.dataset_id, None, sources=[target_registry, down, InstalledConnectors()], target_registry=target_registry + ) assert version.manifest.metadata.dataset_version == DEMO.version assert calls == [] @@ -323,7 +336,7 @@ def card(self, dataset_id): return super().card(dataset_id) install_backends(monkeypatch, _Counted((make_dataset().metadata,), [])) - plan_read(DEMO.dataset_id, None, sources=[InstalledConnectors()], store=LocalRegistry(tmp_path / "store")) + plan_read(DEMO.dataset_id, None, sources=[InstalledConnectors()], target_registry=LocalRegistry(tmp_path / "store")) assert reads == [DEMO.dataset_id] @@ -340,55 +353,62 @@ def card(self, dataset_id): return super().card(dataset_id) install_backends(monkeypatch, _Counted((make_dataset().metadata,), [])) - store, _ = _child_of_the_demo(tmp_path / "store") - read_release(CHILD.dataset_id, None, sources=[store, InstalledConnectors()], store=store) + target_registry, _ = _child_of_the_demo(tmp_path / "store") + read_release( + CHILD.dataset_id, None, sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) # Only the error for a missing payload source names the sources, with the versions that the connectors build. assert asked == [] -def test_a_read_that_needs_a_build_without_a_store_raises(monkeypatch): +def test_a_read_that_needs_a_build_without_a_target_registry_raises(monkeypatch): _install_demo_backend(monkeypatch) with pytest.raises(TimeFValidationError, match=r"timenet/hello-world@1\.0\.0 needs a build"): - read_release(DEMO.dataset_id, None, sources=[InstalledConnectors()], store=None) + read_release(DEMO.dataset_id, None, sources=[InstalledConnectors()], target_registry=None) @pytest.mark.parametrize("parent", [PARENT, DatasetRef.parse("x/parent@1.0.0")], ids=["build-first", "read-first"]) -def test_a_release_that_a_build_copies_is_read_from_the_store_and_appears_once(tmp_path, monkeypatch, parent): +def test_a_release_that_a_build_copies_is_read_from_the_target_registry_and_appears_once(tmp_path, monkeypatch, parent): # CHILD reads the parent and DEMO. Only the connectors build the parent, and its card declares - # DEMO, which origin holds. So the build of the parent copies DEMO into the store, and the read + # DEMO, which origin holds. So the build of the parent copies DEMO into the target registry, and the read # takes it there. The read plans the payload sources in the order of their names. parent_card = _card(parent, parents={ParentAlias("demo"): DEMO}) - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - pins = _child_reading(store, parent_card, make_dataset().metadata) + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + pins = _child_reading(target_registry, parent_card, make_dataset().metadata) origin.put_dataset(make_dataset()) calls = _install_demo_backend(monkeypatch, parent_card) - sources = [store, origin, InstalledConnectors()] + sources = [target_registry, origin, InstalledConnectors()] reads = _count_reads(monkeypatch, origin) - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) # The read and the build of the parent share what the plan read. assert reads == {str(DEMO): 1} assert plan.steps == ( CopyStep(DEMO, origin), _build(parent, depends_on=(DEMO,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=tuple(pins)), + ReadStep(CHILD, target_registry, depends_on=tuple(pins)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(parent, False, {CHILD: pins[parent]})] - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) def test_a_step_that_two_builds_need_appears_once(tmp_path, monkeypatch): # CHILD reads PARENT and OTHER. Only the connectors build them, and both of their cards declare # DEMO, which origin holds. So both builds copy DEMO first. cards = [_card(reference, parents={ParentAlias("demo"): DEMO}) for reference in (PARENT, OTHER)] - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - pins = _child_reading(store, *cards) + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + pins = _child_reading(target_registry, *cards) origin.put_dataset(make_dataset()) _install_demo_backend(monkeypatch, *cards) - plan = plan_read(CHILD.dataset_id, None, sources=[store, origin, InstalledConnectors()], store=store) + plan = plan_read( + CHILD.dataset_id, + None, + sources=[target_registry, origin, InstalledConnectors()], + target_registry=target_registry, + ) assert [(step.release, step.action) for step in plan.steps] == [ (DEMO, "copy"), @@ -405,46 +425,48 @@ def test_two_builds_of_one_release_merge_into_one_build_with_the_pins_of_each(tm # the parent builds OTHER without pins, and the read builds OTHER with the pins of CHILD. The read # plans the payload sources in the order of their names. cards = [_card(parent, parents={ParentAlias("other"): OTHER}), _card(OTHER)] - store = LocalRegistry(tmp_path / "store") - pins = _child_reading(store, *cards) + target_registry = LocalRegistry(tmp_path / "store") + pins = _child_reading(target_registry, *cards) calls = _install_demo_backend(monkeypatch, *cards) - sources = [store, InstalledConnectors()] + sources = [target_registry, InstalledConnectors()] - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) # The one build of OTHER comes before the build of the parent, which needs it. assert plan.steps == ( _build(OTHER, pinned_by={CHILD: pins[OTHER]}), _build(parent, depends_on=(OTHER,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=tuple(pins)), + ReadStep(CHILD, target_registry, depends_on=tuple(pins)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert calls == [(OTHER, False, {CHILD: pins[OTHER]}), (parent, False, {CHILD: pins[parent]})] - assert Path(version.payload_sources[SourceKey(str(OTHER))].root).is_relative_to(store.root) + assert Path(version.payload_sources[SourceKey(str(OTHER))].root).is_relative_to(target_registry.root) def test_two_readers_that_pin_one_file_differently_fail_the_read_before_any_step(tmp_path, monkeypatch): - # CHILD reads PARENT and OTHER. The card of PARENT declares X, which the store holds, and which + # CHILD reads PARENT and OTHER. The card of PARENT declares X, which the target registry holds, and which # reads OTHER too. So the read builds OTHER with the pins of CHILD, and the build of PARENT builds # OTHER with the pins of X. X pins one file of OTHER with another checksum, so no build matches both. x = DatasetRef.parse("test/x@1.0.0") cards = [_card(PARENT, parents={ParentAlias("x"): x}), _card(OTHER)] - store = LocalRegistry(tmp_path / "store") - _child_reading(store, cards[1], child_release=x) - _child_reading(store, *cards) - manifest = store.get_manifest(x.dataset_id) + target_registry = LocalRegistry(tmp_path / "store") + _child_reading(target_registry, cards[1], child_release=x) + _child_reading(target_registry, *cards) + manifest = target_registry.get_manifest(x.dataset_id) ((key, pinned),) = manifest.payload_sources.items() changed = pinned.files[0].model_copy(update={"checksum": "sha256:" + "0" * 64}) pinned = pinned.model_copy(update={"files": (changed, *pinned.files[1:])}) - (release_dir(store.root, x) / "manifest.json").write_text( + (release_dir(target_registry.root, x) / "manifest.json").write_text( manifest.model_copy(update={"payload_sources": {key: pinned}}).to_json() ) # A new registry, so that no manifest that it read before stays. - store = LocalRegistry(store.root) + target_registry = LocalRegistry(target_registry.root) calls = _install_demo_backend(monkeypatch, *cards) message = f"{CHILD} and {x} pin {changed.path!r} of {OTHER} as different files" with pytest.raises(TimeFFormatError, match=f"^{re.escape(message)}"): - read_release(CHILD.dataset_id, None, sources=[store, InstalledConnectors()], store=store) + read_release( + CHILD.dataset_id, None, sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) assert calls == [] @@ -454,7 +476,7 @@ def test_a_remote_read_fetches_each_manifest_once(tmp_path, monkeypatch): remote = RemoteRegistry("http://api.local", transport=transport) reads = _count_reads(monkeypatch, remote) - version = read_release(CHILD.dataset_id, None, sources=[remote], store=None) + version = read_release(CHILD.dataset_id, None, sources=[remote], target_registry=None) assert version.manifest == origin.get_manifest(CHILD.dataset_id) assert version.payload_sources[SourceKey(str(DEMO))].manifest == origin.get_manifest(DEMO.dataset_id) @@ -468,35 +490,35 @@ def test_latest_reads_the_manifest_of_the_version_that_the_answer_found(tmp_path manifest = origin.get_manifest(DEMO.dataset_id, str(DEMO.version)) reads = _count_reads(monkeypatch, origin) # A local registry answers "latest" without its manifest, so the plan reads it at the exact version. - version = read_release(DEMO.dataset_id, None, sources=[origin], store=None) + version = read_release(DEMO.dataset_id, None, sources=[origin], target_registry=None) assert version.manifest == manifest assert reads == {str(DEMO): 1} @pytest.mark.parametrize("parent", [PARENT, DatasetRef.parse("x/parent@1.0.0")], ids=["build-first", "read-first"]) def test_a_copy_that_a_build_needs_repairs_a_damaged_copy_that_the_read_found_first(tmp_path, monkeypatch, parent): - # CHILD reads the parent and DEMO. The store comes first in the list and holds DEMO damaged, and origin + # CHILD reads the parent and DEMO. The target registry comes first in the list and holds DEMO damaged, and origin # holds DEMO. Only the connectors build the parent, and its card declares DEMO. So the build copies DEMO # from origin over the damaged copy, and the read takes it there. parent_card = _card(parent, parents={ParentAlias("demo"): DEMO}) - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - pins = _child_reading(store, parent_card, make_dataset().metadata) - for registry in (store, origin): + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + pins = _child_reading(target_registry, parent_card, make_dataset().metadata) + for registry in (target_registry, origin): registry.put_dataset(make_dataset()) - (release_dir(store.root, DEMO) / "control.duckdb").write_bytes(b"cut short") + (release_dir(target_registry.root, DEMO) / "control.duckdb").write_bytes(b"cut short") _install_demo_backend(monkeypatch, parent_card) - sources = [store, origin, InstalledConnectors()] + sources = [target_registry, origin, InstalledConnectors()] - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert plan.steps == ( CopyStep(DEMO, origin), _build(parent, depends_on=(DEMO,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=tuple(pins)), + ReadStep(CHILD, target_registry, depends_on=tuple(pins)), ) - version = read_release(CHILD.dataset_id, None, sources=sources, store=store) - assert store.damage(DEMO) is None - assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(store.root) + version = read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) + assert target_registry.damage(DEMO) is None + assert Path(version.payload_sources[SourceKey(str(DEMO))].root).is_relative_to(target_registry.root) _SPEC = TimeSeriesSpec(spec_type="sine", name="Sine", unit_value=ureg.dimensionless) @@ -517,36 +539,38 @@ def _store_reading(registry: LocalRegistry, release: DatasetRef, source: Dataset def test_a_payload_source_that_a_build_needs_as_a_parent_comes_with_its_own_payload_sources(tmp_path, monkeypatch): # CHILD reads the own Record of SOURCE, which reads DEMO, and a Record of the parent. Origin holds SOURCE # and DEMO. Only the connectors build the parent, and its card declares SOURCE. The read finds SOURCE in - # origin first. The build then needs SOURCE in the store, so the copy takes DEMO along. + # origin first. The build then needs SOURCE in the target registry, so the copy takes DEMO along. source, parent = DatasetRef.parse("test/source@1.0.0"), DatasetRef.parse("x/parent@1.0.0") parent_card = _card(parent, parents={ParentAlias("source"): source}) - store, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") - store.put_dataset(make_dataset()) - _store_reading(store, source, DEMO) - store.put_dataset(_demo(parent_card)) + target_registry, origin = LocalRegistry(tmp_path / "store"), LocalRegistry(tmp_path / "origin") + target_registry.put_dataset(make_dataset()) + _store_reading(target_registry, source, DEMO) + target_registry.put_dataset(_demo(parent_card)) child = TimeFDataset(metadata=_card(CHILD, parents={ParentAlias("source"): source, ParentAlias("parent"): parent})) for alias, reference, record in (("source", source, 2), ("parent", parent, make_dataset().records[0].id)): - with TimeFReader(open_local_version(release_dir(store.root, reference))) as reader: + with TimeFReader(open_local_version(release_dir(target_registry.root, reference))) as reader: child.import_from(reader, alias=ParentAlias(alias), records=[record]) - store.put_dataset(child) + target_registry.put_dataset(child) for reference in (DEMO, source, parent): - shutil.copytree(release_dir(store.root, reference), release_dir(origin.root, reference)) - shutil.rmtree(release_dir(store.root, reference)) + shutil.copytree(release_dir(target_registry.root, reference), release_dir(origin.root, reference)) + shutil.rmtree(release_dir(target_registry.root, reference)) shutil.rmtree(release_dir(origin.root, parent)) - pins = {pinned.release: pinned for pinned in store.get_manifest(CHILD.dataset_id).payload_sources.values()} + pins = { + pinned.release: pinned for pinned in target_registry.get_manifest(CHILD.dataset_id).payload_sources.values() + } assert list(pins) == [source, parent] _install_demo_backend(monkeypatch, parent_card) - sources = [store, origin, InstalledConnectors()] + sources = [target_registry, origin, InstalledConnectors()] - plan = plan_read(CHILD.dataset_id, None, sources=sources, store=store) + plan = plan_read(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) assert plan.steps == ( CopyStep(source, origin), _build(parent, depends_on=(source,), pinned_by={CHILD: pins[parent]}), - ReadStep(CHILD, store, depends_on=(source, parent)), + ReadStep(CHILD, target_registry, depends_on=(source, parent)), ) - read_release(CHILD.dataset_id, None, sources=sources, store=store) - assert store.exists(DEMO.dataset_id, str(DEMO.version)) + read_release(CHILD.dataset_id, None, sources=sources, target_registry=target_registry) + assert target_registry.exists(DEMO.dataset_id, str(DEMO.version)) def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatch): @@ -559,7 +583,7 @@ def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatc registries.append(RemoteRegistry(f"http://{name}.local", transport=transport)) reads = _count_reads(monkeypatch, registries[0]) - version = read_release(DEMO.dataset_id, "latest", sources=registries, store=None) + version = read_release(DEMO.dataset_id, "latest", sources=registries, target_registry=None) assert version.manifest.metadata.dataset_version == newer.version # The plan takes the answer of the sources for the latest version, and asks for 2.0.0 nowhere. @@ -569,10 +593,12 @@ def test_a_read_of_the_latest_version_asks_each_source_once(tmp_path, monkeypatc def test_a_cycle_through_the_requested_release_starts_at_it(tmp_path, monkeypatch): # CHILD reads PARENT, which only the connectors build, and the card of PARENT declares CHILD. parent_card = _card(PARENT, parents={ParentAlias("child"): CHILD}) - store = LocalRegistry(tmp_path / "store") - _child_reading(store, parent_card) + target_registry = LocalRegistry(tmp_path / "store") + _child_reading(target_registry, parent_card) _install_demo_backend(monkeypatch, parent_card) message = f"dataset parent cycle: {CHILD} -> {PARENT} -> {CHILD}" with pytest.raises(TimeFValidationError, match=f"^{re.escape(message)}$"): - plan_read(CHILD.dataset_id, None, sources=[store, InstalledConnectors()], store=store) + plan_read( + CHILD.dataset_id, None, sources=[target_registry, InstalledConnectors()], target_registry=target_registry + ) diff --git a/packages/timenet/tests/registry/test_copy.py b/packages/timenet/tests/registry/test_copy.py index 37b1e2628..871a76098 100644 --- a/packages/timenet/tests/registry/test_copy.py +++ b/packages/timenet/tests/registry/test_copy.py @@ -25,6 +25,7 @@ from timenet.progress import BytesMoved, ProgressEvent, StepFinished, StepStarted from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, WritableRegistry, copy_release, open_local_version +from timenet.registry._paths import release_dir from timenet.registry.shared import complete_copy from timenet.testing import make_dataset from timenet.types import Access, Annotation, DatasetMetadata, DatasetSchema, License, TimeSeriesSpec, Version, ureg @@ -406,6 +407,21 @@ def test_copy_release_without_payload_sources_copies_the_release_alone(chain_sou assert not dest.exists(*chain[0]) +def test_copy_release_returns_the_releases_that_it_copied(chain_source, dest): + copy_release(grandparent_ref, chain_source, dest) + # Payload sources come first. dest holds the grandparent already, so the tuple leaves it out. + assert copy_release(grandchild_ref, chain_source, dest) == (middle_ref, grandchild_ref) + assert copy_release(grandchild_ref, chain_source, dest) == () + + +def test_copy_release_returns_each_release_that_it_repairs_or_copies_again(source, tmp_path): + dest = LocalRegistry(tmp_path / "dest") + copy_release(child_ref, source, dest) + (release_dir(dest.root, parent_ref) / dest.get_manifest(*parent).files.control[0].path).unlink() + assert copy_release(child_ref, source, dest) == (parent_ref,) + assert copy_release(child_ref, source, dest, force=True) == (child_ref,) + + def test_copy_release_copies_a_shared_payload_source_once(tmp_path): root = tmp_path / "chain" _write_chain(root, grandchild_reads_grandparent=True) diff --git a/packages/timenet/tests/registry/test_origin.py b/packages/timenet/tests/registry/test_origin.py new file mode 100644 index 000000000..7ba2171c5 --- /dev/null +++ b/packages/timenet/tests/registry/test_origin.py @@ -0,0 +1,123 @@ +"""The provenance record says where each release in the store comes from, and stays in the store.""" + +from datetime import UTC, datetime +import json + +from _demo_child import _child_of_the_demo +from _origin_record import origin_file +from pydantic import ValidationError +import pytest + +import timenet +from timenet.errors import TimeFFormatError, TimeNetDatasetNotFoundError +from timenet.registry import LocalRegistry +from timenet.registry.origin import Origin, OriginKind, without_credentials +from timenet.testing import make_dataset +from timenet.types.references import DatasetRef + + +DEMO = DatasetRef(dataset_id=make_dataset().metadata.dataset_id, version=make_dataset().metadata.dataset_version) + + +@pytest.fixture +def store(tmp_path) -> LocalRegistry: + registry = LocalRegistry(tmp_path / "store") + registry.put_dataset(make_dataset()) + return registry + + +def _read(store: LocalRegistry) -> Origin: + """Return the record of the demo release, which must have one.""" + origin = store.read_origin(DEMO) + assert origin is not None + return origin + + +def test_a_record_round_trips(store): + before = datetime.now(UTC).replace(microsecond=0) + store.record_origin(DEMO, OriginKind.BUILD, "timenet-connectors 0.9.0") + origin = _read(store) + + assert (origin.kind, origin.source) == (OriginKind.BUILD, "timenet-connectors 0.9.0") + assert origin.at.tzinfo is UTC + assert before <= origin.at <= datetime.now(UTC) + # The file holds the three keys, with the time in UTC to the second. + record = json.loads(origin_file(store, DEMO).read_text()) + assert record == { + "kind": "build", + "source": "timenet-connectors 0.9.0", + "at": origin.at.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +def test_a_new_record_replaces_the_old_one(store): + store.record_origin(DEMO, OriginKind.DOWNLOAD, "https://registry.example.org") + store.record_origin(DEMO, OriginKind.BUILD, "timenet-connectors 0.9.0") + assert _read(store).kind is OriginKind.BUILD + # The write leaves no temporary file next to the record. + path = origin_file(store, DEMO) + assert list(path.parent.glob(f"{path.name}*")) == [path] + + +@pytest.mark.parametrize( + ("location", "expected"), + [ + ("https://user:secret@registry.example.org:8443/r?sig=abc#top", "https://registry.example.org:8443/r"), + ("/data/registry", "/data/registry"), + ], +) +def test_without_credentials_keeps_only_scheme_host_and_path_of_a_url(location, expected): + assert without_credentials(location) == expected + + +def test_a_release_without_a_record_has_none(store): + assert store.read_origin(DEMO) is None + + +def test_a_record_needs_the_release_in_the_local_registry(tmp_path): + empty = LocalRegistry(tmp_path / "store") + with pytest.raises(TimeNetDatasetNotFoundError, match=f"the local registry holds no {DEMO}"): + empty.record_origin(DEMO, OriginKind.DOWNLOAD, "https://registry.example.org") + + +@pytest.mark.parametrize( + ("text", "problem"), + [ + ("not json", "Invalid JSON: expected ident at line 1 column 2"), + ( + '{"kind": "other", "source": "x", "at": "2026-11-03T14:02:11Z"}', + "kind: Input should be 'download' or 'build'", + ), + ], + ids=["invalid-json", "other-kind"], +) +def test_a_damaged_record_raises_with_its_first_problem(store, text, problem): + path = origin_file(store, DEMO) + path.write_text(text) + with pytest.raises(TimeFFormatError) as caught: + store.read_origin(DEMO) + assert str(caught.value) == f"{path} is not a valid provenance record: {problem}." + assert isinstance(caught.value.__cause__, ValidationError) + + +def test_a_time_with_another_offset_reads_as_utc(store): + path = origin_file(store, DEMO) + path.write_text('{"kind": "download", "source": "x", "at": "2026-11-03T16:02:11+02:00"}') + at = _read(store).at + assert at == datetime(2026, 11, 3, 14, 2, 11, tzinfo=UTC) + assert at.tzinfo is UTC + + +def test_publish_never_copies_the_record(tmp_path): + store = LocalRegistry(tmp_path / "store") + _, parent = _child_of_the_demo(store.root) + child = DatasetRef.parse("test/child@1.0.0") + store.record_origin(parent, OriginKind.DOWNLOAD, "https://registry.example.org") + store.record_origin(child, OriginKind.BUILD, "timenet-connectors 0.9.0") + + timenet.publish(str(child), dest=tmp_path / "dest", source=store) + + dest = LocalRegistry(tmp_path / "dest") + assert dest.exists(child.dataset_id, str(child.version)) + assert dest.exists(parent.dataset_id, str(parent.version)) + assert not list(dest.root.rglob(f"{origin_file(dest, child).name}*")) diff --git a/packages/timenet/tests/registry/test_registry.py b/packages/timenet/tests/registry/test_registry.py index a4ea35ec1..bfc1bbfab 100644 --- a/packages/timenet/tests/registry/test_registry.py +++ b/packages/timenet/tests/registry/test_registry.py @@ -300,6 +300,14 @@ def test_put_release_refuses_a_directory_without_a_readable_manifest(tmp_path, m # ---- s3 catalog ------------------------------------------------------------------------------- +def test_each_registry_names_its_location(tmp_path): + assert RemoteRegistry("https://registry.example.test").location == "https://registry.example.test" + assert open_registry("timenet://").location == TIMENET_REGISTRY_URL + assert S3Registry("s3://bucket/prefix").location == "s3://bucket/prefix" + assert S3Registry("s3://bucket").location == "s3://bucket" + assert LocalRegistry(tmp_path / "other" / ".." / "store").location == str((tmp_path / "store").resolve()) + + def test_s3_registry_has_no_catalog(): # The S3 backend does store + download by explicit id, but has no catalog: list/search raise. s3 = S3Registry("s3://bucket/registry") diff --git a/packages/timenet/tests/registry/test_remote.py b/packages/timenet/tests/registry/test_remote.py index 467dfe285..aa62730d5 100644 --- a/packages/timenet/tests/registry/test_remote.py +++ b/packages/timenet/tests/registry/test_remote.py @@ -204,7 +204,9 @@ def test_full_download_writes_identical_bytes(version_dir, tmp_path): assert (dest / "manifest.json").exists() -def test_a_download_into_the_store_reads_the_remote_manifest_for_the_plan_and_for_the_copy(version_dir, tmp_path): +def test_a_download_into_the_local_registry_reads_the_remote_manifest_for_the_plan_and_for_the_copy( + version_dir, tmp_path +): registry, requests = _remote(version_dir, tmp_path) directory, manifest = version_dir store = LocalRegistry(tmp_path / "store") diff --git a/packages/timenet/tests/registry/test_s3.py b/packages/timenet/tests/registry/test_s3.py index 75bbca840..fa57bb601 100644 --- a/packages/timenet/tests/registry/test_s3.py +++ b/packages/timenet/tests/registry/test_s3.py @@ -98,7 +98,7 @@ def test_objects_live_under_a_datasets_prefix(s3_root, tmp_path): assert all(key.startswith("registry/datasets/") for key in keys) -def test_download_then_load_reads_the_store_copy(s3_root, tmp_path): +def test_download_then_load_reads_the_local_registry_copy(s3_root, tmp_path): registry = S3Registry(s3_root) dataset_id = make_dataset().metadata.dataset_id registry.put_dataset(make_dataset()) diff --git a/packages/timenet/tests/test_builders.py b/packages/timenet/tests/test_builders.py index 1f1727342..d93f42af1 100644 --- a/packages/timenet/tests/test_builders.py +++ b/packages/timenet/tests/test_builders.py @@ -1,7 +1,10 @@ +from types import SimpleNamespace + import pytest +from timenet import builders from timenet.builders import declared, installed_builder -from timenet.errors import TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError +from timenet.errors import TimeNetBuildError, TimeNetDatasetNotFoundError, TimeNetVersionUnavailableError from timenet.testing import install_backends, make_dataset from timenet.types.references import DatasetRef @@ -57,3 +60,18 @@ def test_declared_returns_the_release_of_the_card_and_refuses_another_version(mo ) with pytest.raises(TimeNetDatasetNotFoundError, match="no installed connector builds 'x/y'"): declared("x/y") + + +def test_an_installed_builder_names_the_distribution_of_the_backend(monkeypatch): + install_backends(monkeypatch, _Backend("x/y", card=make_dataset().metadata)) + builder = installed_builder("x/y") + assert builder is not None + assert builder.package == "timenet-testing 0" + + +def test_a_backend_without_a_distribution_gives_no_installed_builder(monkeypatch): + backend = _Backend("a/b", card=make_dataset().metadata) + entry = SimpleNamespace(dist=None, load=lambda: lambda: backend) + monkeypatch.setattr(builders, "entry_points", lambda group: [entry]) + with pytest.raises(TimeNetBuildError, match=f"the entry point of {__name__}._Backend names no installed package"): + installed_builder("a/b") diff --git a/packages/timenet/tests/test_cache.py b/packages/timenet/tests/test_cache.py index 9a391553f..d25c61561 100644 --- a/packages/timenet/tests/test_cache.py +++ b/packages/timenet/tests/test_cache.py @@ -1,24 +1,39 @@ from pathlib import Path +from _demo_child import _child_of_the_demo +from _origin_record import origin_file import pytest from timenet.cache import ( ClearedCache, + KeptRelease, StoredRelease, cache_size, clear_cache, clear_releases, human_bytes, + releases_to_clear, stored_releases, ) +from timenet.client import TimeNet from timenet.config import CACHE_TAG, is_cache_folder, make_cache_folder -from timenet.errors import TimeFValidationError +from timenet.errors import TimeFFormatError, TimeFValidationError, TimeNetRegistryError from timenet.registry import LocalRegistry +from timenet.registry._paths import release_dir +from timenet.registry.origin import OriginKind from timenet.testing import make_dataset from timenet.types import Version +from timenet.types.reference_keys import ParentAlias from timenet.types.references import DatasetRef +PARENT = DatasetRef.parse("test/child@1.0.0") +GRAND = DatasetRef.parse("timenet/hello-world@1.0.0") +BUILT = DatasetRef.parse("test/built@1.0.0") +OTHER = DatasetRef.parse("demo/other@1.0.0") +HAND = DatasetRef.parse("demo/hand@1.0.0") + + def _store_with_two_releases(root): """Return a store that holds timenet/hello-world at 1.0.0 and 1.1.0.""" store = LocalRegistry(root) @@ -29,10 +44,40 @@ def _store_with_two_releases(root): return store -def _other_dataset(): - dataset = make_dataset() - dataset._metadata = dataset.metadata.model_copy(update={"dataset_id": "demo/other"}) - return dataset +def test_clear_releases_refuses_a_read_only_local_registry(tmp_path): + store = LocalRegistry(_store_with_two_releases(tmp_path / "store").root, read_only=True) + before = stored_releases(store) + with pytest.raises(TimeNetRegistryError, match=r"read_only=True, so it refuses a clear of its releases\."): + clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) + assert stored_releases(store) == before + + +def _layout(store): + """Store the five releases of the layout. + + PARENT reads GRAND as a payload source, BUILT declares PARENT as a parent, and OTHER and HAND + stand alone. + """ + _child_of_the_demo(store.root) + store.put_dataset(make_dataset(dataset_id=BUILT.dataset_id, parents={ParentAlias("parent"): PARENT})) + store.put_dataset(make_dataset(dataset_id=OTHER.dataset_id)) + store.put_dataset(make_dataset(dataset_id=HAND.dataset_id)) + + +def _record(store): + """Record PARENT, GRAND and OTHER as copies and BUILT as a build. HAND gets no record.""" + for release in (PARENT, GRAND, OTHER): + store.record_origin(release, OriginKind.DOWNLOAD, "https://registry.example.org") + store.record_origin(BUILT, OriginKind.BUILD, "timenet-connectors 0.9.0") + + +def _held(store): + """Return the releases of the layout that the store holds.""" + return { + release + for release in (BUILT, PARENT, GRAND, OTHER, HAND) + if store.exists(release.dataset_id, str(release.version)) + } def _size(directory): @@ -91,26 +136,48 @@ def test_human_bytes(n, expected): def test_stored_releases_lists_each_release_with_its_size(tmp_path): store = _store_with_two_releases(tmp_path / "store") first, second = (tmp_path / "store/timenet/hello-world" / version for version in ("1.0.0", "1.1.0")) - assert stored_releases(store) == [ - StoredRelease(DatasetRef.parse("timenet/hello-world@1.0.0"), first, _size(first)), - StoredRelease(DatasetRef.parse("timenet/hello-world@1.1.0"), second, _size(second)), + releases = stored_releases(store) + assert releases == [ + StoredRelease(DatasetRef.parse("timenet/hello-world@1.0.0"), first, None), + StoredRelease(DatasetRef.parse("timenet/hello-world@1.1.0"), second, None), ] + assert [item.size() for item in releases] == [_size(first), _size(second)] assert _size(first) > 0 +def test_stored_releases_carry_their_origin(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + releases = stored_releases(store) + assert {item.release: item.origin for item in releases} == { + release: store.read_origin(release) for release in (BUILT, PARENT, GRAND, OTHER, HAND) + } + # A release without a record counts as a build. + assert {item.release: item.kind for item in releases} == { + BUILT: OriginKind.BUILD, + HAND: OriginKind.BUILD, + PARENT: OriginKind.DOWNLOAD, + GRAND: OriginKind.DOWNLOAD, + OTHER: OriginKind.DOWNLOAD, + } + + def test_stored_releases_skips_what_is_not_a_release(tmp_path): store = _store_with_two_releases(tmp_path / "store") manifest = (tmp_path / "store/timenet/hello-world/1.0.0/manifest.json").read_bytes() - # A directory whose name is not a version, a staging directory, a hidden one, a path that is not - # a dataset ID, and a manifest at the root and one level below. - for stray in ( + # A directory whose name is not a version, a version in another spelling, a staging directory, a + # hidden one, a path that is not a dataset ID, and a manifest at the root and one level below. + strays = ( "timenet/hello-world/draft", + "timenet/hello-world/01.0.0", "timenet/hello-world/2.0.0.tmp-1", ".trash/x/1.0.0", "a/b/c/1.0.0", ".", "1.0.0", - ): + ) + for stray in strays: (tmp_path / "store" / stray).mkdir(parents=True, exist_ok=True) (tmp_path / "store" / stray / "manifest.json").write_bytes(manifest) assert [str(item.release) for item in stored_releases(store)] == [ @@ -119,7 +186,17 @@ def test_stored_releases_skips_what_is_not_a_release(tmp_path): ] -def test_clear_cache_keeps_a_store_inside_the_cache(tmp_path): +def test_releases_skips_an_unfinished_write_of_a_dataset_folder(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + # The staging folder of a dataset has a name that is also a valid dataset ID. + staging = store.root / "timenet/hello-world.tmp-1/1.0.0" + staging.mkdir(parents=True) + (staging / "manifest.json").write_bytes((release_dir(store.root, GRAND) / "manifest.json").read_bytes()) + assert [str(release) for release in store.releases()] == ["timenet/hello-world@1.0.0"] + + +def test_clear_cache_keeps_a_local_registry_inside_the_cache(tmp_path): cache = tmp_path / "cache" store = LocalRegistry(cache / "registry") store.put_dataset(make_dataset()) @@ -161,18 +238,22 @@ def test_clear_cache_keeps_and_reports_the_folders_without_the_marker(tmp_path): def test_clear_releases_deletes_every_release_but_no_other_file(tmp_path): store = _store_with_two_releases(tmp_path / "store") - store.put_dataset(_other_dataset()) + store.put_dataset(make_dataset(dataset_id=OTHER.dataset_id)) + # A download goes as well as the builds. + store.record_origin(OTHER, OriginKind.DOWNLOAD, "https://registry.example.org") notes = tmp_path / "store/notes.txt" notes.write_text("kept") staging = tmp_path / "store/timenet/hello-world/2.0.0.tmp-1/part.bin" staging.parent.mkdir() staging.write_bytes(b"kept") releases = stored_releases(store) + size = sum(item.size() for item in releases) - cleared = clear_releases(store) + cleared = clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) - assert cleared.freed == sum(item.size_bytes for item in releases) + assert cleared.freed == size assert cleared.removed == [item.path for item in releases] + assert cleared.kept == [] assert stored_releases(store) == [] # The dataset directories that the call emptied go, and the files that are not releases stay. assert not (tmp_path / "store/demo").exists() @@ -225,7 +306,7 @@ def test_clear_cache_unlinks_a_symlinked_downloads_folder_only_if_marked_and_kee assert (target / "raw.bin").exists() -def test_clear_cache_refuses_a_store_in_a_folder_that_it_deletes(tmp_path): +def test_clear_cache_refuses_a_local_registry_in_a_folder_that_it_deletes(tmp_path): make_cache_folder(tmp_path / "cache" / "downloads") store = LocalRegistry(tmp_path / "cache" / "downloads" / "store") store.put_dataset(make_dataset()) @@ -234,7 +315,7 @@ def test_clear_cache_refuses_a_store_in_a_folder_that_it_deletes(tmp_path): assert store.exists("timenet/hello-world", "1.0.0") -def test_clear_cache_refuses_a_folder_that_it_deletes_in_the_store(tmp_path): +def test_clear_cache_refuses_a_folder_that_it_deletes_in_the_local_registry(tmp_path): # With the cache in the store, the downloads folder can be the directory of a dataset. store = LocalRegistry(tmp_path / "store") make_cache_folder(tmp_path / "store" / "acme" / "downloads") @@ -266,6 +347,121 @@ def no_home(): assert not raw.exists() +def test_clearing_the_downloads_keeps_each_download_that_a_build_needs(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + cleared = clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert _held(store) == {BUILT, PARENT, GRAND, HAND} + # BUILT declares PARENT as a parent, and PARENT reads files of GRAND. + assert cleared.kept == [KeptRelease(PARENT, BUILT), KeptRelease(GRAND, PARENT)] + + +def test_clearing_the_downloads_deletes_a_parent_that_only_a_download_declares(tmp_path): + # A download needs its payload sources to be read, but not its parents. + store = LocalRegistry(tmp_path / "store") + copy = DatasetRef.parse("test/copy@1.0.0") + store.put_dataset(make_dataset(dataset_id=OTHER.dataset_id)) + store.put_dataset(make_dataset(dataset_id=copy.dataset_id, parents={ParentAlias("parent"): OTHER})) + store.put_dataset(make_dataset(dataset_id=BUILT.dataset_id, parents={ParentAlias("parent"): copy})) + for release in (OTHER, copy): + store.record_origin(release, OriginKind.DOWNLOAD, "https://registry.example.org") + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert [item.release for item in stored_releases(store)] == [BUILT, copy] + + +def test_clearing_the_builds_keeps_each_build_that_a_download_needs(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + # PARENT, a download, reads files of GRAND, a build. + store.record_origin(GRAND, OriginKind.BUILD, "timenet-connectors 0.9.0") + cleared = clear_releases(store, kinds=[OriginKind.BUILD]) + assert _held(store) == {PARENT, GRAND, OTHER} + assert cleared.kept == [KeptRelease(GRAND, PARENT)] + + +def test_clearing_every_kind_deletes_every_release_without_reading_a_manifest(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + # Nothing stays, so no release needs another. + (release_dir(store.root, BUILT) / "manifest.json").write_text("{}") + releases = stored_releases(store) + + cleared = clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) + + assert cleared.removed == [item.path for item in releases] + assert cleared.kept == [] + assert list((tmp_path / "store").iterdir()) == [] + + +def test_a_release_clear_with_a_damaged_record_deletes_nothing(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + origin_file(store, OTHER).write_text("{}") + with pytest.raises(TimeFFormatError, match="is not a valid provenance record"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} + + +def test_clearing_the_downloads_with_a_damaged_manifest_of_a_build_deletes_nothing(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + (release_dir(store.root, BUILT) / "manifest.json").write_text("{}") + with pytest.raises(TimeFFormatError, match="is not a valid manifest"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert {item.release for item in stored_releases(store)} == {BUILT, PARENT, GRAND, OTHER, HAND} + + +def test_clearing_the_downloads_refuses_a_local_registry_with_a_symlinked_dataset_folder(tmp_path): + # The listing skips a symlinked folder, so it would miss BUILT and delete the downloads that it needs. + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + moved = tmp_path / "elsewhere/built" + moved.parent.mkdir() + (store.root / "test/built").rename(moved) + (store.root / "test/built").symlink_to(moved, target_is_directory=True) + with pytest.raises(TimeFValidationError, match="test/built is a symlink"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD]) + assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} + + +def test_clearing_every_kind_refuses_a_local_registry_with_a_symlinked_release_folder(tmp_path): + store = LocalRegistry(tmp_path / "store") + _layout(store) + _record(store) + moved = tmp_path / "elsewhere/1.0.0" + moved.parent.mkdir() + release_dir(store.root, OTHER).rename(moved) + release_dir(store.root, OTHER).symlink_to(moved, target_is_directory=True) + with pytest.raises(TimeFValidationError, match=r"demo/other/1\.0\.0 is a symlink"): + clear_releases(store, kinds=[OriginKind.DOWNLOAD, OriginKind.BUILD]) + assert _held(store) == {BUILT, PARENT, GRAND, OTHER, HAND} + # The link hides the release from the listing, not from a read of its exact version. + with TimeNet(sources=[store]).open(str(OTHER)) as reader: + assert reader.metadata.dataset_id == OTHER.dataset_id + + +@pytest.mark.parametrize("clear", [releases_to_clear, clear_releases]) +def test_a_release_clear_refuses_an_empty_list_of_kinds_first(tmp_path, clear): + # A read-only local registry refuses a clear too, so the kinds must come first. + store = LocalRegistry(tmp_path / "store", read_only=True) + with pytest.raises(TimeFValidationError, match=r"^kinds names no kind of release$"): + clear(store, kinds=[]) + + +def test_the_symlink_check_skips_a_hidden_folder(tmp_path): + store = LocalRegistry(tmp_path / "store") + store.put_dataset(make_dataset()) + (tmp_path / "elsewhere").mkdir() + (store.root / ".trash").symlink_to(tmp_path / "elsewhere", target_is_directory=True) + store.check_no_linked_folders() + + def test_clear_cache_of_a_missing_cache_frees_nothing(tmp_path): assert cache_size(tmp_path / "cache") == 0 cleared = clear_cache(tmp_path / "cache", local_registry=LocalRegistry(tmp_path / "store")) diff --git a/packages/timenet/tests/test_cli.py b/packages/timenet/tests/test_cli.py index f943a6dcb..591a7fa08 100644 --- a/packages/timenet/tests/test_cli.py +++ b/packages/timenet/tests/test_cli.py @@ -23,6 +23,7 @@ ) from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, open_local_version +from timenet.registry.origin import OriginKind from timenet.testing import install_backends, make_dataset from timenet.types import Access, Domain from timenet.types.reference_keys import ParentAlias @@ -70,35 +71,92 @@ def state(tmp_path, monkeypatch): return store, cache +def _put(store, dataset, kind): + """Put ``dataset`` into the store, with a provenance record of ``kind``. + + Returns: + The release. + """ + registry = LocalRegistry(store) + release = registry.put_dataset(dataset) + registry.record_origin(release, kind, f"https://{kind.value}.example.org") + return release + + +def _stderr_has(result, text): + """Return whether stderr holds ``text``. + + The console wraps a long line, also in a path, so the check ignores whitespace. + """ + return "".join(text.split()) in "".join(click.unstyle(result.stderr).split()) + + def test_local_info_and_local_cache_info_show_the_paths_and_sizes(state): store, cache = state + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) result = runner.invoke(app, ["local", "info"]) assert result.exit_code == 0, result.output - [release] = stored_releases(LocalRegistry(store)) + copy, build = stored_releases(LocalRegistry(store)) cache_line = f"Cache {cache}: {human_bytes(_downloads_size(cache))}" assert click.unstyle(result.stdout).splitlines() == [ cache_line, - f"Local registry {store}: 1 release, {human_bytes(release.size_bytes)}", + f"Downloads in {store}: 1 release, {human_bytes(copy.size())}", + f"Builds in {store}: 1 release, {human_bytes(build.size())}", ] result = runner.invoke(app, ["local", "cache", "info"]) assert result.exit_code == 0, result.output assert click.unstyle(result.stdout) == f"{cache_line}\n" +@pytest.mark.parametrize( + ("command", "rows"), + [ + (["local", "downloads", "info"], [["demo/copy", "1.0.0", "https://download.example.org"]]), + ( + ["local", "builds", "info"], + [["demo/built", "1.0.0", "https://build.example.org"], ["timenet/hello-world", "1.0.0", "-"]], + ), + ], +) +def test_downloads_info_and_builds_info_list_the_releases_of_their_kind(state, command, rows): + store, _ = state + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + _put(store, make_dataset(dataset_id="demo/built"), OriginKind.BUILD) + result = runner.invoke(app, command) + assert result.exit_code == 0, result.output + cells = _table_cells(click.unstyle(result.stdout)) + sizes = {item.release.dataset_id: item.size() for item in stored_releases(LocalRegistry(store))} + total = sum(sizes[row[0]] for row in rows) + # The header uses another border, so the cells are the body rows. A build without a record has no source. + assert cells == [*([*row, human_bytes(sizes[row[0]])] for row in rows), ["total", "", "", human_bytes(total)]] + + +def test_downloads_info_of_a_local_registry_without_downloads(state): + store, _ = state + result = runner.invoke(app, ["local", "downloads", "info"]) + assert result.exit_code == 0, result.output + assert click.unstyle(result.stdout) == f"No downloads in {store}.\n" + + @pytest.mark.parametrize( ("command", "prompt"), [ - (["local", "clear"], "Delete the cache in {cache} and every release in {store}?"), + (["local", "clear"], "Delete the cache in {cache} and every release in {store}, 2 builds included?"), (["local", "cache", "clear"], "Delete the raw source files and control databases in {cache}?"), + (["local", "downloads", "clear"], "Delete 1 download from {store}?"), + (["local", "builds", "clear"], "Delete 2 builds from {store}?"), ], ) -def test_each_clear_asks_first_and_names_the_paths(state, command, prompt): +def test_each_clear_asks_first_and_counts_what_it_deletes(state, command, prompt): store, cache = state + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + _put(store, make_dataset(dataset_id="demo/built"), OriginKind.BUILD) + releases = stored_releases(LocalRegistry(store)) result = runner.invoke(app, command, input="n\n") assert result.exit_code != 0 # aborted assert f"{prompt.format(cache=cache, store=store)} [y/N]" in click.unstyle(result.output) assert (cache / "downloads").exists() - assert LocalRegistry(store).exists("timenet/hello-world", "1.0.0") + assert stored_releases(LocalRegistry(store)) == releases def test_local_cache_clear_keeps_the_local_registry(state): @@ -108,7 +166,7 @@ def test_local_cache_clear_keeps_the_local_registry(state): assert result.exit_code == 0, result.output assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 1 location(s)." assert not (cache / "downloads").exists() - assert LocalRegistry(store).exists("timenet/hello-world", "1.0.0") + assert [item.release for item in stored_releases(LocalRegistry(store))] == [HELLO] def test_local_cache_clear_warns_about_a_folder_without_the_marker_and_keeps_it(state): @@ -117,20 +175,36 @@ def test_local_cache_clear_warns_about_a_folder_without_the_marker_and_keeps_it( (cache / "control/notes.txt").write_text("not from TimeNet") result = runner.invoke(app, ["local", "cache", "clear", "--yes"]) assert result.exit_code == 0, result.output - # The console wraps a long line, also in a path, so the check ignores whitespace. - expected = f"Kept {cache / 'control'}: it has no CACHEDIR.TAG from TimeNet" - assert "".join(expected.split()) in "".join(click.unstyle(result.stderr).split()) + assert _stderr_has(result, f"Kept {cache / 'control'}: it has no CACHEDIR.TAG from TimeNet") assert click.unstyle(result.stdout).splitlines()[-1].endswith("from 1 location(s).") assert (cache / "control/notes.txt").exists() +def test_downloads_clear_keeps_each_download_that_a_build_needs_and_names_it(state): + store, cache = state + parent = _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + other = _put(store, make_dataset(dataset_id="demo/other"), OriginKind.DOWNLOAD) + child = _put( + store, make_dataset(dataset_id="demo/built", parents={ParentAlias("parent"): parent}), OriginKind.BUILD + ) + [gone] = [item for item in stored_releases(LocalRegistry(store)) if item.release == other] + size = human_bytes(gone.size()) + result = runner.invoke(app, ["local", "downloads", "clear", "--yes"]) + assert result.exit_code == 0, result.output + assert _stderr_has(result, f"Kept {parent}: {child} needs it.") + assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 1 location(s)." + assert [item.release for item in stored_releases(LocalRegistry(store))] == [child, parent, HELLO] + assert (cache / "downloads").exists() + + def test_local_clear_deletes_the_cache_and_every_release(state): store, cache = state - [release] = stored_releases(LocalRegistry(store)) - size = human_bytes(_downloads_size(cache) + release.size_bytes) + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + releases = stored_releases(LocalRegistry(store)) + size = human_bytes(_downloads_size(cache) + sum(item.size() for item in releases)) result = runner.invoke(app, ["local", "clear", "--yes"]) assert result.exit_code == 0, result.output - assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 2 location(s)." + assert click.unstyle(result.stdout).splitlines()[-1] == f"Freed {size} from 3 location(s)." assert stored_releases(LocalRegistry(store)) == [] assert not (cache / "downloads").exists() @@ -147,13 +221,21 @@ def test_local_clear_deletes_no_release_when_the_cache_check_fails(state, monkey def test_the_local_commands_take_the_local_registry_flag(tmp_path): - # The variable of the autouse fixture names an empty store, so the release comes from the flag. + # The variable of the autouse fixture names an empty store, so the releases come from the flag. store = tmp_path / "store" LocalRegistry(store).put_dataset(make_dataset()) - result = runner.invoke(app, ["local", "info", "--local-registry", str(store)]) + _put(store, make_dataset(dataset_id="demo/copy"), OriginKind.DOWNLOAD) + flag = ["--local-registry", str(store)] + result = runner.invoke(app, ["local", "info", *flag]) + assert result.exit_code == 0, result.output + assert f"Builds in {store}: 1 release" in click.unstyle(result.stdout) + result = runner.invoke(app, ["local", "builds", "info", *flag]) + assert result.exit_code == 0, result.output + assert "timenet/hello-world" in click.unstyle(result.stdout) + result = runner.invoke(app, ["local", "downloads", "clear", "--yes", *flag]) assert result.exit_code == 0, result.output - assert f"Local registry {store}: 1 release" in click.unstyle(result.stdout) - result = runner.invoke(app, ["local", "clear", "--yes", "--local-registry", str(store)]) + assert [item.release for item in stored_releases(LocalRegistry(store))] == [HELLO] + result = runner.invoke(app, ["local", "clear", "--yes", *flag]) assert result.exit_code == 0, result.output assert stored_releases(LocalRegistry(store)) == [] @@ -319,7 +401,9 @@ def test_info_lists_the_access_page_of_each_release_that_sets_one(tmp_path): assert ["records", str(len(child.records))] in cells -def test_download_command_copies_into_the_store_and_then_has_nothing_to_do(registry_root, tmp_path, monkeypatch): +def test_download_command_copies_into_the_local_registry_and_then_has_nothing_to_do( + registry_root, tmp_path, monkeypatch +): # The last line names a long path, so the console must not wrap it. monkeypatch.setattr(_cli_module.ui, "rich", Console(stderr=True, width=1000)) monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) @@ -368,7 +452,7 @@ def test_each_command_builds_the_list_from_its_flags(registry_root, tmp_path): assert result.stdout == f"{store / 'timenet/hello-world/1.0.0'}\n" -def test_publish_copies_a_release_from_the_store(tmp_path): +def test_publish_copies_a_release_from_the_local_registry(tmp_path): store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) dest = tmp_path / "dest" @@ -409,7 +493,7 @@ def test_publish_needs_dest(tmp_path): assert "--dest" in click.unstyle(result.output) -def test_publish_of_a_release_that_the_store_lacks_names_the_fix(tmp_path): +def test_publish_of_a_release_that_the_local_registry_lacks_names_the_fix(tmp_path): store, other, dest = tmp_path / "store", tmp_path / "other", str(tmp_path / "dest") result = runner.invoke(app, ["publish", "timenet/hello-world", "--dest", dest, "--local-registry", str(store)]) assert type(result.exception) is TimeNetDatasetNotFoundError @@ -429,7 +513,7 @@ def test_publish_of_a_release_that_the_store_lacks_names_the_fix(tmp_path): assert str(result.exception) == f"timenet/hello-world is not in local registry {other}." -def test_publish_adds_no_hint_to_a_payload_source_that_the_store_lacks(tmp_path): +def test_publish_adds_no_hint_to_a_payload_source_that_the_local_registry_lacks(tmp_path): store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) child = TimeFDataset( diff --git a/packages/timenet/tests/test_client.py b/packages/timenet/tests/test_client.py index 0f24ad853..f9771d96c 100644 --- a/packages/timenet/tests/test_client.py +++ b/packages/timenet/tests/test_client.py @@ -5,6 +5,7 @@ from _demo_child import _child_of_the_demo from _fake_registry import build_service_fake +from _origin_record import origin_file import httpx from pydantic import ValidationError import pytest @@ -33,6 +34,7 @@ from timenet.progress import BytesMoved, ProgressEvent, StepFinished, StepStarted from timenet.reader import Loading, LoadPlan, TimeFReader from timenet.registry import TIMENET_REGISTRY_URL, LocalRegistry, RemoteRegistry, open_local_version +from timenet.registry.origin import OriginKind from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal, install_backends, make_dataset from timenet.types import Access, Annotation, AnswerTask, DatasetMetadata, Domain, License, Version @@ -144,7 +146,7 @@ def test_resampled_child_reads_and_downloads_without_its_build_only_parent(tmp_p assert reader.read().records[0].signals[0].to_arrow().equals(expected) -def test_a_damaged_release_outside_the_store_names_no_download(tmp_path): +def test_a_damaged_release_outside_the_local_registry_names_no_download(tmp_path): origin, _ = _child_of_the_demo(tmp_path / "origin") mirror = LocalRegistry(tmp_path / "mirror") shutil.copytree(origin.root, mirror.root) @@ -157,7 +159,7 @@ def test_a_damaged_release_outside_the_store_names_no_download(tmp_path): assert (caught.value.release, caught.value.hint) == (DatasetRef.parse("test/child@1.0.0"), None) -def test_a_damaged_build_in_the_store_names_the_build_that_replaces_it(tmp_path, monkeypatch): +def test_a_damaged_build_in_the_local_registry_names_the_build_that_replaces_it(tmp_path, monkeypatch): calls = [] install_backends(monkeypatch, _Builder(calls)) store = LocalRegistry(tmp_path / "store") @@ -200,16 +202,16 @@ def test_the_default_list_follows_the_environment(monkeypatch, tmp_path): monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "store")) monkeypatch.setenv("TIMENET_REMOTE_REGISTRY", str(tmp_path / "remote")) monkeypatch.setenv("TIMENET_AUTO_BUILD", "false") - store, remote = default_sources() - assert isinstance(store, LocalRegistry) and isinstance(remote, LocalRegistry) - assert (store.root, remote.root) == (tmp_path / "store", tmp_path / "remote") + local_registry, remote = default_sources() + assert isinstance(local_registry, LocalRegistry) and isinstance(remote, LocalRegistry) + assert (local_registry.root, remote.root) == (tmp_path / "store", tmp_path / "remote") # TimeNet() uses the same list. client = TimeNet() assert [_entry(source) for source in client._sources] == [ ("local", tmp_path / "store"), ("local", tmp_path / "remote"), ] - assert client._store is client._sources[0] + assert client._local_registry is client._sources[0] @pytest.mark.usefixtures("home") @@ -554,6 +556,25 @@ def test_get_manifest_names_every_source_when_none_provides_the_release(tmp_path ) +def test_download_records_each_copied_release(tmp_path): + origin, parent = _child_of_the_demo(tmp_path / "origin") + child = DatasetRef.parse("test/child@1.0.0") + store = LocalRegistry(tmp_path / "store") + client = TimeNet(sources=[store, origin.root]) + client.download("test/child") + for release in (parent, child): + record = store.read_origin(release) + assert record is not None + assert (record.kind, record.source) == (OriginKind.DOWNLOAD, str(origin.root.resolve())) + + # With force, the call copies only the requested release again, so only that release gets a record. + for release in (parent, child): + origin_file(store, release).unlink() + client.download("test/child", force=True) + assert store.read_origin(parent) is None + assert store.read_origin(child) is not None + + def test_download_needs_a_local_registry_that_is_not_read_only(registry_root): with pytest.raises(TimeFValidationError) as caught: TimeNet(sources=[LocalRegistry(registry_root, read_only=True)]).download("timenet/hello-world") @@ -624,10 +645,16 @@ def test_download_copies_each_release_from_the_first_registry_that_provides_it(t assert path == store.root / "test/child/1.0.0" with TimeNet(sources=[store]).open("test/child") as reader: reader.verify() + # Each provenance record names the registry that the release came from. + records = {release: store.read_origin(release) for release in (DatasetRef.parse("test/child@1.0.0"), reference)} + assert {release: (record.kind, record.source) for release, record in records.items() if record} == { + DatasetRef.parse("test/child@1.0.0"): (OriginKind.DOWNLOAD, str((tmp_path / "first").resolve())), + reference: (OriginKind.DOWNLOAD, str((tmp_path / "second").resolve())), + } @pytest.mark.parametrize("case", ["force", "damaged"]) -def test_download_refuses_a_store_copy_with_other_files(registry_root, tmp_path, case): +def test_download_refuses_a_local_registry_copy_with_other_files(registry_root, tmp_path, case): store = LocalRegistry(tmp_path / "store") other = make_dataset() other.records[0].annotate(Annotation(key="build", value="another")) @@ -647,7 +674,7 @@ def test_download_refuses_a_store_copy_with_other_files(registry_root, tmp_path, assert (directory / "manifest.json").read_text() == before -def test_download_names_the_store_when_no_other_registry_provides_its_copy(tmp_path, monkeypatch): +def test_download_names_the_local_registry_when_no_other_registry_provides_its_copy(tmp_path, monkeypatch): calls = [] install_backends(monkeypatch, _Builder(calls)) store = LocalRegistry(tmp_path / "store") @@ -664,7 +691,9 @@ def test_download_names_the_store_when_no_other_registry_provides_its_copy(tmp_p @pytest.mark.parametrize("case", ["force", "damaged"]) -def test_download_counts_a_registry_over_the_directory_of_the_store_as_the_store(tmp_path, monkeypatch, case): +def test_download_counts_a_registry_over_the_directory_of_the_local_registry_as_the_local_registry( + tmp_path, monkeypatch, case +): root = tmp_path / "reg" LocalRegistry(root).put_dataset(make_dataset()) directory = root / "timenet/hello-world/1.0.0" @@ -693,7 +722,7 @@ def test_download_counts_a_registry_over_the_directory_of_the_store_as_the_store assert {path: path.read_bytes() for path in directory.rglob("*") if path.is_file()} == before -def test_download_counts_a_symlink_to_the_store_directory_as_the_store(tmp_path): +def test_download_counts_a_symlink_to_the_local_registry_directory_as_the_local_registry(tmp_path): store = LocalRegistry(tmp_path / "store") store.put_dataset(make_dataset()) (tmp_path / "link").symlink_to(store.root, target_is_directory=True) @@ -704,7 +733,7 @@ def test_download_counts_a_symlink_to_the_store_directory_as_the_store(tmp_path) client.download("timenet/hello-world", force=True) -def test_download_completes_the_payload_sources_of_a_complete_store_copy(tmp_path): +def test_download_completes_the_payload_sources_of_a_complete_local_registry_copy(tmp_path): origin, reference = _child_of_the_demo(tmp_path / "origin") store = LocalRegistry(tmp_path / "store") client = TimeNet(sources=[store, origin]) @@ -753,7 +782,7 @@ def test_download_copies_a_payload_source_without_the_payload_sources_of_that_so assert not store.exists(demo.dataset_id, str(demo.version)) -def test_download_repairs_a_damaged_store_copy(tmp_path): +def test_download_repairs_a_damaged_local_registry_copy(tmp_path): origin, reference = _child_of_the_demo(tmp_path / "origin") store = LocalRegistry(tmp_path / "store") client = TimeNet(sources=[store, origin.root]) @@ -770,7 +799,7 @@ def test_download_repairs_a_damaged_store_copy(tmp_path): reader.verify() -def test_download_returns_a_complete_store_copy_at_once(registry_root, tmp_path): +def test_download_returns_a_complete_local_registry_copy_at_once(registry_root, tmp_path): transport, requests = build_service_fake(registry_root) remote = RemoteRegistry("http://api.local", transport=transport) client = TimeNet(sources=[LocalRegistry(tmp_path / "store"), remote]) diff --git a/packages/timenet/tests/test_config.py b/packages/timenet/tests/test_config.py index f5cdabb47..fdb79c966 100644 --- a/packages/timenet/tests/test_config.py +++ b/packages/timenet/tests/test_config.py @@ -13,7 +13,7 @@ def _unset(monkeypatch): @pytest.mark.usefixtures("home") -def test_the_defaults_are_the_store_the_hosted_registry_the_connectors_and_the_cache(monkeypatch): +def test_the_defaults_are_the_local_registry_the_hosted_registry_the_connectors_and_the_cache(monkeypatch): _unset(monkeypatch) cfg = settings() assert cfg.local_registry_dir == Path("~/.timenet/registry").expanduser() diff --git a/packages/timenet/tests/test_producer.py b/packages/timenet/tests/test_producer.py index e8a804b83..c79a3e964 100644 --- a/packages/timenet/tests/test_producer.py +++ b/packages/timenet/tests/test_producer.py @@ -104,7 +104,7 @@ def test_build_without_sources_copies_a_parent_from_the_remote_registry(tmp_path assert [release for release, _ in cards] == [CHILD] -def test_build_lands_in_the_store_that_the_environment_names(tmp_path, monkeypatch, cards): +def test_build_lands_in_the_local_registry_that_the_environment_names(tmp_path, monkeypatch, cards): monkeypatch.setenv("TIMENET_LOCAL_REGISTRY", str(tmp_path / "env-store")) assert timenet.build(PARENT.dataset_id) == tmp_path / "env-store/test/parent/1.0.0" assert LocalRegistry(tmp_path / "env-store").exists(PARENT.dataset_id, "1.0.0") @@ -180,7 +180,7 @@ def store(tmp_path, monkeypatch): return LocalRegistry(tmp_path / "store") -def test_publish_copies_the_latest_release_of_the_store(tmp_path, store): +def test_publish_copies_the_latest_release_of_the_local_registry(tmp_path, store): store.put_dataset(_demo(_card(DatasetRef.parse("timenet/hello-world@0.9.0")))) store.put_dataset(make_dataset()) dest = tmp_path / "dest"