From f74087e201afe5a0f2af0c65bfc969185724d94b Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 10 Nov 2026 19:50:54 +0100 Subject: [PATCH] feat(registry): refuse a release with local changes in a remote registry A remote or S3 registry now accepts only a release that was built from committed TimeNet code. Each manifest records the git commit of the TimeNet packages and whether they had local changes. If a manifest records local changes, put_release() raises TimeNetBuildError. The same applies to put_dataset(), because it publishes through put_release(). copy_release(), and so timenet.publish() and timenet-build --publish-to, checks every release that it copies before it copies anything. The check reads the manifest, not the installed code, because a publish copies a release that was built earlier. LocalRegistry accepts any release, through the new requires_committed_code property. Part of TNET-165 --- docs/build.md | 11 +++++ docs/cli/build.md | 4 ++ docs/manifest.md | 3 ++ docs/registry.md | 5 ++- packages/timenet/src/timenet/producer.py | 2 + packages/timenet/src/timenet/provenance.py | 16 +++++++ .../timenet/src/timenet/registry/_copy.py | 3 ++ .../timenet/src/timenet/registry/local.py | 9 ++++ .../src/timenet/registry/remote/registry.py | 1 + packages/timenet/src/timenet/registry/s3.py | 1 + .../timenet/src/timenet/registry/writable.py | 37 ++++++++++++++-- packages/timenet/tests/registry/test_copy.py | 44 +++++++++++++++++++ 12 files changed, 131 insertions(+), 5 deletions(-) diff --git a/docs/build.md b/docs/build.md index e9a45f817..658f20e5c 100644 --- a/docs/build.md +++ b/docs/build.md @@ -135,6 +135,17 @@ A remote registry holds only releases with access declared `open`. Before `timenet.publish()` copies anything to such a registry, it checks the card and the lineage of each release. +## Committed code for remote registries + +The manifest of a release records the git commit of the TimeNet code that built it, and whether that +code had local changes (see [`build_env.git`](manifest.md)). A remote registry holds only releases +built from committed code. Before `timenet.publish()` copies anything to such a registry, it checks +the manifest of each release that it copies. If a manifest records local changes, `publish()` raises +`TimeNetBuildError` and copies nothing. + +The local registry accepts any release, so a build with local changes still lands there. To publish +it, commit the changes, build the release again with `force=True`, and publish it. + ## Prepare parent releases A connector card can declare exact parent releases under `parents`. The connector reads these diff --git a/docs/cli/build.md b/docs/cli/build.md index f6768a3ef..f32cc7c2d 100644 --- a/docs/cli/build.md +++ b/docs/cli/build.md @@ -70,6 +70,10 @@ While `--force` builds the release again in the local registry. It cannot overwr version. If the new build wrote the same files, the publish copies nothing. If it wrote other files, the publish raises `TimeNetVersionConflictError`, and the change needs a new version. +A remote destination holds only releases built from committed TimeNet code, as [Committed code for +remote registries](../build.md#committed-code-for-remote-registries) describes. If the code has +local changes, the build lands in the local registry, and the publish fails. + ## Output Status goes to stderr. Before the first step, the command shows its plan as a tree: the requested diff --git a/docs/manifest.md b/docs/manifest.md index 34430a84a..1d71a8141 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -84,6 +84,9 @@ If the commit of an installed package is unknown, the build fails and names the example, git is not on `PATH`, git fails in the checkout, or a plain `uv build` produced the wheel, so the package has no `_build_info.json`. +A remote registry accepts only a release whose packages all record `contains_local_changes` as +`false`. See [Committed code for remote registries](build.md#committed-code-for-remote-registries). + The values locator is backend-neutral. One schema covers both scalar and multidimensional specs. A multidimensional spec records its shape in `value_shape` and `dimension_names`. It does not need a separate format version. diff --git a/docs/registry.md b/docs/registry.md index 05b74a9bc..3db3b5369 100644 --- a/docs/registry.md +++ b/docs/registry.md @@ -209,6 +209,7 @@ so you can publish to a local directory, a remote registry, or an S3 bucket: | `put_dataset(dataset, *, force=False, values_backend="parquet", progress_cb=None)` | Compile a dataset and publish it. Returns the stored release as a `DatasetRef`. Derives the schema first if absent, skips an already-committed version unless `force`, and checks every payload source in this registry. `values_backend` picks the values plane format: `"parquet"` or `"zarr"`. On a remote or S3 registry, `force=True` raises `TimeFValidationError` and uploads nothing, because a published version never changes. | | `put_release(from_directory)` | Publish a complete local release directory as it is; returns the published release as a `DatasetRef`. Only reads `from_directory` and never changes it. Makes sure that the directory holds a readable `manifest.json` and every file that the manifest declares with its recorded size, and raises `TimeFFormatError` or `TimeNetFileError` otherwise. Publishes only those files and `manifest.json`. | | `exists(dataset_id, version)` | Whether a committed version already exists (shared implementation). | +| `requires_committed_code` | Whether the registry accepts only releases built from committed TimeNet code. `False` only for `LocalRegistry`. Otherwise `put_dataset` and `put_release` raise `TimeNetBuildError` for a release whose manifest records local changes. | `LocalRegistry`, `RemoteRegistry` and `S3Registry` take `read_only=True`. A read-only registry refuses each write with `TimeNetRegistryError`. @@ -278,7 +279,9 @@ A dataset can depend on payload files stored as part of another dataset (parent/ It copies only the releases that `dest` does not hold yet. `copy_release` checks every release before it copies anything, so a refused copy leaves `dest` -unchanged. +unchanged. A `dest` with `requires_committed_code` refuses a release whose manifest records TimeNet +code with local changes, as +[Committed code for remote registries](build.md#committed-code-for-remote-registries) describes. ## `search` diff --git a/packages/timenet/src/timenet/producer.py b/packages/timenet/src/timenet/producer.py index e4516104c..a1f2e8355 100644 --- a/packages/timenet/src/timenet/producer.py +++ b/packages/timenet/src/timenet/producer.py @@ -105,6 +105,7 @@ def publish( TimeNetRegistryError: If ``dest`` or ``source`` fails to open as a registry, or ``dest`` is read-only. TimeNetDatasetNotFoundError: If ``source`` lacks the release. TimeNetAccessError: If a registry withholds files or requires open access that a copied release lacks. + TimeNetBuildError: If ``dest`` is not local and a copied release records TimeNet code with local changes. TimeNetVersionConflictError: If ``dest`` holds different files for the release or a required payload source. TimeNetMissingParentError: If a required payload source is missing from ``source`` and incomplete in ``dest``. """ # ruff: ignore[docstring-extraneous-exception] - raised by open_registry and copy_release @@ -142,6 +143,7 @@ def _publish_from( Raises: TimeNetDatasetNotFoundError: If ``origin`` lacks the release. TimeNetAccessError: If a registry withholds files or requires open access that a copied release lacks. + TimeNetBuildError: If ``dest`` is not local and a copied release records TimeNet code with local changes. TimeNetVersionConflictError: If ``dest`` holds different files for the release or a required payload source. TimeNetMissingParentError: If a required payload source is missing from ``origin`` and incomplete in ``dest``. """ # ruff: ignore[docstring-extraneous-exception] - raised by copy_release diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index a6a028cdf..4eed697f7 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -182,6 +182,10 @@ class Snapshot(BaseModel, ABC): def to_source_map(self) -> GitSourceMap: """Return the commits in the form that a manifest records.""" + @abstractmethod + def require_no_local_changes(self) -> None: + """Raise ``TimeNetBuildError`` if the code has local changes.""" + class PackageSnapshot(Snapshot): """The commit of one package that pip installed, such as ``timenet`` or ``timenet-connectors``. @@ -216,6 +220,13 @@ def to_source_map(self) -> GitSourceMap: """Return this package's commit in the form that a manifest records.""" return {self.package_name: self.source} + def require_no_local_changes(self) -> None: + """Raise ``TimeNetBuildError`` if this package has local changes.""" + if self.source.contains_local_changes: + raise TimeNetBuildError( + f"{self.package_name} contains changes that are not in commit {self.source.sha[:12]}" + ) + class TimeNetSnapshot(Snapshot): """The commit of each installed TimeNet package, read at one point in time.""" @@ -264,6 +275,11 @@ def to_source_map(self) -> GitSourceMap: sources.update(package.to_source_map()) return sources + def require_no_local_changes(self) -> None: + """Raise ``TimeNetBuildError`` if a TimeNet package has local changes.""" + for package in self._installed(): + package.require_no_local_changes() + def _installed(self) -> list[PackageSnapshot]: """Return the snapshot of each installed TimeNet package.""" return [package for package in (self.timenet, self.timenet_connectors) if package is not None] diff --git a/packages/timenet/src/timenet/registry/_copy.py b/packages/timenet/src/timenet/registry/_copy.py index 91017cd43..f55fcd371 100644 --- a/packages/timenet/src/timenet/registry/_copy.py +++ b/packages/timenet/src/timenet/registry/_copy.py @@ -50,6 +50,7 @@ def copy_release( # ruff: ignore[too-many-arguments] TimeFValidationError: If ``force`` is set and ``dest`` is not local. TimeNetVersionConflictError: If ``dest`` holds the same release with other files or another card. TimeNetAccessError: If a release lacks open access and either registry requires it. + TimeNetBuildError: If ``dest`` is not local and a copied release records TimeNet code with local changes. TimeNetDatasetNotFoundError: If ``source`` lacks the release. TimeNetMissingParentError: If neither registry holds a complete payload source. TimeFFormatError: If a manifest names the wrong release or a pin fails. @@ -67,6 +68,8 @@ def copy_release( # ruff: ignore[too-many-arguments] if _needs_copy(reference, ref, manifest, source, dest) or (force and ref == reference) ] _check_access(manifests, set(pending), source, dest) + for ref in pending: + dest.require_committed_code(manifests[ref]) if not pending: return () # Only a copy from a non-local source to a non-local dest downloads into this directory, each diff --git a/packages/timenet/src/timenet/registry/local.py b/packages/timenet/src/timenet/registry/local.py index 5f65a45af..b42cfcd0b 100644 --- a/packages/timenet/src/timenet/registry/local.py +++ b/packages/timenet/src/timenet/registry/local.py @@ -69,6 +69,15 @@ def root(self) -> Path: """The directory this registry reads from and writes to.""" return self._root + @property + def requires_committed_code(self) -> bool: + """Whether local releases must record committed code. + + Returns: + ``False``, since a local registry accepts a build with local changes. + """ + return False + @property def requires_open_access(self) -> bool: """Whether local releases must declare open access. diff --git a/packages/timenet/src/timenet/registry/remote/registry.py b/packages/timenet/src/timenet/registry/remote/registry.py index 74143b5eb..e3385bd7f 100644 --- a/packages/timenet/src/timenet/registry/remote/registry.py +++ b/packages/timenet/src/timenet/registry/remote/registry.py @@ -175,6 +175,7 @@ def put_release(self, from_directory: Path) -> DatasetRef: Raises: TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. + TimeNetBuildError: If the manifest records TimeNet code with local changes. TimeNetRegistryError: If this registry is read-only, or the service skips a listed file or requests a missing file. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 19c654e1a..6d876fae3 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -171,6 +171,7 @@ def put_release(self, from_directory: Path) -> DatasetRef: TimeNetRegistryError: If this registry is read-only. TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. + TimeNetBuildError: If the manifest records TimeNet code with local changes. """ # ruff: ignore[docstring-extraneous-exception] - raised by _read_release self.check_writable("put_release()") manifest = self._read_release(from_directory) diff --git a/packages/timenet/src/timenet/registry/writable.py b/packages/timenet/src/timenet/registry/writable.py index a4d434b6f..c19a00b87 100644 --- a/packages/timenet/src/timenet/registry/writable.py +++ b/packages/timenet/src/timenet/registry/writable.py @@ -13,8 +13,10 @@ import pyarrow.fs as pafs from timenet.dataset import TimeFDataset +from timenet.errors import TimeNetBuildError from timenet.format.constants import MANIFEST_FILE from timenet.manifest import Manifest, PayloadSource, read_manifest +from timenet.provenance import TimeNetSnapshot from timenet.registry.base import BaseRegistry from timenet.registry.dependencies import check_file_sizes from timenet.registry.version import DatasetVersion @@ -67,11 +69,36 @@ def put_release(self, from_directory: Path) -> DatasetRef: Raises: TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. + TimeNetBuildError: If this registry accepts only committed code, and the manifest records local changes. """ - @staticmethod - def _read_release(directory: Path) -> Manifest: - """Read a release manifest and check the listed file sizes. + @property + def requires_committed_code(self) -> bool: + """Whether a release must record TimeNet code without local changes.""" + return True + + def require_committed_code(self, manifest: Manifest) -> None: + """Raise ``TimeNetBuildError`` if this registry accepts only committed code, and ``manifest`` records local changes. + + Args: + manifest: The manifest of the release to publish. + + Raises: + TimeNetBuildError: If the manifest records local changes or no commit for a TimeNet package. + """ + if not self.requires_committed_code: + return + try: + TimeNetSnapshot.from_source_map(manifest.build_env.git).require_no_local_changes() + except TimeNetBuildError as exc: + reference = DatasetRef(dataset_id=manifest.dataset_id, version=manifest.metadata.dataset_version) + raise TimeNetBuildError( + f"{self} accepts only releases built from committed TimeNet code, so it refuses {reference}: {exc}. " + "Build the release again from committed code, or keep it in a local registry." + ) from exc + + def _read_release(self, directory: Path) -> Manifest: + """Read a release manifest, check the listed file sizes, and check that this registry accepts its code. Args: directory: The release directory. @@ -82,8 +109,10 @@ def _read_release(directory: Path) -> Manifest: Raises: TimeFFormatError: If ``manifest.json`` is missing, unreadable, or invalid. TimeNetFileError: If a listed file is missing or its size differs from the manifest. - """ # ruff: ignore[docstring-extraneous-exception] - raised by read_manifest and check_file_sizes + TimeNetBuildError: If this registry accepts only committed code, and the manifest records local changes. + """ # ruff: ignore[docstring-extraneous-exception] - raised by read_manifest, check_file_sizes and require_committed_code manifest = read_manifest(directory / MANIFEST_FILE) + self.require_committed_code(manifest) reference = DatasetRef(dataset_id=manifest.dataset_id, version=manifest.metadata.dataset_version) version = DatasetVersion(manifest, pafs.LocalFileSystem(), str(directory)) check_file_sizes(version, manifest.files.all_files(), owner=str(reference)) diff --git a/packages/timenet/tests/registry/test_copy.py b/packages/timenet/tests/registry/test_copy.py index 4360495a2..4659f6da4 100644 --- a/packages/timenet/tests/registry/test_copy.py +++ b/packages/timenet/tests/registry/test_copy.py @@ -20,9 +20,12 @@ TimeFFormatError, TimeFValidationError, TimeNetAccessError, + TimeNetBuildError, TimeNetMissingParentError, TimeNetVersionConflictError, ) +from timenet.format.constants import MANIFEST_FILE +from timenet.manifest import GitSource from timenet.progress import BytesMoved, ProgressEvent, StepFinished, StepStarted from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, RemoteRegistry, WritableRegistry, copy_release, open_local_version @@ -542,3 +545,44 @@ def test_copy_release_refuses_to_fetch_a_release_that_is_not_open_from_a_registr assert raised.value.dataset == child_ref assert not dest.exists(*child) assert not dest.exists(*parent) # the open parent is not copied either: every check runs first + + +def _record_local_changes(root: Path, reference: DatasetRef) -> None: + """Rewrite the manifest of ``reference`` under ``root`` as a build from timenet code with local changes.""" + manifest = LocalRegistry(root).get_manifest(reference.dataset_id, str(reference.version)) + git = {"timenet": GitSource(sha="a" * 40, contains_local_changes=True)} + build_env = manifest.build_env.model_copy(update={"git": git}) + (release_dir(root, reference) / MANIFEST_FILE).write_text( + manifest.model_copy(update={"build_env": build_env}).to_json() + ) + + +@pytest.mark.parametrize("changed", [child_ref, parent_ref], ids=["release", "payload-source"]) +def test_copy_release_refuses_a_build_with_local_changes_for_a_registry_that_is_not_local(changed, tmp_path): + root = tmp_path / "source" + _write_parent_and_child(root) + _record_local_changes(root, changed) + dest, _ = _remote(tmp_path, "dest", token="tok_rw") + with pytest.raises(TimeNetBuildError, match=f"refuses {changed}: timenet contains changes that are not in commit"): + copy_release(child_ref, LocalRegistry(root), dest) + assert not dest.exists(*child) + assert not dest.exists(*parent) # every check runs before the first copy + + +def test_copy_release_copies_a_build_with_local_changes_between_local_registries(tmp_path): + root = tmp_path / "source" + _write_parent_and_child(root) + _record_local_changes(root, child_ref) + dest = LocalRegistry(tmp_path / "dest") + copy_release(child_ref, LocalRegistry(root), dest) + assert dest.get_manifest(*child).build_env.git["timenet"].contains_local_changes + + +def test_put_release_refuses_a_build_with_local_changes_for_a_registry_that_is_not_local(tmp_path): + root = tmp_path / "source" + _write_parent_and_child(root) + _record_local_changes(root, parent_ref) + dest, _ = _remote(tmp_path, "dest", token="tok_rw") + with pytest.raises(TimeNetBuildError, match=f"refuses {parent_ref}: timenet contains changes"): + dest.put_release(release_dir(root, parent_ref)) + assert not dest.exists(*parent)