From 0aa8a0f7d2eab16564c0e2d4121ed10f68c6901f Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 15:43:48 +0100 Subject: [PATCH 01/19] feat(manifest): record the git commit of each TimeNet distribution A manifest now records which TimeNet code built a release. Before, a local checkout reported the same package version for every commit, so two builds from different code looked the same. `build_env.git` maps each installed TimeNet distribution to a `GitSource` with `sha` and `local_changes`. A local checkout asks git, and `local_changes` covers only the package directory, untracked files included. A wheel reads the commit from `_build_info.json`. `scripts/write_build_info.py` writes that file before `uv build`, and `make build` and the publish workflow run the script. A distribution with no known commit is absent from the map. `local_source_path` moves from the connector builder to `timenet.provenance`, so both use the same function. `test_a_built_version_starts_no_isolated_child` now stores its dataset before it blocks subprocesses, because a manifest write runs git. A reader without this change rejects a new manifest, because the manifest forbids unknown keys. This change lands before 1.0, so `timef_format_version` stays at 1. Part of TNET-165 --- .github/workflows/publish.yml | 2 + .gitignore | 3 + Makefile | 1 + docs/build.md | 4 +- docs/manifest.md | 16 ++- .../src/timenet_connectors/builder/env.py | 33 +---- .../timenet/src/timenet/manifest/__init__.py | 3 +- .../timenet/src/timenet/manifest/manifest.py | 18 ++- packages/timenet/src/timenet/provenance.py | 132 +++++++++++++++++- .../src/timenet/schemas/manifest.schema.json | 28 ++++ .../timenet/tests/manifest/test_manifest.py | 5 +- packages/timenet/tests/test_provenance.py | 91 ++++++++++++ scripts/write_build_info.py | 61 ++++++++ 13 files changed, 355 insertions(+), 42 deletions(-) create mode 100644 scripts/write_build_info.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 27c4f721c..a45f762c5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,6 +51,8 @@ jobs: uses: actions/checkout@v5 - name: Set up uv uses: astral-sh/setup-uv@v7 + - name: Record the commit in each package + run: uv run scripts/write_build_info.py - name: Build run: | if [ "${{ inputs.package }}" = "both" ]; then diff --git a/.gitignore b/.gitignore index 284e19e23..a44700f6e 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ docs/api/ # Local superpowers planning artifacts, not part of the repo docs/superpowers/ + +# commit record written into each import package before a wheel build (see scripts/write_build_info.py) +packages/*/src/*/_build_info.json diff --git a/Makefile b/Makefile index b007f42ff..19ee78e73 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ license-check: --format=markdown build: + uv run scripts/write_build_info.py uv build --package timenet uv build --package timenet-connectors diff --git a/docs/build.md b/docs/build.md index 4b33def97..e9a45f817 100644 --- a/docs/build.md +++ b/docs/build.md @@ -48,8 +48,8 @@ variables (see the [settings table](client.md#settings)). For example, `$TIMENET_LOCAL_REGISTRY` & `TIMENET_ISOLATION=off`. These apply to every CLI command, and the Python SDK also honors them. -For each build, the used Python and package and TimeNet versions are recorded to the build's -manifest. +The manifest records the Python and package versions used for the build, and the git commit of +`timenet` and `timenet-connectors`. See [`build_env`](manifest.md) for the fields. ## Pipeline diff --git a/docs/manifest.md b/docs/manifest.md index 10eaf3ebc..6fa56e04a 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -39,7 +39,7 @@ Manifest( counts=counts, # ManifestCounts (default: empty) values_backend="parquet", # "parquet" (default) or "zarr" value_encoding={}, # spec_type -> the encoding its shards carry - build_env={}, # {"python", "packages"} (default: empty) + build_env={}, # {"python", "packages", "git"} (default: empty) timef_format_version=1, # required; validated against the supported set {1} ) ``` @@ -57,6 +57,20 @@ selected. The field is empty for a backend that has no such choice. every installed package with its version. `timenet.provenance.build_env` collects this data. Like `value_encoding`, `build_env` is provenance only. No code reads it to interpret the data. +`build_env.git` records the git commit of each TimeNet distribution, keyed by distribution name: + +```json +"git": { + "timenet": {"sha": "4818e37f...", "local_changes": false}, + "timenet-connectors": {"sha": "4818e37f...", "local_changes": false} +} +``` + +A local checkout reads `sha` from git. A wheel reads it from the commit that built the wheel. +`local_changes` is `true` if the package directory had uncommitted changes, untracked files +included. A change elsewhere in the repository does not set it. A distribution whose commit is +unknown is absent. + The values locator is backend-neutral. One schema covers both scalar and multidimensional specs. A multidimensional spec records its shape in `value_shape` and `dimension_names`. It does not need a separate format version. diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index c325c035b..e425a9d66 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -18,13 +18,12 @@ import subprocess # ruff: ignore[suspicious-subprocess-import] - the command is a fixed argument vector import sys import tempfile -from urllib.parse import urlparse -from urllib.request import url2pathname from uv import find_uv_bin from timenet.errors import TimeNetBuildError, TimeNetIsolationError from timenet.manifest import PayloadSource +from timenet.provenance import local_source_path from timenet.types.references import DatasetRef from timenet_connectors.discovery import requirements_for @@ -213,37 +212,9 @@ def _base_args() -> tuple[str, ...]: raise TimeNetIsolationError( f"cannot make a build environment: {name} is not installed in {sys.executable}." ) from exc - source = _local_source_path(dist) + source = local_source_path(dist) if source is not None: args += ["--with-editable", str(source)] else: args += ["--with", f"{name}=={dist.version}"] return tuple(args) - - -def _local_source_path(dist: Distribution) -> Path | None: - """Return the on-disk project directory an editable or local path install points at. - - ``direct_url.json`` records how a distribution was installed. An editable install carries - ``dir_info.editable``. A plain path install (``pip install ./packages/timenet``) records a - ``file://`` URL to the same source tree without that flag. Both let the child build against the - parent's on-disk source, so both return the directory. An index or wheel install has no - ``file://`` directory and returns ``None``, leaving the caller to pin the exact version. - - Args: - dist: The installed distribution. - - Returns: - The source directory for an editable or local path install, else ``None``. - """ - raw = dist.read_text("direct_url.json") - if raw is None: - return None - info = json.loads(raw) - parsed = urlparse(info.get("url", "")) - if parsed.scheme != "file": - return None - path = Path(url2pathname(parsed.path)) - if info.get("dir_info", {}).get("editable") or path.is_dir(): - return path - return None diff --git a/packages/timenet/src/timenet/manifest/__init__.py b/packages/timenet/src/timenet/manifest/__init__.py index 480bdf788..54752de10 100644 --- a/packages/timenet/src/timenet/manifest/__init__.py +++ b/packages/timenet/src/timenet/manifest/__init__.py @@ -2,13 +2,14 @@ from timenet.manifest.counts import ManifestCounts from timenet.manifest.files import FilePart, ManifestFiles -from timenet.manifest.manifest import BuildEnvironment, Manifest, parse_manifest, read_manifest +from timenet.manifest.manifest import BuildEnvironment, GitSource, Manifest, parse_manifest, read_manifest from timenet.manifest.payload_source import PayloadSource __all__ = [ "BuildEnvironment", "FilePart", + "GitSource", "Manifest", "ManifestCounts", "ManifestFiles", diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index c2b099509..9d01d6609 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Annotated, Any, Self -from pydantic import Field, StrictStr, ValidationError, model_validator +from pydantic import Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator from typing_extensions import TypedDict from timenet.errors import ReleaseDifference, TimeFFormatError, TimeNetManifestError @@ -20,11 +20,27 @@ from timenet.values_backends import ValuesBackend +GitSha = Annotated[str, StringConstraints(strict=True, pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")] +"""A full git commit id: 40 hex digits, or 64 in a SHA-256 repository.""" + + +class GitSource(TypedDict): + """The git commit that one TimeNet distribution was built from.""" + + sha: GitSha + """The commit of the checkout, or of the checkout that built the wheel.""" + local_changes: StrictBool + """Whether the package directory had uncommitted changes, untracked files included.""" + + class BuildEnvironment(TypedDict, total=False): """Build-provenance fields recorded in a manifest.""" python: StrictStr packages: dict[StrictStr, StrictStr] + git: dict[StrictStr, GitSource] + """The commit of each TimeNet distribution, keyed by distribution name. A distribution whose + commit is unknown is absent.""" class Manifest(TimeFModel): diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 3da28946d..6d072fea4 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,24 +1,146 @@ """Describe the environment a build ran in, for the manifest's provenance block.""" -from importlib.metadata import distributions +from importlib.metadata import Distribution, PackageNotFoundError, distributions +import json +from pathlib import Path import platform +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors +from urllib.parse import urlparse +from urllib.request import url2pathname -from timenet.manifest import BuildEnvironment +from pydantic import TypeAdapter, ValidationError + +from timenet.errors import TimeNetBuildError +from timenet.manifest import BuildEnvironment, GitSource + + +TIMENET_DISTRIBUTIONS = ("timenet", "timenet-connectors") +"""The distributions whose git commit a build records.""" + +BUILD_INFO_FILE = "_build_info.json" +"""The file that ``scripts/write_build_info.py`` writes into an import package before a wheel is +built. It holds the :class:`~timenet.manifest.GitSource` of the checkout that built the wheel.""" + +_GIT_SOURCE = TypeAdapter(GitSource) def build_env() -> BuildEnvironment: - """Describe this process's Python version and installed packages. + """Describe this process's Python version, installed packages, and TimeNet commits. Under an isolated build the recorded set is exactly the base, the connector's requirements, and their transitives. Under ``--no-isolation`` it is the whole ambient environment, which is noisier but still the truth about what produced the dataset. Returns: - A dict with ``python`` (the interpreter version) and ``packages`` (name -> version, sorted). + A dict with ``python`` (the interpreter version), ``packages`` (name -> version, sorted), and + ``git`` (see :func:`git_sources`). """ packages: dict[str, str] = {} for dist in distributions(): name = dist.metadata["Name"] if name: packages[name] = dist.version - return {"python": platform.python_version(), "packages": dict(sorted(packages.items()))} + return {"python": platform.python_version(), "packages": dict(sorted(packages.items())), "git": git_sources()} + + +def git_sources() -> dict[str, GitSource]: + """Return the git commit of each installed TimeNet distribution. + + A local checkout (an editable or path install) asks git directly. Its ``local_changes`` covers + the package directory only, so a scratch file elsewhere in the repository does not set it. A + wheel carries the commit in its ``_build_info.json``. A distribution with neither is absent. + + Returns: + The commits, keyed by distribution name. + """ + sources: dict[str, GitSource] = {} + for name in TIMENET_DISTRIBUTIONS: + try: + dist = Distribution.from_name(name) + except PackageNotFoundError: + continue + directory = local_source_path(dist) + source = _checkout_source(directory) if directory is not None else _wheel_source(dist) + if source is not None: + sources[name] = source + return sources + + +def local_source_path(dist: Distribution) -> Path | None: + """Return the on-disk project directory an editable or local path install points at. + + ``direct_url.json`` records how a distribution was installed. An editable install carries + ``dir_info.editable``. A plain path install (``pip install ./packages/timenet``) records a + ``file://`` URL to the same source tree without that flag. Both return the directory. An index or + wheel install has no ``file://`` directory and returns ``None``. + + Args: + dist: The installed distribution. + + Returns: + The source directory for an editable or local path install, else ``None``. + """ + raw = dist.read_text("direct_url.json") + if raw is None: + return None + info = json.loads(raw) + parsed = urlparse(info.get("url", "")) + if parsed.scheme != "file": + return None + path = Path(url2pathname(parsed.path)) + if info.get("dir_info", {}).get("editable") or path.is_dir(): + return path + return None + + +def _checkout_source(directory: Path) -> GitSource | None: + """Ask git for the commit of a checkout and whether its package directory has changes. + + Returns: + The commit, or ``None`` if git is missing or the directory is not in a git checkout. + """ + git = shutil.which("git") + if git is None: + return None + try: + sha = _git(git, directory, "rev-parse", "HEAD") + # The pathspec "." scopes the status to the package directory, untracked files included. + status = _git(git, directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + except (OSError, subprocess.CalledProcessError): + return None + return {"sha": sha, "local_changes": bool(status)} + + +def _git(git: str, directory: Path, *args: str) -> str: + """Run one git command in ``directory`` and return its stripped stdout. + + Returns: + The command's stdout without surrounding whitespace. + """ + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + [git, "-C", str(directory), *args], capture_output=True, text=True, check=True + ) + return result.stdout.strip() + + +def _wheel_source(dist: Distribution) -> GitSource | None: + """Read the commit that ``scripts/write_build_info.py`` wrote into a wheel. + + Returns: + The commit, or ``None`` if the distribution carries no build info. + + Raises: + TimeNetBuildError: If the build info file is not a valid commit record. + """ + for file in dist.files or (): + # The file sits at the top of the import package: ``/_build_info.json``. + if file.name == BUILD_INFO_FILE and len(file.parent.parts) == 1: + text = file.read_text(encoding="utf-8") + try: + return _GIT_SOURCE.validate_json(text) + except ValidationError as exc: + raise TimeNetBuildError( + f"{dist.metadata['Name']} carries an invalid {BUILD_INFO_FILE} at {file}: {exc}" + ) from exc + return None diff --git a/packages/timenet/src/timenet/schemas/manifest.schema.json b/packages/timenet/src/timenet/schemas/manifest.schema.json index f75f97d41..28c6f5690 100644 --- a/packages/timenet/src/timenet/schemas/manifest.schema.json +++ b/packages/timenet/src/timenet/schemas/manifest.schema.json @@ -98,6 +98,13 @@ }, "title": "Packages", "type": "object" + }, + "git": { + "additionalProperties": { + "$ref": "#/$defs/GitSource" + }, + "title": "Git", + "type": "object" } }, "title": "BuildEnvironment", @@ -363,6 +370,27 @@ "title": "FilePart", "type": "object" }, + "GitSource": { + "additionalProperties": false, + "description": "The git commit that one TimeNet distribution was built from.", + "properties": { + "sha": { + "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$", + "title": "Sha", + "type": "string" + }, + "local_changes": { + "title": "Local Changes", + "type": "boolean" + } + }, + "required": [ + "sha", + "local_changes" + ], + "title": "GitSource", + "type": "object" + }, "InputModality": { "description": "A kind of input supplied to a task.", "enum": [ diff --git a/packages/timenet/tests/manifest/test_manifest.py b/packages/timenet/tests/manifest/test_manifest.py index 1f9acc0ac..8599a9912 100644 --- a/packages/timenet/tests/manifest/test_manifest.py +++ b/packages/timenet/tests/manifest/test_manifest.py @@ -488,7 +488,8 @@ def test_build_env_defaults_to_empty(): def test_build_env_round_trips(): - m = _replace(_manifest(), build_env={"python": "3.11.9", "packages": {"timenet": "0.1.0"}}) + git = {"timenet": {"sha": "a" * 40, "local_changes": False}} + m = _replace(_manifest(), build_env={"python": "3.11.9", "packages": {"timenet": "0.1.0"}, "git": git}) assert Manifest.model_validate(m.to_dict()).build_env == m.build_env @@ -496,6 +497,8 @@ def test_build_env_round_trips(): "change", [ {"build_env": {"python": 313}}, + {"build_env": {"git": {"timenet": {"sha": "4818e37f", "local_changes": False}}}}, + {"build_env": {"git": {"timenet": {"sha": "a" * 40, "local_changes": "no"}}}}, {"value_encoding": {"ecg_lead": "zip"}}, ], ) diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 074d144fa..bc1e10921 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -1,8 +1,20 @@ +from importlib.metadata import PathDistribution +import json +from pathlib import Path +import shutil +import subprocess import sys +import pytest + +from timenet import provenance +from timenet.errors import TimeNetBuildError from timenet.provenance import build_env +GIT = shutil.which("git") or pytest.fail("the provenance tests need git on PATH") + + def test_build_env_records_the_running_python(): assert build_env()["python"] == ".".join(str(part) for part in sys.version_info[:3]) @@ -11,3 +23,82 @@ def test_build_env_records_installed_packages(): packages = build_env()["packages"] assert packages["timenet"] assert packages == dict(sorted(packages.items())) + + +def test_build_env_records_the_commit_of_this_checkout(): + # The dev environment installs timenet editable from this repository. + head = subprocess.run([GIT, "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip() + assert build_env()["git"]["timenet"]["sha"] == head + + +def _git(directory: Path, *args: str) -> None: + subprocess.run([GIT, "-C", str(directory), *args], capture_output=True, check=True) + + +@pytest.fixture +def checkout(tmp_path: Path) -> Path: + """A committed repository with one package directory, ``packages/pkg``.""" + package = tmp_path / "packages" / "pkg" + package.mkdir(parents=True) + (package / "module.py").write_text("x = 1\n") + (tmp_path / "README.md").write_text("readme\n") + _git(tmp_path, "init", "-q") + _git(tmp_path, "add", ".") + _git(tmp_path, "-c", "user.name=t", "-c", "user.email=t@example.org", "commit", "-qm", "init") + return package + + +def test_a_clean_checkout_has_no_local_changes(checkout): + source = provenance._checkout_source(checkout) + assert source is not None + assert len(source["sha"]) == 40 + assert source["local_changes"] is False + + +@pytest.mark.parametrize("change", ["module.py", "untracked.py"]) +def test_a_change_in_the_package_directory_is_a_local_change(checkout, change): + (checkout / change).write_text("x = 2\n") + source = provenance._checkout_source(checkout) + assert source is not None + assert source["local_changes"] is True + + +def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout): + (checkout.parents[1] / "README.md").write_text("edited\n") + (checkout.parents[1] / "scratch.ipynb").write_text("{}\n") + source = provenance._checkout_source(checkout) + assert source is not None + assert source["local_changes"] is False + + +def test_a_directory_outside_git_has_no_commit(tmp_path): + assert provenance._checkout_source(tmp_path) is None + + +def _wheel_install(site: Path, build_info: str | None) -> PathDistribution: + """Lay out an installed wheel of ``pkg`` with an optional ``_build_info.json``.""" + dist_info = site / "pkg-1.0.dist-info" + dist_info.mkdir(parents=True) + (site / "pkg").mkdir() + (site / "pkg" / "__init__.py").write_text("") + record = ["pkg/__init__.py,,", "pkg-1.0.dist-info/RECORD,,"] + if build_info is not None: + (site / "pkg" / "_build_info.json").write_text(build_info) + record.append("pkg/_build_info.json,,") + (dist_info / "METADATA").write_text("Metadata-Version: 2.1\nName: pkg\nVersion: 1.0\n") + (dist_info / "RECORD").write_text("\n".join(record) + "\n") + return PathDistribution(dist_info) + + +def test_a_wheel_reads_its_commit_from_the_build_info(tmp_path): + info = {"sha": "a" * 40, "local_changes": False} + assert provenance._wheel_source(_wheel_install(tmp_path, json.dumps(info))) == info + + +def test_a_wheel_without_build_info_has_no_commit(tmp_path): + assert provenance._wheel_source(_wheel_install(tmp_path, None)) is None + + +def test_a_wheel_with_invalid_build_info_fails(tmp_path): + with pytest.raises(TimeNetBuildError, match=r"invalid _build_info\.json"): + provenance._wheel_source(_wheel_install(tmp_path, json.dumps({"sha": "main"}))) diff --git a/scripts/write_build_info.py b/scripts/write_build_info.py new file mode 100644 index 000000000..b72db6a64 --- /dev/null +++ b/scripts/write_build_info.py @@ -0,0 +1,61 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Write the git commit of each package into its import package before ``uv build``. + +A wheel has no git directory, so ``timenet.provenance.git_sources`` reads the commit from the +``_build_info.json`` that this script writes. ``local_changes`` covers the package directory only, +the same scope that a local checkout records. The file is git-ignored, so it never reports itself as +a change. ``make build`` and the publish workflow run this script first. + +Usage:: + + uv run scripts/write_build_info.py +""" + +import json +from pathlib import Path +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors + + +ROOT = Path(__file__).resolve().parents[1] +PACKAGES = { + ROOT / "packages" / "timenet": "timenet", + ROOT / "packages" / "timenet-connectors": "timenet_connectors", +} +BUILD_INFO_FILE = "_build_info.json" + + +def git(directory: Path, *args: str) -> str: + """Run one git command in ``directory`` and return its stripped stdout. + + Returns: + The command's stdout without surrounding whitespace. + + Raises: + SystemExit: If git is not on ``PATH``. + """ + executable = shutil.which("git") + if executable is None: + raise SystemExit("git is not on PATH; cannot record the commit of each package") + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + [executable, "-C", str(directory), *args], capture_output=True, text=True, check=True + ) + return result.stdout.strip() + + +def main() -> None: + """Write ``_build_info.json`` into each import package.""" + for directory, module in PACKAGES.items(): + sha = git(directory, "rev-parse", "HEAD") + status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + info = {"sha": sha, "local_changes": bool(status)} + target = directory / "src" / module / BUILD_INFO_FILE + target.write_text(json.dumps(info) + "\n", encoding="utf-8") + print(f"{target.relative_to(ROOT)}: {info}") + + +if __name__ == "__main__": + main() From 21b4de548f82581065d062f502d9b5d361639ba5 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Thu, 5 Nov 2026 16:31:17 +0100 Subject: [PATCH 02/19] fix(provenance): record the commit of the code that runs Review findings on the first version: - A non-editable path install recorded the live checkout, not the installed copy. Only an editable install now asks git. Any other install reads the `_build_info.json` in the package. - A git install (`pip install git+https://...`) now records the commit in its `direct_url.json`. - `make build` left `_build_info.json` in the source tree, so a later plain `uv build` shipped an old commit. `scripts/build_dists.py` replaces `scripts/write_build_info.py`. It writes the file, runs `uv build`, and removes the file again, also on failure. - The publish workflow builds through the script. Its smoke step now makes sure that each artifact reports `$GITHUB_SHA` with no local changes. - git runs with `--no-optional-locks`, so a build never takes the index lock, and with a timeout. - A lookup that fails keeps its reason, so a later check can report it. A malformed `direct_url.json` or `_build_info.json`, or one with unknown keys, raises `TimeNetBuildError`. Tests: `packages/conftest.py` reports a clean commit to every test unless the test has the `real_git` marker. Test results no longer depend on the state of the checkout, and the core suite runs about 40% faster. The provenance tests go through `git_sources()` with fake installed distributions, and switch off the user's git configuration. Part of TNET-165 --- .github/workflows/publish.yml | 22 ++- .gitignore | 2 +- AGENTS.md | 4 +- Makefile | 6 +- docs/manifest.md | 18 +- packages/conftest.py | 19 ++ .../src/timenet_connectors/builder/env.py | 4 +- .../timenet/src/timenet/manifest/manifest.py | 3 +- packages/timenet/src/timenet/provenance.py | 160 +++++++++++---- packages/timenet/tests/test_provenance.py | 184 ++++++++++++------ pyproject.toml | 1 + scripts/build_dists.py | 97 +++++++++ scripts/write_build_info.py | 61 ------ 13 files changed, 405 insertions(+), 176 deletions(-) create mode 100644 packages/conftest.py create mode 100644 scripts/build_dists.py delete mode 100644 scripts/write_build_info.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a45f762c5..95c3799c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,18 +51,18 @@ jobs: uses: actions/checkout@v5 - name: Set up uv uses: astral-sh/setup-uv@v7 - - name: Record the commit in each package - run: uv run scripts/write_build_info.py - name: Build + # build_dists.py records the commit inside each package, so a build can name its code. run: | if [ "${{ inputs.package }}" = "both" ]; then - uv build --all-packages + uv run scripts/build_dists.py --all-packages else - uv build --package "${{ inputs.package }}" + uv run scripts/build_dists.py --package "${{ inputs.package }}" fi - name: Smoke test built artifacts # Import each built wheel and sdist in an isolated environment to catch missing files or - # broken metadata before uploading. timenet-connectors depends on timenet, so hand it the + # broken metadata before uploading. Each artifact must also report this commit, clean, as + # the code it was built from. timenet-connectors depends on timenet, so hand it the # wheel we just built when it is present (a `both` run); otherwise timenet resolves from the # default index, which requires it to be published already. run: | @@ -70,20 +70,26 @@ jobs: shopt -s nullglob timenet_whl=(dist/timenet-*.whl) smoke() { - local module="$1"; shift + local module="$1" dist="$2"; shift 2 local args=() artifact for artifact in "$@"; do args+=(--with "$artifact"); done echo "::group::import $module" uv run --isolated --no-project "${args[@]}" \ python -c "import $module; print('smoke ok:', $module.__name__)" + uv run --isolated --no-project "${args[@]}" python -c " + from timenet.provenance import git_sources + commit = git_sources().get('$dist') + expected = {'sha': '$GITHUB_SHA', 'local_changes': False} + assert commit == expected, f'$dist records {commit}, expected {expected}' + print('commit ok:', commit)" echo "::endgroup::" } for artifact in dist/*; do case "$artifact" in dist/timenet-*.whl | dist/timenet-*.tar.gz) - smoke timenet "$artifact" ;; + smoke timenet timenet "$artifact" ;; dist/timenet_connectors-*.whl | dist/timenet_connectors-*.tar.gz) - smoke timenet_connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;; + smoke timenet_connectors timenet-connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;; esac done - name: Publish diff --git a/.gitignore b/.gitignore index a44700f6e..785af413f 100644 --- a/.gitignore +++ b/.gitignore @@ -48,5 +48,5 @@ docs/api/ # Local superpowers planning artifacts, not part of the repo docs/superpowers/ -# commit record written into each import package before a wheel build (see scripts/write_build_info.py) +# commit record written into each import package before a wheel build (scripts/build_dists.py removes it after the build) packages/*/src/*/_build_info.json diff --git a/AGENTS.md b/AGENTS.md index d9e1d389c..9261a7f54 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,9 @@ workspace definition and the shared ruff/ty/pytest config; per-package workspace extras (`timenet[cli,torch]`). It does not install connector dependencies. Each connector declares its own in a `requirements.txt`, and `make test-connectors` runs that connector's tests and type-check in an environment built from it. -- Build distributables with `make build` (`uv build --package ` per member). +- Build distributables with `make build`. It runs `scripts/build_dists.py`, which records the git + commit inside each package and then runs `uv build --package ` per member. A plain + `uv build` gives a wheel whose commit is unknown, so build release artifacts through the script. - For one-off scripts, use inline `uv` metadata and run with `uv run `. Never `pip install`. - Keep `uv.lock` committed; the `uv-lock` pre-commit hook enforces freshness. diff --git a/Makefile b/Makefile index 19ee78e73..4c688e34d 100644 --- a/Makefile +++ b/Makefile @@ -55,10 +55,10 @@ license-check: --fail-on="GPL;LGPL;AGPL;SSPL;EUPL;CDDL;OSL" \ --format=markdown +# Builds through scripts/build_dists.py, so each wheel records the commit it was built from. build: - uv run scripts/write_build_info.py - uv build --package timenet - uv build --package timenet-connectors + uv run scripts/build_dists.py --package timenet + uv run scripts/build_dists.py --package timenet-connectors check: schemas-check uv run ruff format . diff --git a/docs/manifest.md b/docs/manifest.md index 6fa56e04a..df351f03b 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -66,10 +66,20 @@ every installed package with its version. `timenet.provenance.build_env` collect } ``` -A local checkout reads `sha` from git. A wheel reads it from the commit that built the wheel. -`local_changes` is `true` if the package directory had uncommitted changes, untracked files -included. A change elsewhere in the repository does not set it. A distribution whose commit is -unknown is absent. +Each distribution records the commit of the code that runs: + +- An editable install, such as a workspace checkout, asks git. `local_changes` is `true` if the + package directory has uncommitted changes, untracked files included. A change elsewhere in the + repository does not set it. +- A git install (`pip install git+https://...`) records the commit that pip installed, with + `local_changes` set to `false`. +- Any other install, such as a wheel or `pip install ./packages/timenet`, reads the + `_build_info.json` file in the package. `scripts/build_dists.py` writes that file while it + builds the wheel, and records the state of the checkout at that time. `make build` and the + publish workflow run the script. + +A distribution whose commit is unknown is absent, for example a wheel that a plain `uv build` +produced. The values locator is backend-neutral. One schema covers both scalar and multidimensional specs. A multidimensional spec records its shape in `value_shape` and `dimension_names`. It does not need a diff --git a/packages/conftest.py b/packages/conftest.py new file mode 100644 index 000000000..da9bdb489 --- /dev/null +++ b/packages/conftest.py @@ -0,0 +1,19 @@ +"""Shared pytest configuration for every workspace package's tests. + +Each manifest write asks git for the commit of the TimeNet distributions. Real git calls would make +a test's result depend on the state of this checkout, and they cost seconds across the suite. So +every test sees a clean, committed checkout, unless it carries the ``real_git`` marker. +""" + +import pytest + +from timenet import provenance + + +_CLEAN = {"sha": "0" * 40, "local_changes": False} + + +@pytest.fixture(autouse=True) +def _committed_checkout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + if request.node.get_closest_marker("real_git") is None: + monkeypatch.setattr(provenance, "_read_git", lambda: dict.fromkeys(provenance.TIMENET_DISTRIBUTIONS, _CLEAN)) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index e425a9d66..06c7cbc27 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -23,12 +23,12 @@ from timenet.errors import TimeNetBuildError, TimeNetIsolationError from timenet.manifest import PayloadSource -from timenet.provenance import local_source_path +from timenet.provenance import TIMENET_DISTRIBUTIONS, local_source_path from timenet.types.references import DatasetRef from timenet_connectors.discovery import requirements_for -BASE_DISTRIBUTIONS = ("timenet", "timenet-connectors") +BASE_DISTRIBUTIONS = TIMENET_DISTRIBUTIONS """Pinned to the parent's own versions, so the code that writes the TimeF bytes is the code that reads them back. ``timenet-connectors`` depends on ``timenet[build]``, so the card-YAML dependencies arrive transitively and the extra never needs naming here.""" diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index 9d01d6609..799fb872f 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Annotated, Any, Self -from pydantic import Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator +from pydantic import ConfigDict, Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator, with_config from typing_extensions import TypedDict from timenet.errors import ReleaseDifference, TimeFFormatError, TimeNetManifestError @@ -24,6 +24,7 @@ """A full git commit id: 40 hex digits, or 64 in a SHA-256 repository.""" +@with_config(ConfigDict(extra="forbid")) class GitSource(TypedDict): """The git commit that one TimeNet distribution was built from.""" diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 6d072fea4..f574603f5 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,5 +1,6 @@ """Describe the environment a build ran in, for the manifest's provenance block.""" +from dataclasses import dataclass from importlib.metadata import Distribution, PackageNotFoundError, distributions import json from pathlib import Path @@ -19,10 +20,19 @@ """The distributions whose git commit a build records.""" BUILD_INFO_FILE = "_build_info.json" -"""The file that ``scripts/write_build_info.py`` writes into an import package before a wheel is -built. It holds the :class:`~timenet.manifest.GitSource` of the checkout that built the wheel.""" +"""The file that ``scripts/build_dists.py`` writes into an import package while it builds a wheel. +It holds the :class:`~timenet.manifest.GitSource` of the checkout that built the wheel. The script +keeps the same name in its own constant.""" _GIT_SOURCE = TypeAdapter(GitSource) +_GIT_TIMEOUT_S = 30 + + +@dataclass(frozen=True) +class UnknownCommit: + """Why the commit of an installed TimeNet distribution is unknown.""" + + reason: str def build_env() -> BuildEnvironment: @@ -45,26 +55,22 @@ def build_env() -> BuildEnvironment: def git_sources() -> dict[str, GitSource]: - """Return the git commit of each installed TimeNet distribution. + """Return the git commit of each installed TimeNet distribution whose commit is known. + + Each distribution reports the commit of the code that runs: - A local checkout (an editable or path install) asks git directly. Its ``local_changes`` covers - the package directory only, so a scratch file elsewhere in the repository does not set it. A - wheel carries the commit in its ``_build_info.json``. A distribution with neither is absent. + - An editable install asks git. ``local_changes`` covers the package directory only, so a + scratch file elsewhere in the repository does not set it. + - A git install (``pip install git+https://...``) reports the commit that pip recorded. + - Any other install, such as a wheel or ``pip install ./packages/timenet``, reads the + ``_build_info.json`` that the build wrote into the package. + + A distribution whose commit is unknown is absent. Returns: The commits, keyed by distribution name. """ - sources: dict[str, GitSource] = {} - for name in TIMENET_DISTRIBUTIONS: - try: - dist = Distribution.from_name(name) - except PackageNotFoundError: - continue - directory = local_source_path(dist) - source = _checkout_source(directory) if directory is not None else _wheel_source(dist) - if source is not None: - sources[name] = source - return sources + return {name: commit for name, commit in _read_git().items() if not isinstance(commit, UnknownCommit)} def local_source_path(dist: Distribution) -> Path | None: @@ -81,10 +87,9 @@ def local_source_path(dist: Distribution) -> Path | None: Returns: The source directory for an editable or local path install, else ``None``. """ - raw = dist.read_text("direct_url.json") - if raw is None: + info = _direct_url(dist) + if info is None: return None - info = json.loads(raw) parsed = urlparse(info.get("url", "")) if parsed.scheme != "file": return None @@ -94,53 +99,134 @@ def local_source_path(dist: Distribution) -> Path | None: return None -def _checkout_source(directory: Path) -> GitSource | None: +def _read_git() -> dict[str, GitSource | UnknownCommit]: + """Look up the commit of each installed TimeNet distribution. This is the only call that runs git. + + Returns: + The commit or the reason it is unknown, keyed by distribution name. A distribution that is + not installed is absent. + """ + commits: dict[str, GitSource | UnknownCommit] = {} + for name in TIMENET_DISTRIBUTIONS: + try: + dist = Distribution.from_name(name) + except PackageNotFoundError: + continue + commits[name] = _commit_of(dist) + return commits + + +def _commit_of(dist: Distribution) -> GitSource | UnknownCommit: + """Return the commit of the code that one installed distribution runs. + + Returns: + The commit, or the reason it is unknown. + + Raises: + TimeNetBuildError: If a recorded commit is not a valid commit record. + """ # ruff: ignore[docstring-extraneous-exception] - raised by _direct_url, _validated, and _wheel_commit + info = _direct_url(dist) + if info is not None: + vcs = info.get("vcs_info", {}) + if vcs.get("vcs") == "git" and "commit_id" in vcs: + return _validated(dist, {"sha": vcs["commit_id"], "local_changes": False}, "direct_url.json") + if info.get("dir_info", {}).get("editable"): + directory = local_source_path(dist) + if directory is not None: + return _checkout_commit(directory) + return _wheel_commit(dist) + + +def _direct_url(dist: Distribution) -> dict | None: + """Read the PEP 610 ``direct_url.json`` of a distribution. + + Returns: + The parsed record, or ``None`` for an install from an index. + + Raises: + TimeNetBuildError: If the record is not a JSON object. + """ + raw = dist.read_text("direct_url.json") + if raw is None: + return None + try: + info = json.loads(raw) + except json.JSONDecodeError as exc: + raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed direct_url.json: {exc}") from exc + if not isinstance(info, dict): + raise TimeNetBuildError(f"{dist.metadata['Name']} has a direct_url.json that is not a JSON object") + return info + + +def _checkout_commit(directory: Path) -> GitSource | UnknownCommit: """Ask git for the commit of a checkout and whether its package directory has changes. Returns: - The commit, or ``None`` if git is missing or the directory is not in a git checkout. + The commit, or the reason git could not give it. """ git = shutil.which("git") if git is None: - return None + return UnknownCommit("git is not on PATH") try: sha = _git(git, directory, "rev-parse", "HEAD") # The pathspec "." scopes the status to the package directory, untracked files included. status = _git(git, directory, "status", "--porcelain", "--untracked-files=all", "--", ".") - except (OSError, subprocess.CalledProcessError): - return None + except subprocess.CalledProcessError as exc: + return UnknownCommit(f"git failed in {directory}: {(exc.stderr or '').strip() or exc}") + except (OSError, subprocess.TimeoutExpired) as exc: + return UnknownCommit(f"git failed in {directory}: {exc}") return {"sha": sha, "local_changes": bool(status)} def _git(git: str, directory: Path, *args: str) -> str: """Run one git command in ``directory`` and return its stripped stdout. + ``--no-optional-locks`` keeps ``git status`` from refreshing the index, so a build never takes + the index lock while the user runs git. + Returns: The command's stdout without surrounding whitespace. """ result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] - [git, "-C", str(directory), *args], capture_output=True, text=True, check=True + [git, "--no-optional-locks", "-C", str(directory), *args], + capture_output=True, + text=True, + check=True, + timeout=_GIT_TIMEOUT_S, ) return result.stdout.strip() -def _wheel_source(dist: Distribution) -> GitSource | None: - """Read the commit that ``scripts/write_build_info.py`` wrote into a wheel. +def _wheel_commit(dist: Distribution) -> GitSource | UnknownCommit: + """Read the commit that ``scripts/build_dists.py`` wrote into the installed package. Returns: - The commit, or ``None`` if the distribution carries no build info. + The commit, or the reason it is unknown. Raises: - TimeNetBuildError: If the build info file is not a valid commit record. + TimeNetBuildError: If the file is not valid JSON or not a valid commit record. """ for file in dist.files or (): # The file sits at the top of the import package: ``/_build_info.json``. if file.name == BUILD_INFO_FILE and len(file.parent.parts) == 1: - text = file.read_text(encoding="utf-8") try: - return _GIT_SOURCE.validate_json(text) - except ValidationError as exc: - raise TimeNetBuildError( - f"{dist.metadata['Name']} carries an invalid {BUILD_INFO_FILE} at {file}: {exc}" - ) from exc - return None + record = json.loads(file.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed {file}: {exc}") from exc + return _validated(dist, record, str(file)) + return UnknownCommit(f"the installed package carries no {BUILD_INFO_FILE}") + + +def _validated(dist: Distribution, record: object, where: str) -> GitSource: + """Validate a recorded commit. + + Returns: + The commit. + + Raises: + TimeNetBuildError: If the record is not a valid commit record. + """ + try: + return _GIT_SOURCE.validate_python(record) + except ValidationError as exc: + raise TimeNetBuildError(f"{dist.metadata['Name']} records an invalid commit in {where}: {exc}") from exc diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index bc1e10921..307b1df89 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -1,5 +1,6 @@ -from importlib.metadata import PathDistribution +from importlib.metadata import Distribution, PackageNotFoundError, PathDistribution import json +import os from pathlib import Path import shutil import subprocess @@ -7,12 +8,9 @@ import pytest -from timenet import provenance +import timenet from timenet.errors import TimeNetBuildError -from timenet.provenance import build_env - - -GIT = shutil.which("git") or pytest.fail("the provenance tests need git on PATH") +from timenet.provenance import build_env, git_sources def test_build_env_records_the_running_python(): @@ -25,80 +23,150 @@ def test_build_env_records_installed_packages(): assert packages == dict(sorted(packages.items())) -def test_build_env_records_the_commit_of_this_checkout(): - # The dev environment installs timenet editable from this repository. - head = subprocess.run([GIT, "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip() - assert build_env()["git"]["timenet"]["sha"] == head +@pytest.fixture +def git(monkeypatch: pytest.MonkeyPatch) -> str: + """The git executable, with the user's and the system's git configuration switched off.""" + executable = shutil.which("git") + if executable is None: + pytest.skip("git is not on PATH") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + return executable + +def _run(git: str, directory: Path, *args: str) -> str: + return subprocess.run([git, "-C", str(directory), *args], capture_output=True, text=True, check=True).stdout.strip() -def _git(directory: Path, *args: str) -> None: - subprocess.run([GIT, "-C", str(directory), *args], capture_output=True, check=True) + +@pytest.mark.real_git +def test_build_env_records_the_commit_of_this_checkout(git): + # The dev environment installs timenet editable from this repository. + head = _run(git, Path(timenet.__file__).parent, "rev-parse", "HEAD") + assert build_env()["git"]["timenet"]["sha"] == head @pytest.fixture -def checkout(tmp_path: Path) -> Path: +def checkout(tmp_path: Path, git: str) -> Path: """A committed repository with one package directory, ``packages/pkg``.""" - package = tmp_path / "packages" / "pkg" + repo = tmp_path / "repo" + package = repo / "packages" / "pkg" package.mkdir(parents=True) (package / "module.py").write_text("x = 1\n") - (tmp_path / "README.md").write_text("readme\n") - _git(tmp_path, "init", "-q") - _git(tmp_path, "add", ".") - _git(tmp_path, "-c", "user.name=t", "-c", "user.email=t@example.org", "commit", "-qm", "init") + (repo / "README.md").write_text("readme\n") + _run(git, repo, "init", "-q") + # Hide untracked files from a plain status, so the lookup must ask for them itself. + _run(git, repo, "config", "status.showUntrackedFiles", "no") + _run(git, repo, "add", ".") + _run(git, repo, "-c", "user.name=t", "-c", "user.email=t@example.org", "commit", "-qm", "init") return package -def test_a_clean_checkout_has_no_local_changes(checkout): - source = provenance._checkout_source(checkout) - assert source is not None - assert len(source["sha"]) == 40 - assert source["local_changes"] is False +def _install(site: Path, *, direct_url: dict | None = None, build_info: str | None = None) -> PathDistribution: + """Lay out an installed ``timenet`` with an optional ``direct_url.json`` and build info.""" + dist_info = site / "timenet-1.0.dist-info" + dist_info.mkdir(parents=True) + (site / "timenet").mkdir() + record = ["timenet/__init__.py,,", "timenet-1.0.dist-info/RECORD,,"] + if build_info is not None: + (site / "timenet" / "_build_info.json").write_text(build_info) + record.append("timenet/_build_info.json,,") + if direct_url is not None: + (dist_info / "direct_url.json").write_text(json.dumps(direct_url)) + (dist_info / "METADATA").write_text("Metadata-Version: 2.1\nName: timenet\nVersion: 1.0\n") + (dist_info / "RECORD").write_text("\n".join(record) + "\n") + return PathDistribution(dist_info) -@pytest.mark.parametrize("change", ["module.py", "untracked.py"]) -def test_a_change_in_the_package_directory_is_a_local_change(checkout, change): - (checkout / change).write_text("x = 2\n") - source = provenance._checkout_source(checkout) - assert source is not None - assert source["local_changes"] is True +@pytest.fixture +def installed(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Make a fake ``timenet`` the only installed TimeNet distribution. Call it to lay one out.""" + def install(**kwargs) -> None: + dist = _install(tmp_path / "site", **kwargs) -def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout): - (checkout.parents[1] / "README.md").write_text("edited\n") - (checkout.parents[1] / "scratch.ipynb").write_text("{}\n") - source = provenance._checkout_source(checkout) - assert source is not None - assert source["local_changes"] is False + def from_name(name: str) -> Distribution: + if name != "timenet": + raise PackageNotFoundError(name) + return dist + monkeypatch.setattr(Distribution, "from_name", staticmethod(from_name)) -def test_a_directory_outside_git_has_no_commit(tmp_path): - assert provenance._checkout_source(tmp_path) is None + return install -def _wheel_install(site: Path, build_info: str | None) -> PathDistribution: - """Lay out an installed wheel of ``pkg`` with an optional ``_build_info.json``.""" - dist_info = site / "pkg-1.0.dist-info" - dist_info.mkdir(parents=True) - (site / "pkg").mkdir() - (site / "pkg" / "__init__.py").write_text("") - record = ["pkg/__init__.py,,", "pkg-1.0.dist-info/RECORD,,"] - if build_info is not None: - (site / "pkg" / "_build_info.json").write_text(build_info) - record.append("pkg/_build_info.json,,") - (dist_info / "METADATA").write_text("Metadata-Version: 2.1\nName: pkg\nVersion: 1.0\n") - (dist_info / "RECORD").write_text("\n".join(record) + "\n") - return PathDistribution(dist_info) +def _editable(directory: Path) -> dict: + return {"url": directory.as_uri(), "dir_info": {"editable": True}} -def test_a_wheel_reads_its_commit_from_the_build_info(tmp_path): - info = {"sha": "a" * 40, "local_changes": False} - assert provenance._wheel_source(_wheel_install(tmp_path, json.dumps(info))) == info +@pytest.mark.real_git +def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, installed): + installed(direct_url=_editable(checkout)) + head = _run(git, checkout, "rev-parse", "HEAD") + assert git_sources() == {"timenet": {"sha": head, "local_changes": False}} + +@pytest.mark.real_git +@pytest.mark.parametrize("change", ["module.py", "untracked.py"]) +def test_a_change_in_the_package_directory_is_a_local_change(checkout, installed, change): + installed(direct_url=_editable(checkout)) + (checkout / change).write_text("x = 2\n") + assert git_sources()["timenet"]["local_changes"] is True + + +@pytest.mark.real_git +def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout, installed): + installed(direct_url=_editable(checkout)) + (checkout.parents[1] / "README.md").write_text("edited\n") + (checkout.parents[1] / "scratch.ipynb").write_text("{}\n") + assert git_sources()["timenet"]["local_changes"] is False -def test_a_wheel_without_build_info_has_no_commit(tmp_path): - assert provenance._wheel_source(_wheel_install(tmp_path, None)) is None +@pytest.mark.real_git +def test_an_editable_install_outside_git_has_no_commit(git, tmp_path, installed): + outside = tmp_path / "outside" + outside.mkdir() + installed(direct_url=_editable(outside)) + assert git_sources() == {} -def test_a_wheel_with_invalid_build_info_fails(tmp_path): - with pytest.raises(TimeNetBuildError, match=r"invalid _build_info\.json"): - provenance._wheel_source(_wheel_install(tmp_path, json.dumps({"sha": "main"}))) + +@pytest.mark.real_git +def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): + info = {"sha": "a" * 40, "local_changes": False} + installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=json.dumps(info)) + (checkout / "module.py").write_text("x = 2\n") + assert git_sources() == {"timenet": info} + + +@pytest.mark.real_git +def test_a_git_install_records_the_commit_that_pip_installed(installed): + sha = "b" * 40 + installed(direct_url={"url": "https://example.org/timenet.git", "vcs_info": {"vcs": "git", "commit_id": sha}}) + assert git_sources() == {"timenet": {"sha": sha, "local_changes": False}} + + +@pytest.mark.real_git +def test_a_wheel_reads_its_commit_from_the_build_info(installed): + info = {"sha": "a" * 40, "local_changes": True} + installed(build_info=json.dumps(info)) + assert git_sources() == {"timenet": info} + + +@pytest.mark.real_git +def test_a_wheel_without_build_info_has_no_commit(installed): + installed() + assert git_sources() == {} + + +@pytest.mark.real_git +@pytest.mark.parametrize( + "build_info", + [ + json.dumps({"sha": "main", "local_changes": False}), + json.dumps({"sha": "a" * 40, "local_changes": False, "branch": "main"}), + "not json", + ], +) +def test_a_wheel_with_invalid_build_info_fails(installed, build_info): + installed(build_info=build_info) + with pytest.raises(TimeNetBuildError, match=r"_build_info\.json"): + git_sources() diff --git a/pyproject.toml b/pyproject.toml index 84360d028..3de3c56bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,7 @@ testpaths = ["packages", "benchmarks/end_to_end/tests"] markers = [ "slow: full-disk round-trip and property tests (deselect with '-m \"not slow\"')", "value_dtypes: scalar value-dtype round-trip tests (run with '-m value_dtypes')", + "real_git: run real git for provenance instead of the clean commit that packages/conftest.py reports", ] [tool.ty.src] diff --git a/scripts/build_dists.py b/scripts/build_dists.py new file mode 100644 index 000000000..0e53bceaa --- /dev/null +++ b/scripts/build_dists.py @@ -0,0 +1,97 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Build distributions with the git commit of each package recorded inside it. + +A wheel has no git directory, so ``timenet.provenance.git_sources`` reads the commit from the +``_build_info.json`` that this script writes into each import package. ``local_changes`` covers the +package directory only, the same scope that a local checkout records. The script then runs +``uv build`` with the given arguments, and removes the files again even if the build fails. A +later plain ``uv build`` therefore never ships an old record. ``make build`` and the publish +workflow run this script. + +Usage:: + + uv run scripts/build_dists.py --all-packages + uv run scripts/build_dists.py --package timenet --out-dir dist +""" + +import json +import os +from pathlib import Path +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors +import sys + + +ROOT = Path(__file__).resolve().parents[1] +PACKAGES = { + ROOT / "packages" / "timenet": "timenet", + ROOT / "packages" / "timenet-connectors": "timenet_connectors", +} +# Keep in step with timenet.provenance.BUILD_INFO_FILE, which reads the file. +BUILD_INFO_FILE = "_build_info.json" + + +def executable(name: str) -> str: + """Return the absolute path of a command on ``PATH``. + + Returns: + The path of the command. + + Raises: + SystemExit: If the command is not on ``PATH``. + """ + path = shutil.which(name) + if path is None: + raise SystemExit(f"{name} is not on PATH") + return path + + +def git(directory: Path, *args: str) -> str: + """Run one git command in ``directory`` and return its stripped stdout. + + Returns: + The command's stdout without surrounding whitespace. + + Raises: + SystemExit: If git fails, for example outside a git checkout. + """ + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + [executable("git"), "-C", str(directory), *args], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + raise SystemExit(f"git {' '.join(args)} failed in {directory}: {result.stderr.strip()}") + return result.stdout.strip() + + +def main(build_args: list[str]) -> int: + """Write ``_build_info.json`` into each import package, build, and remove the files. + + Args: + build_args: The arguments for ``uv build``. + + Returns: + The exit code of ``uv build``. + """ + written: list[Path] = [] + try: + for directory, module in PACKAGES.items(): + sha = git(directory, "rev-parse", "HEAD") + status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + info = {"sha": sha, "local_changes": bool(status)} + target = directory / "src" / module / BUILD_INFO_FILE + target.write_text(json.dumps(info) + "\n", encoding="utf-8") + written.append(target) + print(f"{target.relative_to(ROOT)}: {info}", file=sys.stderr) + # uv run sets UV to its own executable. + uv = os.environ.get("UV") or executable("uv") + return subprocess.run([uv, "build", *build_args], cwd=ROOT, check=False).returncode # ruff: ignore[subprocess-without-shell-equals-true] + finally: + for path in written: + path.unlink(missing_ok=True) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/write_build_info.py b/scripts/write_build_info.py deleted file mode 100644 index b72db6a64..000000000 --- a/scripts/write_build_info.py +++ /dev/null @@ -1,61 +0,0 @@ -# /// script -# requires-python = ">=3.11" -# dependencies = [] -# /// -"""Write the git commit of each package into its import package before ``uv build``. - -A wheel has no git directory, so ``timenet.provenance.git_sources`` reads the commit from the -``_build_info.json`` that this script writes. ``local_changes`` covers the package directory only, -the same scope that a local checkout records. The file is git-ignored, so it never reports itself as -a change. ``make build`` and the publish workflow run this script first. - -Usage:: - - uv run scripts/write_build_info.py -""" - -import json -from pathlib import Path -import shutil -import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors - - -ROOT = Path(__file__).resolve().parents[1] -PACKAGES = { - ROOT / "packages" / "timenet": "timenet", - ROOT / "packages" / "timenet-connectors": "timenet_connectors", -} -BUILD_INFO_FILE = "_build_info.json" - - -def git(directory: Path, *args: str) -> str: - """Run one git command in ``directory`` and return its stripped stdout. - - Returns: - The command's stdout without surrounding whitespace. - - Raises: - SystemExit: If git is not on ``PATH``. - """ - executable = shutil.which("git") - if executable is None: - raise SystemExit("git is not on PATH; cannot record the commit of each package") - result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] - [executable, "-C", str(directory), *args], capture_output=True, text=True, check=True - ) - return result.stdout.strip() - - -def main() -> None: - """Write ``_build_info.json`` into each import package.""" - for directory, module in PACKAGES.items(): - sha = git(directory, "rev-parse", "HEAD") - status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") - info = {"sha": sha, "local_changes": bool(status)} - target = directory / "src" / module / BUILD_INFO_FILE - target.write_text(json.dumps(info) + "\n", encoding="utf-8") - print(f"{target.relative_to(ROOT)}: {info}") - - -if __name__ == "__main__": - main() From b280788f5bbdb9404684b7f9bd2f848708f260ed Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 6 Nov 2026 14:48:27 +0100 Subject: [PATCH 03/19] test(provenance): trim the provenance tests Drop the test that ran real git against this repository. Its one check, that build_env() carries the git block, moves into the editable-install test. Drop the invalid-sha case, which the manifest tests already cover, and keep the extra-key case, the only test of the forbid rule on GitSource. Remove a setup line that no assertion needed. Part of TNET-165 --- packages/timenet/tests/test_provenance.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 307b1df89..c5a78de4b 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -8,7 +8,6 @@ import pytest -import timenet from timenet.errors import TimeNetBuildError from timenet.provenance import build_env, git_sources @@ -38,13 +37,6 @@ def _run(git: str, directory: Path, *args: str) -> str: return subprocess.run([git, "-C", str(directory), *args], capture_output=True, text=True, check=True).stdout.strip() -@pytest.mark.real_git -def test_build_env_records_the_commit_of_this_checkout(git): - # The dev environment installs timenet editable from this repository. - head = _run(git, Path(timenet.__file__).parent, "rev-parse", "HEAD") - assert build_env()["git"]["timenet"]["sha"] == head - - @pytest.fixture def checkout(tmp_path: Path, git: str) -> Path: """A committed repository with one package directory, ``packages/pkg``.""" @@ -102,7 +94,7 @@ def _editable(directory: Path) -> dict: def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, installed): installed(direct_url=_editable(checkout)) head = _run(git, checkout, "rev-parse", "HEAD") - assert git_sources() == {"timenet": {"sha": head, "local_changes": False}} + assert build_env()["git"] == {"timenet": {"sha": head, "local_changes": False}} @pytest.mark.real_git @@ -133,7 +125,6 @@ def test_an_editable_install_outside_git_has_no_commit(git, tmp_path, installed) def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): info = {"sha": "a" * 40, "local_changes": False} installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=json.dumps(info)) - (checkout / "module.py").write_text("x = 2\n") assert git_sources() == {"timenet": info} @@ -161,7 +152,8 @@ def test_a_wheel_without_build_info_has_no_commit(installed): @pytest.mark.parametrize( "build_info", [ - json.dumps({"sha": "main", "local_changes": False}), + # The manifest forbids extra keys for every nested block, but this standalone check must + # forbid them itself. json.dumps({"sha": "a" * 40, "local_changes": False, "branch": "main"}), "not json", ], From 2d78834b52c5ab2f387a496a94da84f9f2fe98b4 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 6 Nov 2026 15:49:31 +0100 Subject: [PATCH 04/19] fix(provenance): address review of the commit record - Allow only 40-digit SHA-1 commit ids, because TimeNet uses SHA-1. - Shorten the docstrings of the provenance helpers and of `scripts/build_dists.py` to one line. Ruff now exempts a one-line docstring from the Returns and Raises sections (`ignore-one-line-docstrings`). - Rename `_validated` to `_validate_recorded_commit`. - `scripts/build_dists.py` registers each record for cleanup before it writes it, so a write that fails halfway leaves nothing behind. It compares the commit and `git status` of each package before and after `uv build`, and fails if a file changed during the build. - Simplify the build bullet in AGENTS.md. Part of TNET-165 --- AGENTS.md | 6 +- .../timenet/src/timenet/manifest/manifest.py | 4 +- packages/timenet/src/timenet/provenance.py | 70 ++++--------------- packages/timenet/src/timenet/registry/s3.py | 1 - .../src/timenet/schemas/manifest.schema.json | 2 +- pyproject.toml | 4 ++ scripts/build_dists.py | 64 +++++++---------- 7 files changed, 48 insertions(+), 103 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9261a7f54..65da0ba29 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,9 +60,9 @@ workspace definition and the shared ruff/ty/pytest config; per-package workspace extras (`timenet[cli,torch]`). It does not install connector dependencies. Each connector declares its own in a `requirements.txt`, and `make test-connectors` runs that connector's tests and type-check in an environment built from it. -- Build distributables with `make build`. It runs `scripts/build_dists.py`, which records the git - commit inside each package and then runs `uv build --package ` per member. A plain - `uv build` gives a wheel whose commit is unknown, so build release artifacts through the script. +- Build distributables with `make build`. It runs the build through a proxy script, which records + the git commit inside each package for provenance, before running `uv build --package ` + per member. - For one-off scripts, use inline `uv` metadata and run with `uv run `. Never `pip install`. - Keep `uv.lock` committed; the `uv-lock` pre-commit hook enforces freshness. diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index 799fb872f..f7743ef99 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -20,8 +20,8 @@ from timenet.values_backends import ValuesBackend -GitSha = Annotated[str, StringConstraints(strict=True, pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")] -"""A full git commit id: 40 hex digits, or 64 in a SHA-256 repository.""" +GitSha = Annotated[str, StringConstraints(strict=True, pattern=r"^[0-9a-f]{40}$")] +"""A full SHA-1 git commit id: 40 lowercase hex digits.""" @with_config(ConfigDict(extra="forbid")) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index f574603f5..e5417d100 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -20,9 +20,7 @@ """The distributions whose git commit a build records.""" BUILD_INFO_FILE = "_build_info.json" -"""The file that ``scripts/build_dists.py`` writes into an import package while it builds a wheel. -It holds the :class:`~timenet.manifest.GitSource` of the checkout that built the wheel. The script -keeps the same name in its own constant.""" +"""The commit record that ``scripts/build_dists.py`` writes into each package of a wheel.""" _GIT_SOURCE = TypeAdapter(GitSource) _GIT_TIMEOUT_S = 30 @@ -100,12 +98,7 @@ def local_source_path(dist: Distribution) -> Path | None: def _read_git() -> dict[str, GitSource | UnknownCommit]: - """Look up the commit of each installed TimeNet distribution. This is the only call that runs git. - - Returns: - The commit or the reason it is unknown, keyed by distribution name. A distribution that is - not installed is absent. - """ + """Look up the commit of each installed TimeNet distribution. This is the only call that runs git.""" commits: dict[str, GitSource | UnknownCommit] = {} for name in TIMENET_DISTRIBUTIONS: try: @@ -117,19 +110,12 @@ def _read_git() -> dict[str, GitSource | UnknownCommit]: def _commit_of(dist: Distribution) -> GitSource | UnknownCommit: - """Return the commit of the code that one installed distribution runs. - - Returns: - The commit, or the reason it is unknown. - - Raises: - TimeNetBuildError: If a recorded commit is not a valid commit record. - """ # ruff: ignore[docstring-extraneous-exception] - raised by _direct_url, _validated, and _wheel_commit + """Return the commit of the code that one installed distribution runs, or why it is unknown.""" info = _direct_url(dist) if info is not None: vcs = info.get("vcs_info", {}) if vcs.get("vcs") == "git" and "commit_id" in vcs: - return _validated(dist, {"sha": vcs["commit_id"], "local_changes": False}, "direct_url.json") + return _validate_recorded_commit(dist, {"sha": vcs["commit_id"], "local_changes": False}, "direct_url.json") if info.get("dir_info", {}).get("editable"): directory = local_source_path(dist) if directory is not None: @@ -138,14 +124,7 @@ def _commit_of(dist: Distribution) -> GitSource | UnknownCommit: def _direct_url(dist: Distribution) -> dict | None: - """Read the PEP 610 ``direct_url.json`` of a distribution. - - Returns: - The parsed record, or ``None`` for an install from an index. - - Raises: - TimeNetBuildError: If the record is not a JSON object. - """ + """Read the PEP 610 ``direct_url.json`` of a distribution, or ``None`` for an index install.""" raw = dist.read_text("direct_url.json") if raw is None: return None @@ -159,11 +138,7 @@ def _direct_url(dist: Distribution) -> dict | None: def _checkout_commit(directory: Path) -> GitSource | UnknownCommit: - """Ask git for the commit of a checkout and whether its package directory has changes. - - Returns: - The commit, or the reason git could not give it. - """ + """Ask git for the commit of a checkout and whether its package directory has changes.""" git = shutil.which("git") if git is None: return UnknownCommit("git is not on PATH") @@ -179,15 +154,10 @@ def _checkout_commit(directory: Path) -> GitSource | UnknownCommit: def _git(git: str, directory: Path, *args: str) -> str: - """Run one git command in ``directory`` and return its stripped stdout. - - ``--no-optional-locks`` keeps ``git status`` from refreshing the index, so a build never takes - the index lock while the user runs git. - - Returns: - The command's stdout without surrounding whitespace. - """ + """Run one git command in ``directory`` and return its stripped stdout.""" result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + # --no-optional-locks keeps `git status` from refreshing the index, so a build never takes + # the index lock while the user runs git. [git, "--no-optional-locks", "-C", str(directory), *args], capture_output=True, text=True, @@ -198,14 +168,7 @@ def _git(git: str, directory: Path, *args: str) -> str: def _wheel_commit(dist: Distribution) -> GitSource | UnknownCommit: - """Read the commit that ``scripts/build_dists.py`` wrote into the installed package. - - Returns: - The commit, or the reason it is unknown. - - Raises: - TimeNetBuildError: If the file is not valid JSON or not a valid commit record. - """ + """Read the commit that ``scripts/build_dists.py`` wrote into the installed package.""" for file in dist.files or (): # The file sits at the top of the import package: ``/_build_info.json``. if file.name == BUILD_INFO_FILE and len(file.parent.parts) == 1: @@ -213,19 +176,12 @@ def _wheel_commit(dist: Distribution) -> GitSource | UnknownCommit: record = json.loads(file.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed {file}: {exc}") from exc - return _validated(dist, record, str(file)) + return _validate_recorded_commit(dist, record, str(file)) return UnknownCommit(f"the installed package carries no {BUILD_INFO_FILE}") -def _validated(dist: Distribution, record: object, where: str) -> GitSource: - """Validate a recorded commit. - - Returns: - The commit. - - Raises: - TimeNetBuildError: If the record is not a valid commit record. - """ +def _validate_recorded_commit(dist: Distribution, record: object, where: str) -> GitSource: + """Validate a commit record that ``where`` holds, and raise ``TimeNetBuildError`` if it is invalid.""" try: return _GIT_SOURCE.validate_python(record) except ValidationError as exc: diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 1209c028f..19c654e1a 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -244,7 +244,6 @@ def _reaching(self) -> Generator[None]: raise TimeNetUnreachableError(f"cannot reach {self._display_root()}: {error}") from error def _key(self, *parts: str) -> str: - # ruff: ignore[docstring-missing-returns] """Build a bucket-relative key: the prefix, then ``datasets/``, then the path parts.""" segments = [self._prefix, _DATASETS_PREFIX, *parts] return "/".join(segment.strip("/") for segment in segments if segment) diff --git a/packages/timenet/src/timenet/schemas/manifest.schema.json b/packages/timenet/src/timenet/schemas/manifest.schema.json index 28c6f5690..9e92dc8a2 100644 --- a/packages/timenet/src/timenet/schemas/manifest.schema.json +++ b/packages/timenet/src/timenet/schemas/manifest.schema.json @@ -375,7 +375,7 @@ "description": "The git commit that one TimeNet distribution was built from.", "properties": { "sha": { - "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$", + "pattern": "^[0-9a-f]{40}$", "title": "Sha", "type": "string" }, diff --git a/pyproject.toml b/pyproject.toml index 3de3c56bc..58723e1fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,10 @@ banned-module-level-imports = ["typer", "timenet.cli", "timenet_connectors.build # timenet-connectors depends on timenet, so timenet never imports it and works without it. "timenet_connectors".msg = "timenet must work without timenet-connectors" +[tool.ruff.lint.pydoclint] +# A one-line docstring of a small helper does not need Returns or Raises sections. +ignore-one-line-docstrings = true + [tool.ruff.lint.flake8-bugbear] # Typer expects Option/Argument descriptors as parameter defaults. extend-immutable-calls = ["typer.Option", "typer.Argument"] diff --git a/scripts/build_dists.py b/scripts/build_dists.py index 0e53bceaa..d071074bf 100644 --- a/scripts/build_dists.py +++ b/scripts/build_dists.py @@ -2,19 +2,9 @@ # requires-python = ">=3.11" # dependencies = [] # /// -"""Build distributions with the git commit of each package recorded inside it. +"""Run ``uv build`` with the git commit of each package recorded in its ``_build_info.json``. -A wheel has no git directory, so ``timenet.provenance.git_sources`` reads the commit from the -``_build_info.json`` that this script writes into each import package. ``local_changes`` covers the -package directory only, the same scope that a local checkout records. The script then runs -``uv build`` with the given arguments, and removes the files again even if the build fails. A -later plain ``uv build`` therefore never ships an old record. ``make build`` and the publish -workflow run this script. - -Usage:: - - uv run scripts/build_dists.py --all-packages - uv run scripts/build_dists.py --package timenet --out-dir dist +Usage: ``uv run scripts/build_dists.py --all-packages`` (any ``uv build`` arguments work). """ import json @@ -35,14 +25,7 @@ def executable(name: str) -> str: - """Return the absolute path of a command on ``PATH``. - - Returns: - The path of the command. - - Raises: - SystemExit: If the command is not on ``PATH``. - """ + """Return the absolute path of a command on ``PATH``, or exit if it is missing.""" path = shutil.which(name) if path is None: raise SystemExit(f"{name} is not on PATH") @@ -50,14 +33,7 @@ def executable(name: str) -> str: def git(directory: Path, *args: str) -> str: - """Run one git command in ``directory`` and return its stripped stdout. - - Returns: - The command's stdout without surrounding whitespace. - - Raises: - SystemExit: If git fails, for example outside a git checkout. - """ + """Run one git command in ``directory`` and return its stripped stdout, or exit if it fails.""" result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] [executable("git"), "-C", str(directory), *args], capture_output=True, text=True, check=False ) @@ -66,28 +42,38 @@ def git(directory: Path, *args: str) -> str: return result.stdout.strip() -def main(build_args: list[str]) -> int: - """Write ``_build_info.json`` into each import package, build, and remove the files. +def state(directory: Path) -> tuple[str, str]: + """Return the commit of a package directory and its ``git status`` output.""" + status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + return git(directory, "rev-parse", "HEAD"), status - Args: - build_args: The arguments for ``uv build``. - Returns: - The exit code of ``uv build``. - """ +def main(build_args: list[str]) -> int: + """Write the commit records, build, and remove the records again, also on failure.""" written: list[Path] = [] + before = {directory: state(directory) for directory in PACKAGES} try: for directory, module in PACKAGES.items(): - sha = git(directory, "rev-parse", "HEAD") - status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + sha, status = before[directory] info = {"sha": sha, "local_changes": bool(status)} target = directory / "src" / module / BUILD_INFO_FILE - target.write_text(json.dumps(info) + "\n", encoding="utf-8") + # Register the file before the write, so a write that fails halfway is removed too. written.append(target) + target.write_text(json.dumps(info) + "\n", encoding="utf-8") print(f"{target.relative_to(ROOT)}: {info}", file=sys.stderr) # uv run sets UV to its own executable. uv = os.environ.get("UV") or executable("uv") - return subprocess.run([uv, "build", *build_args], cwd=ROOT, check=False).returncode # ruff: ignore[subprocess-without-shell-equals-true] + returncode = subprocess.run([uv, "build", *build_args], cwd=ROOT, check=False).returncode # ruff: ignore[subprocess-without-shell-equals-true] + # A file that changed while uv built would ship under a record that does not describe it. + changed = [str(directory.relative_to(ROOT)) for directory in PACKAGES if state(directory) != before[directory]] + if changed: + print( + f"error: {', '.join(changed)} changed during the build, so the built artifacts do not " + "match their commit records. Build again.", + file=sys.stderr, + ) + return 1 + return returncode finally: for path in written: path.unlink(missing_ok=True) From b135f11c1b34dcc934150612ef0a811c5f555225 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Fri, 6 Nov 2026 16:20:38 +0100 Subject: [PATCH 05/19] refactor(provenance): read the wheel's commit record by its path The record sits at a known place, /_build_info.json, so look it up there instead of walking the distribution's RECORD. Part of TNET-165 --- packages/timenet/src/timenet/provenance.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index e5417d100..931e87d9f 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -169,15 +169,15 @@ def _git(git: str, directory: Path, *args: str) -> str: def _wheel_commit(dist: Distribution) -> GitSource | UnknownCommit: """Read the commit that ``scripts/build_dists.py`` wrote into the installed package.""" - for file in dist.files or (): - # The file sits at the top of the import package: ``/_build_info.json``. - if file.name == BUILD_INFO_FILE and len(file.parent.parts) == 1: - try: - record = json.loads(file.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed {file}: {exc}") from exc - return _validate_recorded_commit(dist, record, str(file)) - return UnknownCommit(f"the installed package carries no {BUILD_INFO_FILE}") + module = dist.metadata["Name"].replace("-", "_") + path = Path(str(dist.locate_file(f"{module}/{BUILD_INFO_FILE}"))) + if not path.is_file(): + return UnknownCommit(f"the installed package carries no {BUILD_INFO_FILE}") + try: + record = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed {path}: {exc}") from exc + return _validate_recorded_commit(dist, record, str(path)) def _validate_recorded_commit(dist: Distribution, record: object, where: str) -> GitSource: From deb34d811d31246af364ceafe891dbcafa065953 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 09:50:05 +0100 Subject: [PATCH 06/19] refactor(provenance): give the commit lookup types that own it Review feedback asked for types that own the provenance logic, instead of module functions and raw dicts. - `GitSource` is a strict model with `sha: GitSHA` and `contains_local_changes`, and it loads a wheel's build info itself. `GitSourceMap` names the `build_env.git` block. The manifest key `local_changes` is now `contains_local_changes`. - `TimeNetDistribution` owns one installed distribution: it lists the installed ones, finds the on-disk project directory, and reads the commit of the code it runs. The editable, git-install and wheel cases are its private methods. - `GitSnapshot` holds the commit of every installed distribution, read at one time. `GitSnapshot.read()` is the only call that runs git, and `recorded_sources()` gives the block a manifest records. - A private model validates PEP 610 `direct_url.json`, so the lookup reads named fields instead of dict keys. `builder/env.py`, the test stub, the publish workflow and `scripts/build_dists.py` use the new names. Part of TNET-165 --- .github/workflows/publish.yml | 7 +- docs/manifest.md | 12 +- packages/conftest.py | 7 +- .../src/timenet_connectors/builder/env.py | 6 +- .../timenet/src/timenet/manifest/__init__.py | 3 +- .../timenet/src/timenet/manifest/manifest.py | 25 +- packages/timenet/src/timenet/provenance.py | 302 ++++++++++-------- .../src/timenet/schemas/manifest.schema.json | 6 +- .../timenet/tests/manifest/test_manifest.py | 6 +- packages/timenet/tests/test_provenance.py | 31 +- scripts/build_dists.py | 5 +- 11 files changed, 230 insertions(+), 180 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 95c3799c7..99896f885 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -77,9 +77,10 @@ jobs: uv run --isolated --no-project "${args[@]}" \ python -c "import $module; print('smoke ok:', $module.__name__)" uv run --isolated --no-project "${args[@]}" python -c " - from timenet.provenance import git_sources - commit = git_sources().get('$dist') - expected = {'sha': '$GITHUB_SHA', 'local_changes': False} + from timenet.manifest import GitSource + from timenet.provenance import GitSnapshot + commit = GitSnapshot.read().recorded_sources().get('$dist') + expected = GitSource(sha='$GITHUB_SHA', contains_local_changes=False) assert commit == expected, f'$dist records {commit}, expected {expected}' print('commit ok:', commit)" echo "::endgroup::" diff --git a/docs/manifest.md b/docs/manifest.md index df351f03b..efdec76f0 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -61,18 +61,18 @@ every installed package with its version. `timenet.provenance.build_env` collect ```json "git": { - "timenet": {"sha": "4818e37f...", "local_changes": false}, - "timenet-connectors": {"sha": "4818e37f...", "local_changes": false} + "timenet": {"sha": "4818e37f...", "contains_local_changes": false}, + "timenet-connectors": {"sha": "4818e37f...", "contains_local_changes": false} } ``` Each distribution records the commit of the code that runs: -- An editable install, such as a workspace checkout, asks git. `local_changes` is `true` if the - package directory has uncommitted changes, untracked files included. A change elsewhere in the - repository does not set it. +- An editable install, such as a workspace checkout, asks git. `contains_local_changes` is + `true` if the package directory has uncommitted changes, untracked files included. A change + elsewhere in the repository does not set it. - A git install (`pip install git+https://...`) records the commit that pip installed, with - `local_changes` set to `false`. + `contains_local_changes` set to `false`. - Any other install, such as a wheel or `pip install ./packages/timenet`, reads the `_build_info.json` file in the package. `scripts/build_dists.py` writes that file while it builds the wheel, and records the state of the checkout at that time. `make build` and the diff --git a/packages/conftest.py b/packages/conftest.py index da9bdb489..5d212fdf8 100644 --- a/packages/conftest.py +++ b/packages/conftest.py @@ -7,13 +7,14 @@ import pytest -from timenet import provenance +from timenet.manifest import GitSource +from timenet.provenance import GitSnapshot, TimeNetDistribution -_CLEAN = {"sha": "0" * 40, "local_changes": False} +_CLEAN = GitSnapshot(dict.fromkeys(TimeNetDistribution.NAMES, GitSource(sha="0" * 40, contains_local_changes=False))) @pytest.fixture(autouse=True) def _committed_checkout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: if request.node.get_closest_marker("real_git") is None: - monkeypatch.setattr(provenance, "_read_git", lambda: dict.fromkeys(provenance.TIMENET_DISTRIBUTIONS, _CLEAN)) + monkeypatch.setattr(GitSnapshot, "read", staticmethod(lambda: _CLEAN)) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index 06c7cbc27..af002a98e 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -23,12 +23,12 @@ from timenet.errors import TimeNetBuildError, TimeNetIsolationError from timenet.manifest import PayloadSource -from timenet.provenance import TIMENET_DISTRIBUTIONS, local_source_path +from timenet.provenance import TimeNetDistribution from timenet.types.references import DatasetRef from timenet_connectors.discovery import requirements_for -BASE_DISTRIBUTIONS = TIMENET_DISTRIBUTIONS +BASE_DISTRIBUTIONS = TimeNetDistribution.NAMES """Pinned to the parent's own versions, so the code that writes the TimeF bytes is the code that reads them back. ``timenet-connectors`` depends on ``timenet[build]``, so the card-YAML dependencies arrive transitively and the extra never needs naming here.""" @@ -212,7 +212,7 @@ def _base_args() -> tuple[str, ...]: raise TimeNetIsolationError( f"cannot make a build environment: {name} is not installed in {sys.executable}." ) from exc - source = local_source_path(dist) + source = TimeNetDistribution(dist).path_to_ondisk_project_directory() if source is not None: args += ["--with-editable", str(source)] else: diff --git a/packages/timenet/src/timenet/manifest/__init__.py b/packages/timenet/src/timenet/manifest/__init__.py index 54752de10..297f0101c 100644 --- a/packages/timenet/src/timenet/manifest/__init__.py +++ b/packages/timenet/src/timenet/manifest/__init__.py @@ -2,7 +2,7 @@ from timenet.manifest.counts import ManifestCounts from timenet.manifest.files import FilePart, ManifestFiles -from timenet.manifest.manifest import BuildEnvironment, GitSource, Manifest, parse_manifest, read_manifest +from timenet.manifest.manifest import BuildEnvironment, GitSource, GitSourceMap, Manifest, parse_manifest, read_manifest from timenet.manifest.payload_source import PayloadSource @@ -10,6 +10,7 @@ "BuildEnvironment", "FilePart", "GitSource", + "GitSourceMap", "Manifest", "ManifestCounts", "ManifestFiles", diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index f7743ef99..b90100daa 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Annotated, Any, Self -from pydantic import ConfigDict, Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator, with_config +from pydantic import Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator from typing_extensions import TypedDict from timenet.errors import ReleaseDifference, TimeFFormatError, TimeNetManifestError @@ -20,28 +20,35 @@ from timenet.values_backends import ValuesBackend -GitSha = Annotated[str, StringConstraints(strict=True, pattern=r"^[0-9a-f]{40}$")] +GitSHA = Annotated[str, StringConstraints(strict=True, pattern=r"^[0-9a-f]{40}$")] """A full SHA-1 git commit id: 40 lowercase hex digits.""" -@with_config(ConfigDict(extra="forbid")) -class GitSource(TypedDict): +class GitSource(TimeFModel): """The git commit that one TimeNet distribution was built from.""" - sha: GitSha + sha: GitSHA """The commit of the checkout, or of the checkout that built the wheel.""" - local_changes: StrictBool + contains_local_changes: StrictBool """Whether the package directory had uncommitted changes, untracked files included.""" + @classmethod + def load_from_build_info(cls, path: Path) -> Self: + """Read the record that ``scripts/build_dists.py`` wrote into a built package.""" + return cls.model_validate_json(path.read_text(encoding="utf-8")) + + +GitSourceMap = dict[StrictStr, GitSource] +"""The commit of each TimeNet distribution, keyed by distribution name.""" + class BuildEnvironment(TypedDict, total=False): """Build-provenance fields recorded in a manifest.""" python: StrictStr packages: dict[StrictStr, StrictStr] - git: dict[StrictStr, GitSource] - """The commit of each TimeNet distribution, keyed by distribution name. A distribution whose - commit is unknown is absent.""" + git: GitSourceMap + """The commit of each TimeNet distribution. A distribution whose commit is unknown is absent.""" class Manifest(TimeFModel): diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 931e87d9f..0997447d2 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,38 +1,25 @@ """Describe the environment a build ran in, for the manifest's provenance block.""" +from collections.abc import Mapping from dataclasses import dataclass from importlib.metadata import Distribution, PackageNotFoundError, distributions -import json from pathlib import Path import platform import shutil import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors +from typing import Self from urllib.parse import urlparse from urllib.request import url2pathname -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, StrictBool, StrictStr, ValidationError from timenet.errors import TimeNetBuildError -from timenet.manifest import BuildEnvironment, GitSource +from timenet.manifest import BuildEnvironment, GitSource, GitSourceMap -TIMENET_DISTRIBUTIONS = ("timenet", "timenet-connectors") -"""The distributions whose git commit a build records.""" - -BUILD_INFO_FILE = "_build_info.json" -"""The commit record that ``scripts/build_dists.py`` writes into each package of a wheel.""" - -_GIT_SOURCE = TypeAdapter(GitSource) _GIT_TIMEOUT_S = 30 -@dataclass(frozen=True) -class UnknownCommit: - """Why the commit of an installed TimeNet distribution is unknown.""" - - reason: str - - def build_env() -> BuildEnvironment: """Describe this process's Python version, installed packages, and TimeNet commits. @@ -42,118 +29,190 @@ def build_env() -> BuildEnvironment: Returns: A dict with ``python`` (the interpreter version), ``packages`` (name -> version, sorted), and - ``git`` (see :func:`git_sources`). + ``git`` (see :meth:`GitSnapshot.recorded_sources`). """ packages: dict[str, str] = {} for dist in distributions(): name = dist.metadata["Name"] if name: packages[name] = dist.version - return {"python": platform.python_version(), "packages": dict(sorted(packages.items())), "git": git_sources()} + return { + "python": platform.python_version(), + "packages": dict(sorted(packages.items())), + "git": GitSnapshot.read().recorded_sources(), + } + +@dataclass(frozen=True) +class UnknownGitSource: + """Why the commit of an installed TimeNet distribution is unknown.""" -def git_sources() -> dict[str, GitSource]: - """Return the git commit of each installed TimeNet distribution whose commit is known. + reason: str - Each distribution reports the commit of the code that runs: - - An editable install asks git. ``local_changes`` covers the package directory only, so a - scratch file elsewhere in the repository does not set it. - - A git install (``pip install git+https://...``) reports the commit that pip recorded. - - Any other install, such as a wheel or ``pip install ./packages/timenet``, reads the - ``_build_info.json`` that the build wrote into the package. +class TimeNetDistribution: + """One installed TimeNet distribution, and the commit of the code it runs. - A distribution whose commit is unknown is absent. + Each install type finds its commit in its own place: - Returns: - The commits, keyed by distribution name. + - An editable install asks git. ``contains_local_changes`` covers the package directory only, + so a scratch file elsewhere in the repository does not set it. + - A git install (``pip install git+https://...``) has the commit that pip recorded. + - Any other install, such as a wheel or ``pip install ./packages/timenet``, reads the + ``_build_info.json`` that ``scripts/build_dists.py`` wrote into the package. """ - return {name: commit for name, commit in _read_git().items() if not isinstance(commit, UnknownCommit)} + NAMES = ("timenet", "timenet-connectors") + """The distributions whose git commit a build records.""" + + BUILD_INFO_FILE = "_build_info.json" + """The commit record in each package of a wheel. ``scripts/build_dists.py`` uses the same name.""" + + def __init__(self, distribution: Distribution) -> None: + self._distribution = distribution + + @classmethod + def installed(cls) -> list[Self]: + """Return each TimeNet distribution that is installed in this interpreter.""" + installed: list[Self] = [] + for name in cls.NAMES: + try: + installed.append(cls(Distribution.from_name(name))) + except PackageNotFoundError: + continue + return installed + + @property + def name(self) -> str: + """The distribution name, for example ``timenet-connectors``.""" + return self._distribution.metadata["Name"] + + def path_to_ondisk_project_directory(self) -> Path | None: + """Return the project directory of an editable or local path install, else ``None``.""" + direct_url = self._read_direct_url() + return None if direct_url is None else direct_url.local_directory() + + def read_git_source(self) -> GitSource | UnknownGitSource: + """Return the commit of the code that this distribution runs, or why it is unknown.""" + direct_url = self._read_direct_url() + if direct_url is None: + return self._read_build_info_git_source() + installed_commit = direct_url.installed_git_commit() + if installed_commit is not None: + return self._git_source_of_installed_commit(installed_commit) + project_directory = direct_url.local_directory() + if direct_url.is_editable() and project_directory is not None: + return TimeNetDistribution._read_checkout_git_source(project_directory) + return self._read_build_info_git_source() + + def _read_direct_url(self) -> "_DirectUrl | None": + """Read the PEP 610 ``direct_url.json``, or return ``None`` for an install from an index.""" + raw = self._distribution.read_text("direct_url.json") + if raw is None: + return None + try: + return _DirectUrl.model_validate_json(raw) + except ValidationError as exc: + raise TimeNetBuildError(f"{self.name} has a malformed direct_url.json: {exc}") from exc -def local_source_path(dist: Distribution) -> Path | None: - """Return the on-disk project directory an editable or local path install points at. + def _git_source_of_installed_commit(self, sha: str) -> GitSource: + """Return the clean commit record of a git install, and raise if pip recorded an invalid sha.""" + try: + return GitSource(sha=sha, contains_local_changes=False) + except ValidationError as exc: + raise TimeNetBuildError(f"{self.name} records an invalid commit in direct_url.json: {exc}") from exc + + @staticmethod + def _read_checkout_git_source(project_directory: Path) -> GitSource | UnknownGitSource: + """Ask git for the commit of a checkout and whether its package directory has changes.""" + git = shutil.which("git") + if git is None: + return UnknownGitSource("git is not on PATH") + try: + sha = _run_git(git, project_directory, "rev-parse", "HEAD") + # The pathspec "." scopes the status to the package directory, untracked files included. + status = _run_git(git, project_directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + except subprocess.CalledProcessError as exc: + return UnknownGitSource(f"git failed in {project_directory}: {(exc.stderr or '').strip() or exc}") + except (OSError, subprocess.TimeoutExpired) as exc: + return UnknownGitSource(f"git failed in {project_directory}: {exc}") + return GitSource(sha=sha, contains_local_changes=bool(status)) + + def _read_build_info_git_source(self) -> GitSource | UnknownGitSource: + """Read the commit that ``scripts/build_dists.py`` wrote into the installed package.""" + module = self.name.replace("-", "_") + path = Path(str(self._distribution.locate_file(f"{module}/{self.BUILD_INFO_FILE}"))) + if not path.is_file(): + return UnknownGitSource(f"the installed package carries no {self.BUILD_INFO_FILE}") + try: + return GitSource.load_from_build_info(path) + except ValidationError as exc: + raise TimeNetBuildError(f"{self.name} records an invalid commit in {path}: {exc}") from exc - ``direct_url.json`` records how a distribution was installed. An editable install carries - ``dir_info.editable``. A plain path install (``pip install ./packages/timenet``) records a - ``file://`` URL to the same source tree without that flag. Both return the directory. An index or - wheel install has no ``file://`` directory and returns ``None``. - Args: - dist: The installed distribution. +@dataclass(frozen=True) +class GitSnapshot: + """The commit of each installed TimeNet distribution, read at one point in time.""" - Returns: - The source directory for an editable or local path install, else ``None``. - """ - info = _direct_url(dist) - if info is None: - return None - parsed = urlparse(info.get("url", "")) - if parsed.scheme != "file": - return None - path = Path(url2pathname(parsed.path)) - if info.get("dir_info", {}).get("editable") or path.is_dir(): - return path - return None - - -def _read_git() -> dict[str, GitSource | UnknownCommit]: - """Look up the commit of each installed TimeNet distribution. This is the only call that runs git.""" - commits: dict[str, GitSource | UnknownCommit] = {} - for name in TIMENET_DISTRIBUTIONS: - try: - dist = Distribution.from_name(name) - except PackageNotFoundError: - continue - commits[name] = _commit_of(dist) - return commits - - -def _commit_of(dist: Distribution) -> GitSource | UnknownCommit: - """Return the commit of the code that one installed distribution runs, or why it is unknown.""" - info = _direct_url(dist) - if info is not None: - vcs = info.get("vcs_info", {}) - if vcs.get("vcs") == "git" and "commit_id" in vcs: - return _validate_recorded_commit(dist, {"sha": vcs["commit_id"], "local_changes": False}, "direct_url.json") - if info.get("dir_info", {}).get("editable"): - directory = local_source_path(dist) - if directory is not None: - return _checkout_commit(directory) - return _wheel_commit(dist) - - -def _direct_url(dist: Distribution) -> dict | None: - """Read the PEP 610 ``direct_url.json`` of a distribution, or ``None`` for an index install.""" - raw = dist.read_text("direct_url.json") - if raw is None: - return None - try: - info = json.loads(raw) - except json.JSONDecodeError as exc: - raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed direct_url.json: {exc}") from exc - if not isinstance(info, dict): - raise TimeNetBuildError(f"{dist.metadata['Name']} has a direct_url.json that is not a JSON object") - return info - - -def _checkout_commit(directory: Path) -> GitSource | UnknownCommit: - """Ask git for the commit of a checkout and whether its package directory has changes.""" - git = shutil.which("git") - if git is None: - return UnknownCommit("git is not on PATH") - try: - sha = _git(git, directory, "rev-parse", "HEAD") - # The pathspec "." scopes the status to the package directory, untracked files included. - status = _git(git, directory, "status", "--porcelain", "--untracked-files=all", "--", ".") - except subprocess.CalledProcessError as exc: - return UnknownCommit(f"git failed in {directory}: {(exc.stderr or '').strip() or exc}") - except (OSError, subprocess.TimeoutExpired) as exc: - return UnknownCommit(f"git failed in {directory}: {exc}") - return {"sha": sha, "local_changes": bool(status)} - - -def _git(git: str, directory: Path, *args: str) -> str: + sources: Mapping[str, GitSource | UnknownGitSource] + """The commit, or why it is unknown, keyed by distribution name.""" + + @classmethod + def read(cls) -> Self: + """Read the commit of each installed TimeNet distribution. This is the only call that runs git.""" + return cls( + {distribution.name: distribution.read_git_source() for distribution in TimeNetDistribution.installed()} + ) + + def recorded_sources(self) -> GitSourceMap: + """Return the known commits, in the form that a manifest records.""" + return {name: source for name, source in self.sources.items() if isinstance(source, GitSource)} + + +class _DirInfo(BaseModel): + model_config = ConfigDict(frozen=True) + + editable: StrictBool = False + + +class _VcsInfo(BaseModel): + model_config = ConfigDict(frozen=True) + + vcs: StrictStr + commit_id: StrictStr + + +class _DirectUrl(BaseModel): + """The PEP 610 record of how a distribution was installed. Keys this module does not read are ignored.""" + + model_config = ConfigDict(frozen=True) + + url: StrictStr + dir_info: _DirInfo | None = None + vcs_info: _VcsInfo | None = None + + def installed_git_commit(self) -> str | None: + """Return the commit that pip installed from a git repository, else ``None``.""" + if self.vcs_info is None or self.vcs_info.vcs != "git": + return None + return self.vcs_info.commit_id + + def is_editable(self) -> bool: + """Whether the distribution was installed editable.""" + return self.dir_info is not None and self.dir_info.editable + + def local_directory(self) -> Path | None: + """Return the directory of an editable or local path install, else ``None``.""" + if not self._is_file_url(): + return None + path = Path(url2pathname(urlparse(self.url).path)) + return path if self.is_editable() or path.is_dir() else None + + def _is_file_url(self) -> bool: + return urlparse(self.url).scheme == "file" + + +def _run_git(git: str, directory: Path, *args: str) -> str: """Run one git command in ``directory`` and return its stripped stdout.""" result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # --no-optional-locks keeps `git status` from refreshing the index, so a build never takes @@ -165,24 +224,3 @@ def _git(git: str, directory: Path, *args: str) -> str: timeout=_GIT_TIMEOUT_S, ) return result.stdout.strip() - - -def _wheel_commit(dist: Distribution) -> GitSource | UnknownCommit: - """Read the commit that ``scripts/build_dists.py`` wrote into the installed package.""" - module = dist.metadata["Name"].replace("-", "_") - path = Path(str(dist.locate_file(f"{module}/{BUILD_INFO_FILE}"))) - if not path.is_file(): - return UnknownCommit(f"the installed package carries no {BUILD_INFO_FILE}") - try: - record = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise TimeNetBuildError(f"{dist.metadata['Name']} has a malformed {path}: {exc}") from exc - return _validate_recorded_commit(dist, record, str(path)) - - -def _validate_recorded_commit(dist: Distribution, record: object, where: str) -> GitSource: - """Validate a commit record that ``where`` holds, and raise ``TimeNetBuildError`` if it is invalid.""" - try: - return _GIT_SOURCE.validate_python(record) - except ValidationError as exc: - raise TimeNetBuildError(f"{dist.metadata['Name']} records an invalid commit in {where}: {exc}") from exc diff --git a/packages/timenet/src/timenet/schemas/manifest.schema.json b/packages/timenet/src/timenet/schemas/manifest.schema.json index 9e92dc8a2..984b5ed3d 100644 --- a/packages/timenet/src/timenet/schemas/manifest.schema.json +++ b/packages/timenet/src/timenet/schemas/manifest.schema.json @@ -379,14 +379,14 @@ "title": "Sha", "type": "string" }, - "local_changes": { - "title": "Local Changes", + "contains_local_changes": { + "title": "Contains Local Changes", "type": "boolean" } }, "required": [ "sha", - "local_changes" + "contains_local_changes" ], "title": "GitSource", "type": "object" diff --git a/packages/timenet/tests/manifest/test_manifest.py b/packages/timenet/tests/manifest/test_manifest.py index 8599a9912..c31128686 100644 --- a/packages/timenet/tests/manifest/test_manifest.py +++ b/packages/timenet/tests/manifest/test_manifest.py @@ -488,7 +488,7 @@ def test_build_env_defaults_to_empty(): def test_build_env_round_trips(): - git = {"timenet": {"sha": "a" * 40, "local_changes": False}} + git = {"timenet": {"sha": "a" * 40, "contains_local_changes": False}} m = _replace(_manifest(), build_env={"python": "3.11.9", "packages": {"timenet": "0.1.0"}, "git": git}) assert Manifest.model_validate(m.to_dict()).build_env == m.build_env @@ -497,8 +497,8 @@ def test_build_env_round_trips(): "change", [ {"build_env": {"python": 313}}, - {"build_env": {"git": {"timenet": {"sha": "4818e37f", "local_changes": False}}}}, - {"build_env": {"git": {"timenet": {"sha": "a" * 40, "local_changes": "no"}}}}, + {"build_env": {"git": {"timenet": {"sha": "4818e37f", "contains_local_changes": False}}}}, + {"build_env": {"git": {"timenet": {"sha": "a" * 40, "contains_local_changes": "no"}}}}, {"value_encoding": {"ecg_lead": "zip"}}, ], ) diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index c5a78de4b..cc61d8ea6 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -9,7 +9,8 @@ import pytest from timenet.errors import TimeNetBuildError -from timenet.provenance import build_env, git_sources +from timenet.manifest import GitSource +from timenet.provenance import GitSnapshot, build_env def test_build_env_records_the_running_python(): @@ -94,7 +95,7 @@ def _editable(directory: Path) -> dict: def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, installed): installed(direct_url=_editable(checkout)) head = _run(git, checkout, "rev-parse", "HEAD") - assert build_env()["git"] == {"timenet": {"sha": head, "local_changes": False}} + assert build_env()["git"] == {"timenet": GitSource(sha=head, contains_local_changes=False)} @pytest.mark.real_git @@ -102,7 +103,7 @@ def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, i def test_a_change_in_the_package_directory_is_a_local_change(checkout, installed, change): installed(direct_url=_editable(checkout)) (checkout / change).write_text("x = 2\n") - assert git_sources()["timenet"]["local_changes"] is True + assert GitSnapshot.read().recorded_sources()["timenet"].contains_local_changes is True @pytest.mark.real_git @@ -110,7 +111,7 @@ def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout, installed(direct_url=_editable(checkout)) (checkout.parents[1] / "README.md").write_text("edited\n") (checkout.parents[1] / "scratch.ipynb").write_text("{}\n") - assert git_sources()["timenet"]["local_changes"] is False + assert GitSnapshot.read().recorded_sources()["timenet"].contains_local_changes is False @pytest.mark.real_git @@ -118,34 +119,34 @@ def test_an_editable_install_outside_git_has_no_commit(git, tmp_path, installed) outside = tmp_path / "outside" outside.mkdir() installed(direct_url=_editable(outside)) - assert git_sources() == {} + assert GitSnapshot.read().recorded_sources() == {} @pytest.mark.real_git def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): - info = {"sha": "a" * 40, "local_changes": False} - installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=json.dumps(info)) - assert git_sources() == {"timenet": info} + info = GitSource(sha="a" * 40, contains_local_changes=False) + installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=info.model_dump_json()) + assert GitSnapshot.read().recorded_sources() == {"timenet": info} @pytest.mark.real_git def test_a_git_install_records_the_commit_that_pip_installed(installed): sha = "b" * 40 installed(direct_url={"url": "https://example.org/timenet.git", "vcs_info": {"vcs": "git", "commit_id": sha}}) - assert git_sources() == {"timenet": {"sha": sha, "local_changes": False}} + assert GitSnapshot.read().recorded_sources() == {"timenet": GitSource(sha=sha, contains_local_changes=False)} @pytest.mark.real_git def test_a_wheel_reads_its_commit_from_the_build_info(installed): - info = {"sha": "a" * 40, "local_changes": True} - installed(build_info=json.dumps(info)) - assert git_sources() == {"timenet": info} + info = GitSource(sha="a" * 40, contains_local_changes=True) + installed(build_info=info.model_dump_json()) + assert GitSnapshot.read().recorded_sources() == {"timenet": info} @pytest.mark.real_git def test_a_wheel_without_build_info_has_no_commit(installed): installed() - assert git_sources() == {} + assert GitSnapshot.read().recorded_sources() == {} @pytest.mark.real_git @@ -154,11 +155,11 @@ def test_a_wheel_without_build_info_has_no_commit(installed): [ # The manifest forbids extra keys for every nested block, but this standalone check must # forbid them itself. - json.dumps({"sha": "a" * 40, "local_changes": False, "branch": "main"}), + json.dumps({"sha": "a" * 40, "contains_local_changes": False, "branch": "main"}), "not json", ], ) def test_a_wheel_with_invalid_build_info_fails(installed, build_info): installed(build_info=build_info) with pytest.raises(TimeNetBuildError, match=r"_build_info\.json"): - git_sources() + GitSnapshot.read().recorded_sources() diff --git a/scripts/build_dists.py b/scripts/build_dists.py index d071074bf..29dad87d8 100644 --- a/scripts/build_dists.py +++ b/scripts/build_dists.py @@ -20,7 +20,7 @@ ROOT / "packages" / "timenet": "timenet", ROOT / "packages" / "timenet-connectors": "timenet_connectors", } -# Keep in step with timenet.provenance.BUILD_INFO_FILE, which reads the file. +# Keep in step with timenet.provenance.TimeNetDistribution.BUILD_INFO_FILE, which reads the file. BUILD_INFO_FILE = "_build_info.json" @@ -55,7 +55,8 @@ def main(build_args: list[str]) -> int: try: for directory, module in PACKAGES.items(): sha, status = before[directory] - info = {"sha": sha, "local_changes": bool(status)} + # Keep the keys in step with timenet.manifest.GitSource. + info = {"sha": sha, "contains_local_changes": bool(status)} target = directory / "src" / module / BUILD_INFO_FILE # Register the file before the write, so a write that fails halfway is removed too. written.append(target) From 1ea8374bc5b282cf2f4cfb0b98728ea9d86943b5 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 10:25:37 +0100 Subject: [PATCH 07/19] fix(provenance): honor subdirectory in direct_url.json PEP 610 lets a directory install name the repository in url and the project in subdirectory. The lookup ignored subdirectory, so it asked git about the whole repository and builder/env.py installed the repository root. Both now use the package directory. Part of TNET-165 --- packages/timenet/src/timenet/provenance.py | 4 ++++ packages/timenet/tests/test_provenance.py | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 0997447d2..66f9ab1ca 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -190,6 +190,8 @@ class _DirectUrl(BaseModel): url: StrictStr dir_info: _DirInfo | None = None vcs_info: _VcsInfo | None = None + subdirectory: StrictStr | None = None + """The project's folder inside ``url``, for a project that does not sit at its root.""" def installed_git_commit(self) -> str | None: """Return the commit that pip installed from a git repository, else ``None``.""" @@ -206,6 +208,8 @@ def local_directory(self) -> Path | None: if not self._is_file_url(): return None path = Path(url2pathname(urlparse(self.url).path)) + if self.subdirectory is not None: + path /= self.subdirectory return path if self.is_editable() or path.is_dir() else None def _is_file_url(self) -> bool: diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index cc61d8ea6..2996ca0b8 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -10,7 +10,7 @@ from timenet.errors import TimeNetBuildError from timenet.manifest import GitSource -from timenet.provenance import GitSnapshot, build_env +from timenet.provenance import GitSnapshot, TimeNetDistribution, build_env def test_build_env_records_the_running_python(): @@ -122,6 +122,22 @@ def test_an_editable_install_outside_git_has_no_commit(git, tmp_path, installed) assert GitSnapshot.read().recorded_sources() == {} +@pytest.mark.real_git +@pytest.mark.parametrize("editable", [True, False]) +def test_an_install_of_a_subdirectory_finds_the_package_directory(checkout, installed, editable): + # direct_url.json may name the repository and put the package in "subdirectory". + repository = checkout.parents[1] + installed( + direct_url={"url": repository.as_uri(), "dir_info": {"editable": editable}, "subdirectory": "packages/pkg"} + ) + (distribution,) = TimeNetDistribution.installed() + assert distribution.path_to_ondisk_project_directory() == checkout + if editable: + # A change outside the package directory does not count. + (repository / "README.md").write_text("edited\n") + assert GitSnapshot.read().recorded_sources()["timenet"].contains_local_changes is False + + @pytest.mark.real_git def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): info = GitSource(sha="a" * 40, contains_local_changes=False) From 0e1f450a7cd9fdb539407707889c2ee08f25aac7 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 11:05:31 +0100 Subject: [PATCH 08/19] refactor(build): drop the BASE_DISTRIBUTIONS alias It only renamed TimeNetDistribution.NAMES. The loop in _base_args reads the names directly, and its comment keeps the reason for the pin. Part of TNET-165 --- .../src/timenet_connectors/builder/env.py | 11 ++++------- packages/timenet-connectors/tests/test_build_env.py | 5 +++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index af002a98e..ae344e9f9 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -28,12 +28,6 @@ from timenet_connectors.discovery import requirements_for -BASE_DISTRIBUTIONS = TimeNetDistribution.NAMES -"""Pinned to the parent's own versions, so the code that writes the TimeF bytes is the code that -reads them back. ``timenet-connectors`` depends on ``timenet[build]``, so the card-YAML -dependencies arrive transitively and the extra never needs naming here.""" - - @dataclass(frozen=True) class EnvSpec: """What one connector's environment needs.""" @@ -205,7 +199,10 @@ def _base_args() -> tuple[str, ...]: TimeNetIsolationError: If a base distribution is not installed in this interpreter. """ args: list[str] = [] - for name in BASE_DISTRIBUTIONS: + # Pin TimeNet to the parent's own versions, so the code that writes the TimeF bytes is the code + # that reads them back. timenet-connectors depends on timenet[build], so the card-YAML + # dependencies arrive transitively and the extra never needs naming here. + for name in TimeNetDistribution.NAMES: try: dist = Distribution.from_name(name) except PackageNotFoundError as exc: diff --git a/packages/timenet-connectors/tests/test_build_env.py b/packages/timenet-connectors/tests/test_build_env.py index f42bb2038..71997a296 100644 --- a/packages/timenet-connectors/tests/test_build_env.py +++ b/packages/timenet-connectors/tests/test_build_env.py @@ -9,6 +9,7 @@ from timenet.errors import TimeNetBuildError from timenet.manifest import PayloadSource from timenet.manifest.files import FilePart +from timenet.provenance import TimeNetDistribution from timenet.types.references import DatasetRef from timenet_connectors.builder import env as env_module from timenet_connectors.builder.env import EnvSpec, env_spec, run_isolated_build, uv_command @@ -33,7 +34,7 @@ def read_text(self, name): def _base_args_for(monkeypatch, dist): - monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet",)) + monkeypatch.setattr(TimeNetDistribution, "NAMES", ("timenet",)) monkeypatch.setattr(env_module.Distribution, "from_name", lambda name: dist) return env_module._base_args() @@ -149,7 +150,7 @@ def test_run_build_captures_stderr_and_the_exit_code(): def test_env_spec_reports_a_base_distribution_that_is_not_installed(monkeypatch): - monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet", "timenet-not-installed")) + monkeypatch.setattr(TimeNetDistribution, "NAMES", ("timenet", "timenet-not-installed")) with pytest.raises(TimeNetBuildError, match="timenet-not-installed"): env_spec("timenet/hello-world") From b137930b3cc008151ea9395a381eac682689ab95 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 14:15:01 +0100 Subject: [PATCH 09/19] refactor(provenance): read commits per package and fail if one is unknown Split the commit lookup into two layers. InstalledDistribution picks the InstallSource of one package: an editable checkout, a git install, or a built package. Each install source reads the commit in its own place. PackageSnapshot, GroupSnapshot and TimeNetSnapshot are typed views of the commits that a manifest records in build_env.git. TimeNetSnapshot is now the one list of the TimeNet distributions. The build environment and the test stub use it. An unknown commit now stops the build, also for a local registry. The error gives the reason: git is not on PATH, git failed, or a built package has no _build_info.json. UnknownGitSource is gone. BuildEnvironment is now a model with a capture constructor. The manifest requires it, because each manifest that the writer produces records it. Part of TNET-165 --- .github/workflows/publish.yml | 4 +- docs/manifest.md | 17 +- packages/conftest.py | 8 +- .../src/timenet_connectors/builder/env.py | 10 +- .../tests/test_build_env.py | 6 +- .../tests/test_end_to_end.py | 4 +- .../timenet/src/timenet/manifest/__init__.py | 11 +- .../timenet/src/timenet/manifest/manifest.py | 33 +- packages/timenet/src/timenet/provenance.py | 303 ++++++++++++------ .../src/timenet/schemas/manifest.schema.json | 8 +- packages/timenet/src/timenet/writer/writer.py | 6 +- .../tests/format/test_control_cache.py | 3 +- .../timenet/tests/manifest/test_manifest.py | 56 +++- .../tests/manifest/test_manifest_schema.py | 10 +- packages/timenet/tests/test_provenance.py | 78 +++-- scripts/build_dists.py | 2 +- 16 files changed, 377 insertions(+), 182 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 99896f885..38c57d279 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -78,8 +78,8 @@ jobs: python -c "import $module; print('smoke ok:', $module.__name__)" uv run --isolated --no-project "${args[@]}" python -c " from timenet.manifest import GitSource - from timenet.provenance import GitSnapshot - commit = GitSnapshot.read().recorded_sources().get('$dist') + from timenet.provenance import PackageSnapshot + commit = PackageSnapshot.read('$dist').source expected = GitSource(sha='$GITHUB_SHA', contains_local_changes=False) assert commit == expected, f'$dist records {commit}, expected {expected}' print('commit ok:', commit)" diff --git a/docs/manifest.md b/docs/manifest.md index efdec76f0..8825713ab 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -39,7 +39,7 @@ Manifest( counts=counts, # ManifestCounts (default: empty) values_backend="parquet", # "parquet" (default) or "zarr" value_encoding={}, # spec_type -> the encoding its shards carry - build_env={}, # {"python", "packages", "git"} (default: empty) + build_env=build_env, # BuildEnvironment (required) timef_format_version=1, # required; validated against the supported set {1} ) ``` @@ -53,11 +53,13 @@ of each spec type. No code reads this field. Parquet records the applied encodin each file, so the reader does not need it. The field lets a builder see which encoding a build selected. The field is empty for a backend that has no such choice. -`build_env` records the environment that produced the version. It gives the interpreter version and -every installed package with its version. `timenet.provenance.build_env` collects this data. Like -`value_encoding`, `build_env` is provenance only. No code reads it to interpret the data. +`build_env` records the environment that produced the version. It gives the interpreter version, +every installed package with its version, and the git commit of the TimeNet code. +`BuildEnvironment.capture` collects this data. Like `value_encoding`, `build_env` is provenance +only. No code reads it to interpret the data. -`build_env.git` records the git commit of each TimeNet distribution, keyed by distribution name: +`build_env.git` records the git commit of each installed TimeNet distribution, keyed by +distribution name. A distribution that is not installed has no entry: ```json "git": { @@ -78,8 +80,9 @@ Each distribution records the commit of the code that runs: builds the wheel, and records the state of the checkout at that time. `make build` and the publish workflow run the script. -A distribution whose commit is unknown is absent, for example a wheel that a plain `uv build` -produced. +If the commit of an installed distribution is unknown, the build fails and names the reason. For +example, git is not on `PATH`, git fails in the checkout, or a plain `uv build` produced the wheel, +so the package has no `_build_info.json`. The values locator is backend-neutral. One schema covers both scalar and multidimensional specs. A multidimensional spec records its shape in `value_shape` and `dimension_names`. It does not need a diff --git a/packages/conftest.py b/packages/conftest.py index 5d212fdf8..46d82ae56 100644 --- a/packages/conftest.py +++ b/packages/conftest.py @@ -2,19 +2,19 @@ Each manifest write asks git for the commit of the TimeNet distributions. Real git calls would make a test's result depend on the state of this checkout, and they cost seconds across the suite. So -every test sees a clean, committed checkout, unless it carries the ``real_git`` marker. +every installed distribution reports a clean commit, unless the test carries the ``real_git`` marker. """ import pytest from timenet.manifest import GitSource -from timenet.provenance import GitSnapshot, TimeNetDistribution +from timenet.provenance import InstalledDistribution -_CLEAN = GitSnapshot(dict.fromkeys(TimeNetDistribution.NAMES, GitSource(sha="0" * 40, contains_local_changes=False))) +_CLEAN = GitSource(sha="0" * 40, contains_local_changes=False) @pytest.fixture(autouse=True) def _committed_checkout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: if request.node.get_closest_marker("real_git") is None: - monkeypatch.setattr(GitSnapshot, "read", staticmethod(lambda: _CLEAN)) + monkeypatch.setattr(InstalledDistribution, "read_git_source", lambda _self: _CLEAN) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index ae344e9f9..3a125fcfa 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -23,7 +23,7 @@ from timenet.errors import TimeNetBuildError, TimeNetIsolationError from timenet.manifest import PayloadSource -from timenet.provenance import TimeNetDistribution +from timenet.provenance import InstalledDistribution, TimeNetSnapshot from timenet.types.references import DatasetRef from timenet_connectors.discovery import requirements_for @@ -202,16 +202,16 @@ def _base_args() -> tuple[str, ...]: # Pin TimeNet to the parent's own versions, so the code that writes the TimeF bytes is the code # that reads them back. timenet-connectors depends on timenet[build], so the card-YAML # dependencies arrive transitively and the extra never needs naming here. - for name in TimeNetDistribution.NAMES: + for name in TimeNetSnapshot.distributions(): try: - dist = Distribution.from_name(name) + installed = InstalledDistribution.from_name(name) except PackageNotFoundError as exc: raise TimeNetIsolationError( f"cannot make a build environment: {name} is not installed in {sys.executable}." ) from exc - source = TimeNetDistribution(dist).path_to_ondisk_project_directory() + source = installed.path_to_ondisk_project_directory() if source is not None: args += ["--with-editable", str(source)] else: - args += ["--with", f"{name}=={dist.version}"] + args += ["--with", f"{name}=={installed.version}"] return tuple(args) diff --git a/packages/timenet-connectors/tests/test_build_env.py b/packages/timenet-connectors/tests/test_build_env.py index 71997a296..4ba053da5 100644 --- a/packages/timenet-connectors/tests/test_build_env.py +++ b/packages/timenet-connectors/tests/test_build_env.py @@ -9,7 +9,7 @@ from timenet.errors import TimeNetBuildError from timenet.manifest import PayloadSource from timenet.manifest.files import FilePart -from timenet.provenance import TimeNetDistribution +from timenet.provenance import TimeNetSnapshot from timenet.types.references import DatasetRef from timenet_connectors.builder import env as env_module from timenet_connectors.builder.env import EnvSpec, env_spec, run_isolated_build, uv_command @@ -34,7 +34,7 @@ def read_text(self, name): def _base_args_for(monkeypatch, dist): - monkeypatch.setattr(TimeNetDistribution, "NAMES", ("timenet",)) + monkeypatch.setattr(TimeNetSnapshot, "distributions", classmethod(lambda cls: ("timenet",))) monkeypatch.setattr(env_module.Distribution, "from_name", lambda name: dist) return env_module._base_args() @@ -150,7 +150,7 @@ def test_run_build_captures_stderr_and_the_exit_code(): def test_env_spec_reports_a_base_distribution_that_is_not_installed(monkeypatch): - monkeypatch.setattr(TimeNetDistribution, "NAMES", ("timenet", "timenet-not-installed")) + monkeypatch.setattr(TimeNetSnapshot, "distributions", classmethod(lambda cls: ("timenet", "timenet-not-installed"))) with pytest.raises(TimeNetBuildError, match="timenet-not-installed"): env_spec("timenet/hello-world") diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 7066215fa..32abc908f 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -6,7 +6,7 @@ from timenet.client import TimeNet from timenet.config import settings -from timenet.provenance import build_env +from timenet.manifest import BuildEnvironment from timenet.registry import LocalRegistry from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal @@ -113,5 +113,5 @@ def test_an_isolated_build_records_its_environment(tmp_path): # pytest runs this process and is a runtime dependency of neither package, so it separates a # real isolated build from one that quietly inherited the parent's site-packages. Assert both # directions: without the first the second would also pass on a machine that simply lacks it. - assert "pytest" in build_env()["packages"] + assert "pytest" in BuildEnvironment.capture(git={}).packages assert "pytest" not in packages diff --git a/packages/timenet/src/timenet/manifest/__init__.py b/packages/timenet/src/timenet/manifest/__init__.py index 297f0101c..e7b6d5f3f 100644 --- a/packages/timenet/src/timenet/manifest/__init__.py +++ b/packages/timenet/src/timenet/manifest/__init__.py @@ -2,12 +2,21 @@ from timenet.manifest.counts import ManifestCounts from timenet.manifest.files import FilePart, ManifestFiles -from timenet.manifest.manifest import BuildEnvironment, GitSource, GitSourceMap, Manifest, parse_manifest, read_manifest +from timenet.manifest.manifest import ( + BuildEnvironment, + DistributionName, + GitSource, + GitSourceMap, + Manifest, + parse_manifest, + read_manifest, +) from timenet.manifest.payload_source import PayloadSource __all__ = [ "BuildEnvironment", + "DistributionName", "FilePart", "GitSource", "GitSourceMap", diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index b90100daa..584ef4920 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -1,10 +1,11 @@ """The :class:`Manifest`: the compiled ``manifest.json`` file and its JSON codec.""" +from importlib.metadata import distributions from pathlib import Path +import platform from typing import Annotated, Any, Self from pydantic import Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator -from typing_extensions import TypedDict from timenet.errors import ReleaseDifference, TimeFFormatError, TimeNetManifestError from timenet.manifest.counts import ManifestCounts @@ -38,17 +39,31 @@ def load_from_build_info(cls, path: Path) -> Self: return cls.model_validate_json(path.read_text(encoding="utf-8")) -GitSourceMap = dict[StrictStr, GitSource] -"""The commit of each TimeNet distribution, keyed by distribution name.""" +DistributionName = StrictStr +"""The name of an installed distribution, for example ``timenet-connectors``.""" +GitSourceMap = dict[DistributionName, GitSource] +"""The commit of each installed TimeNet distribution, keyed by distribution name.""" -class BuildEnvironment(TypedDict, total=False): - """Build-provenance fields recorded in a manifest.""" + +class BuildEnvironment(TimeFModel): + """The Python version, the installed packages, and the TimeNet commits that produced a version.""" python: StrictStr - packages: dict[StrictStr, StrictStr] + """The interpreter version, for example ``3.13.1``.""" + packages: dict[DistributionName, StrictStr] + """Each installed distribution and its version, sorted by name.""" git: GitSourceMap - """The commit of each TimeNet distribution. A distribution whose commit is unknown is absent.""" + """The commit of each installed TimeNet distribution. A distribution that is not installed is absent.""" + + @classmethod + def capture(cls, git: GitSourceMap) -> Self: + """Describe this interpreter, with ``git`` from ``TimeNetSnapshot.to_source_map()``.""" + # Under an isolated build the packages are exactly the base, the connector's requirements, + # and their transitives. Under --no-isolation they are the whole ambient environment, which + # is noisier but still the truth about what produced the dataset. + packages = {dist.metadata["Name"]: dist.version for dist in distributions() if dist.metadata["Name"]} + return cls(python=platform.python_version(), packages=dict(sorted(packages.items())), git=git) class Manifest(TimeFModel): @@ -76,8 +91,8 @@ class Manifest(TimeFModel): """Every release that stores a payload file this release reads, by source key, pinned to those files. The writer flattens them, so a source's own sources are not listed unless this release reads their files too.""" - build_env: BuildEnvironment = Field(default_factory=dict) - """The Python version and package set that produced this version.""" + build_env: BuildEnvironment + """The Python version, the package set, and the TimeNet commits that produced this version.""" timef_format_version: Annotated[int, Field(strict=True, ge=1, le=1)] """The TimeF manifest format version.""" diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 66f9ab1ca..a4552c08a 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,110 +1,157 @@ -"""Describe the environment a build ran in, for the manifest's provenance block.""" +"""Read the git commit of the installed TimeNet code, for the manifest's ``build_env.git`` block. -from collections.abc import Mapping +Two layers do this. :class:`InstalledDistribution` and its :class:`InstallSource` read the commit +of any one installed package. The snapshot types (:class:`PackageSnapshot`, :class:`GroupSnapshot`, +and :class:`TimeNetSnapshot`) are typed views of the commits that a manifest records. +""" + +from abc import ABC, abstractmethod from dataclasses import dataclass -from importlib.metadata import Distribution, PackageNotFoundError, distributions +from importlib.metadata import Distribution, PackageNotFoundError from pathlib import Path -import platform import shutil import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors -from typing import Self +import sys +from typing import ClassVar, Self from urllib.parse import urlparse from urllib.request import url2pathname -from pydantic import BaseModel, ConfigDict, StrictBool, StrictStr, ValidationError +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, ValidationError from timenet.errors import TimeNetBuildError -from timenet.manifest import BuildEnvironment, GitSource, GitSourceMap +from timenet.manifest import DistributionName, GitSource, GitSourceMap _GIT_TIMEOUT_S = 30 -def build_env() -> BuildEnvironment: - """Describe this process's Python version, installed packages, and TimeNet commits. +@dataclass(frozen=True) +class InstallSource(ABC): + """How one distribution was installed. The install type decides where its commit is recorded.""" - Under an isolated build the recorded set is exactly the base, the connector's requirements, and - their transitives. Under ``--no-isolation`` it is the whole ambient environment, which is noisier - but still the truth about what produced the dataset. + distribution: DistributionName - Returns: - A dict with ``python`` (the interpreter version), ``packages`` (name -> version, sorted), and - ``git`` (see :meth:`GitSnapshot.recorded_sources`). - """ - packages: dict[str, str] = {} - for dist in distributions(): - name = dist.metadata["Name"] - if name: - packages[name] = dist.version - return { - "python": platform.python_version(), - "packages": dict(sorted(packages.items())), - "git": GitSnapshot.read().recorded_sources(), - } + @abstractmethod + def read_git_source(self) -> GitSource: + """Return the commit of the code that this install runs. Raises ``TimeNetBuildError`` if it is unknown.""" + + def _unknown_commit(self, reason: str) -> TimeNetBuildError: + return TimeNetBuildError( + f"cannot find the git commit of {self.distribution}, which each build records in its manifest: {reason}" + ) @dataclass(frozen=True) -class UnknownGitSource: - """Why the commit of an installed TimeNet distribution is unknown.""" +class EditableCheckout(InstallSource): + """An editable install of a git checkout. Git reports the commit and the local changes.""" - reason: str + project_directory: Path + def read_git_source(self) -> GitSource: + """Ask git for the commit of the checkout and whether the package directory has changes.""" + git = shutil.which("git") + if git is None: + raise self._unknown_commit( + f"it is installed editable from {self.project_directory}, and git is not on PATH. Install git, " + f"or install {self.distribution} from a wheel that scripts/build_dists.py built" + ) + try: + sha = _run_git(git, self.project_directory, "rev-parse", "HEAD") + # The pathspec "." scopes the status to the package directory, untracked files included. A + # scratch file elsewhere in the repository does not count as a local change. + status = _run_git(git, self.project_directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or "").strip() or str(exc) + raise self._unknown_commit(f"git failed in {self.project_directory}: {detail}") from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise self._unknown_commit(f"git failed in {self.project_directory}: {exc}") from exc + return GitSource(sha=sha, contains_local_changes=bool(status)) -class TimeNetDistribution: - """One installed TimeNet distribution, and the commit of the code it runs. - Each install type finds its commit in its own place: +@dataclass(frozen=True) +class GitInstall(InstallSource): + """An install from a git URL (``pip install git+https://...``). Pip recorded the commit.""" + + commit_id: str + + def read_git_source(self) -> GitSource: + """Return the commit that pip installed, which has no local changes.""" + try: + return GitSource(sha=self.commit_id, contains_local_changes=False) + except ValidationError as exc: + raise TimeNetBuildError(f"{self.distribution} records an invalid commit in direct_url.json: {exc}") from exc - - An editable install asks git. ``contains_local_changes`` covers the package directory only, - so a scratch file elsewhere in the repository does not set it. - - A git install (``pip install git+https://...``) has the commit that pip recorded. - - Any other install, such as a wheel or ``pip install ./packages/timenet``, reads the - ``_build_info.json`` that ``scripts/build_dists.py`` wrote into the package. - """ - NAMES = ("timenet", "timenet-connectors") - """The distributions whose git commit a build records.""" +@dataclass(frozen=True) +class BuiltPackage(InstallSource): + """A wheel, or a path install that is not editable. The package carries its commit record.""" + + BUILD_INFO_FILE: ClassVar[str] = "_build_info.json" + """The commit record in each built package. ``scripts/build_dists.py`` writes it.""" + + build_info: Path - BUILD_INFO_FILE = "_build_info.json" - """The commit record in each package of a wheel. ``scripts/build_dists.py`` uses the same name.""" + def read_git_source(self) -> GitSource: + """Read the commit that ``scripts/build_dists.py`` wrote into the package.""" + if not self.build_info.is_file(): + raise self._unknown_commit( + f"the installed package carries no {self.BUILD_INFO_FILE}. Build it with `make build` or " + f"scripts/build_dists.py, or upgrade {self.distribution} to a release that records its commit" + ) + try: + return GitSource.load_from_build_info(self.build_info) + except ValidationError as exc: + raise TimeNetBuildError( + f"{self.distribution} records an invalid commit in {self.build_info}: {exc}" + ) from exc + + +class InstalledDistribution: + """One distribution that is installed in this interpreter.""" def __init__(self, distribution: Distribution) -> None: self._distribution = distribution @classmethod - def installed(cls) -> list[Self]: - """Return each TimeNet distribution that is installed in this interpreter.""" - installed: list[Self] = [] - for name in cls.NAMES: - try: - installed.append(cls(Distribution.from_name(name))) - except PackageNotFoundError: - continue - return installed + def from_name(cls, name: DistributionName) -> Self: + """Find an installed distribution. Raises ``PackageNotFoundError`` if it is not installed.""" + return cls(Distribution.from_name(name)) @property - def name(self) -> str: + def name(self) -> DistributionName: """The distribution name, for example ``timenet-connectors``.""" return self._distribution.metadata["Name"] + @property + def version(self) -> str: + """The installed version.""" + return self._distribution.version + + def install_source(self) -> InstallSource: + """Return how this distribution was installed, which decides where its commit is recorded.""" + direct_url = self._read_direct_url() + if direct_url is not None: + commit_id = direct_url.installed_git_commit() + if commit_id is not None: + return GitInstall(self.name, commit_id) + project_directory = direct_url.local_directory() + if direct_url.is_editable() and project_directory is not None: + return EditableCheckout(self.name, project_directory) + # An install from an index has no direct_url.json. A path install that is not editable + # copied the package, so the live checkout can differ from the code that runs. + module = self.name.replace("-", "_") + build_info = self._distribution.locate_file(f"{module}/{BuiltPackage.BUILD_INFO_FILE}") + return BuiltPackage(self.name, Path(str(build_info))) + + def read_git_source(self) -> GitSource: + """Return the commit of the code that this distribution runs. Raises ``TimeNetBuildError`` if it is unknown.""" + return self.install_source().read_git_source() + def path_to_ondisk_project_directory(self) -> Path | None: """Return the project directory of an editable or local path install, else ``None``.""" direct_url = self._read_direct_url() return None if direct_url is None else direct_url.local_directory() - def read_git_source(self) -> GitSource | UnknownGitSource: - """Return the commit of the code that this distribution runs, or why it is unknown.""" - direct_url = self._read_direct_url() - if direct_url is None: - return self._read_build_info_git_source() - installed_commit = direct_url.installed_git_commit() - if installed_commit is not None: - return self._git_source_of_installed_commit(installed_commit) - project_directory = direct_url.local_directory() - if direct_url.is_editable() and project_directory is not None: - return TimeNetDistribution._read_checkout_git_source(project_directory) - return self._read_build_info_git_source() - def _read_direct_url(self) -> "_DirectUrl | None": """Read the PEP 610 ``direct_url.json``, or return ``None`` for an install from an index.""" raw = self._distribution.read_text("direct_url.json") @@ -115,59 +162,107 @@ def _read_direct_url(self) -> "_DirectUrl | None": except ValidationError as exc: raise TimeNetBuildError(f"{self.name} has a malformed direct_url.json: {exc}") from exc - def _git_source_of_installed_commit(self, sha: str) -> GitSource: - """Return the clean commit record of a git install, and raise if pip recorded an invalid sha.""" - try: - return GitSource(sha=sha, contains_local_changes=False) - except ValidationError as exc: - raise TimeNetBuildError(f"{self.name} records an invalid commit in direct_url.json: {exc}") from exc - @staticmethod - def _read_checkout_git_source(project_directory: Path) -> GitSource | UnknownGitSource: - """Ask git for the commit of a checkout and whether its package directory has changes.""" - git = shutil.which("git") - if git is None: - return UnknownGitSource("git is not on PATH") +class Snapshot(BaseModel, ABC): + """The commit of some installed code, read at one point in time.""" + + model_config = ConfigDict(frozen=True, validate_by_name=True) + + @abstractmethod + def to_source_map(self) -> GitSourceMap: + """Return the commits in the form that a manifest records.""" + + +class PackageSnapshot(Snapshot): + """The commit of one installed distribution.""" + + distribution: DistributionName + source: GitSource + + @classmethod + def read(cls, distribution: DistributionName) -> Self: + """Read the commit of an installed distribution. Raises ``TimeNetBuildError`` if it is not installed.""" + snapshot = cls.read_if_installed(distribution) + if snapshot is None: + raise TimeNetBuildError( + f"cannot find the git commit of {distribution}: it is not installed in {sys.executable}" + ) + return snapshot + + @classmethod + def read_if_installed(cls, distribution: DistributionName) -> Self | None: + """Read the commit of a distribution, or return ``None`` if it is not installed.""" try: - sha = _run_git(git, project_directory, "rev-parse", "HEAD") - # The pathspec "." scopes the status to the package directory, untracked files included. - status = _run_git(git, project_directory, "status", "--porcelain", "--untracked-files=all", "--", ".") - except subprocess.CalledProcessError as exc: - return UnknownGitSource(f"git failed in {project_directory}: {(exc.stderr or '').strip() or exc}") - except (OSError, subprocess.TimeoutExpired) as exc: - return UnknownGitSource(f"git failed in {project_directory}: {exc}") - return GitSource(sha=sha, contains_local_changes=bool(status)) + installed = InstalledDistribution.from_name(distribution) + except PackageNotFoundError: + return None + return cls(distribution=distribution, source=installed.read_git_source()) - def _read_build_info_git_source(self) -> GitSource | UnknownGitSource: - """Read the commit that ``scripts/build_dists.py`` wrote into the installed package.""" - module = self.name.replace("-", "_") - path = Path(str(self._distribution.locate_file(f"{module}/{self.BUILD_INFO_FILE}"))) - if not path.is_file(): - return UnknownGitSource(f"the installed package carries no {self.BUILD_INFO_FILE}") + def to_source_map(self) -> GitSourceMap: + """Return this package's commit in the form that a manifest records.""" + return {self.distribution: self.source} + + +class GroupSnapshot(Snapshot, ABC): + """The commits of a fixed group of distributions, one :class:`PackageSnapshot` field each. + + A field takes its name from its distribution, or carries the distribution name as its alias. A + field that may be ``None`` is a distribution that need not be installed. + """ + + @classmethod + @abstractmethod + def read(cls) -> Self: + """Read the commit of each distribution in the group.""" + + @classmethod + def distributions(cls) -> tuple[DistributionName, ...]: + """Return the name of each distribution in the group.""" + return tuple(field.alias or name for name, field in cls.model_fields.items()) + + @classmethod + def from_source_map(cls, sources: GitSourceMap) -> Self: + """Return the typed view of the commits that a manifest records.""" + packages = {name: PackageSnapshot(distribution=name, source=source) for name, source in sources.items()} try: - return GitSource.load_from_build_info(path) + # The field aliases map each distribution name to its field. + return cls.model_validate(packages) except ValidationError as exc: - raise TimeNetBuildError(f"{self.name} records an invalid commit in {path}: {exc}") from exc + missing = ", ".join(str(error["loc"][0]) for error in exc.errors()) + raise TimeNetBuildError(f"the manifest records no git commit for {missing}") from exc + def to_source_map(self) -> GitSourceMap: + """Return the commit of each installed distribution in the group.""" + sources: GitSourceMap = {} + for snapshot in self._snapshots(): + sources.update(snapshot.to_source_map()) + return sources + + def _snapshots(self) -> list[Snapshot]: + """Return the snapshot of each installed distribution in the group.""" + children = (getattr(self, name) for name in type(self).model_fields) + return [child for child in children if child is not None] -@dataclass(frozen=True) -class GitSnapshot: - """The commit of each installed TimeNet distribution, read at one point in time.""" - sources: Mapping[str, GitSource | UnknownGitSource] - """The commit, or why it is unknown, keyed by distribution name.""" +class TimeNetSnapshot(GroupSnapshot): + """The commit of each installed TimeNet distribution, read at one point in time. + + Each build records it in its manifest. This class is the one list of the distributions that + count as TimeNet code. + """ + + timenet: PackageSnapshot + timenet_connectors: PackageSnapshot | None = Field(default=None, alias="timenet-connectors") + """``None`` if timenet-connectors is not installed.""" @classmethod def read(cls) -> Self: - """Read the commit of each installed TimeNet distribution. This is the only call that runs git.""" + """Read the commit of each installed TimeNet distribution. Raises ``TimeNetBuildError`` if one is unknown.""" return cls( - {distribution.name: distribution.read_git_source() for distribution in TimeNetDistribution.installed()} + timenet=PackageSnapshot.read("timenet"), + timenet_connectors=PackageSnapshot.read_if_installed("timenet-connectors"), ) - def recorded_sources(self) -> GitSourceMap: - """Return the known commits, in the form that a manifest records.""" - return {name: source for name, source in self.sources.items() if isinstance(source, GitSource)} - class _DirInfo(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/packages/timenet/src/timenet/schemas/manifest.schema.json b/packages/timenet/src/timenet/schemas/manifest.schema.json index 984b5ed3d..ab4f926aa 100644 --- a/packages/timenet/src/timenet/schemas/manifest.schema.json +++ b/packages/timenet/src/timenet/schemas/manifest.schema.json @@ -86,7 +86,7 @@ }, "BuildEnvironment": { "additionalProperties": false, - "description": "Build-provenance fields recorded in a manifest.", + "description": "The Python version, the installed packages, and the TimeNet commits that produced a version.", "properties": { "python": { "title": "Python", @@ -107,6 +107,11 @@ "type": "object" } }, + "required": [ + "python", + "packages", + "git" + ], "title": "BuildEnvironment", "type": "object" }, @@ -779,6 +784,7 @@ "dataset_id", "metadata", "files", + "build_env", "timef_format_version" ], "title": "Manifest", diff --git a/packages/timenet/src/timenet/writer/writer.py b/packages/timenet/src/timenet/writer/writer.py index f8e094d92..95540b11e 100644 --- a/packages/timenet/src/timenet/writer/writer.py +++ b/packages/timenet/src/timenet/writer/writer.py @@ -27,8 +27,8 @@ from timenet.format.storage import StorageSources from timenet.format.stored_array import StoredArray from timenet.format.stored_values import PackedValueWriter -from timenet.manifest import FilePart, Manifest, ManifestCounts, ManifestFiles, PayloadSource -from timenet.provenance import build_env +from timenet.manifest import BuildEnvironment, FilePart, Manifest, ManifestCounts, ManifestFiles, PayloadSource +from timenet.provenance import TimeNetSnapshot from timenet.types.ids import EntityId from timenet.types.reference_keys import SourceKey from timenet.types.references import DatasetRef @@ -475,7 +475,7 @@ def _write_manifest(self) -> Manifest: payload_sources=self._storage_sources.sources, parent_cards=import_state.parent_cards, lineage=tuple(import_state.lineage.values()), - build_env=build_env(), + build_env=BuildEnvironment.capture(git=TimeNetSnapshot.read().to_source_map()), timef_format_version=1, ) (self._staging_dir / MANIFEST_FILE).write_text(manifest.to_json()) diff --git a/packages/timenet/tests/format/test_control_cache.py b/packages/timenet/tests/format/test_control_cache.py index 18746c4ac..acbc0242a 100644 --- a/packages/timenet/tests/format/test_control_cache.py +++ b/packages/timenet/tests/format/test_control_cache.py @@ -6,7 +6,7 @@ from timenet.format.control_writer import DuckDBControlWriter from timenet.format.storage import StorageSources from timenet.format.stored_values import PackedValueWriter -from timenet.manifest import FilePart, Manifest, ManifestFiles +from timenet.manifest import BuildEnvironment, FilePart, Manifest, ManifestFiles from timenet.registry.version import DatasetVersion from .test_control_writer import _dataset @@ -22,6 +22,7 @@ def test_local_control_database_uses_its_existing_path(tmp_path): files=ManifestFiles( control=(FilePart(path=path.name, checksum=file_checksum(path), size=path.stat().st_size),), ), + build_env=BuildEnvironment(python="3.13.1", packages={}, git={}), timef_format_version=1, ) version = DatasetVersion( diff --git a/packages/timenet/tests/manifest/test_manifest.py b/packages/timenet/tests/manifest/test_manifest.py index c31128686..3eaa40631 100644 --- a/packages/timenet/tests/manifest/test_manifest.py +++ b/packages/timenet/tests/manifest/test_manifest.py @@ -1,3 +1,4 @@ +import sys from typing import Any, TypeVar, cast from hypothesis import given, strategies as st @@ -6,7 +7,15 @@ from pydantic import BaseModel, ValidationError import pytest -from timenet.manifest import FilePart, Manifest, ManifestCounts, ManifestFiles, PayloadSource +from timenet.manifest import ( + BuildEnvironment, + FilePart, + GitSource, + Manifest, + ManifestCounts, + ManifestFiles, + PayloadSource, +) from timenet.schemas import MANIFEST_SCHEMA from timenet.types import ( AnnotationDescriptor, @@ -31,6 +40,11 @@ _CONTROL = (FilePart(path="control.duckdb", checksum="sha256:" + "0" * 64, size=5),) +_BUILD_ENV = BuildEnvironment( + python="3.13.1", + packages={"timenet": "0.1.0"}, + git={"timenet": GitSource(sha="a" * 40, contains_local_changes=False)}, +) _ModelT = TypeVar("_ModelT", bound=BaseModel) @@ -89,6 +103,7 @@ def _manifest(*, values_backend: str = ValuesBackend.PARQUET) -> Manifest: ), ), values_backend=cast("ValuesBackend", values_backend), + build_env=_BUILD_ENV, timef_format_version=1, ) @@ -161,6 +176,7 @@ def test_missing_values_round_trip_at_format_version_1(): metadata=base.metadata, files=base.files, dataset_schema=_replace(base.dataset_schema, time_series_specs=(spec,)), + build_env=_BUILD_ENV, timef_format_version=1, ) assert manifest.timef_format_version == 1 @@ -299,6 +315,7 @@ def test_direct_construction_validates_format_version(): dataset_id="x", metadata=_manifest().metadata, files=_manifest().files, + build_env=_BUILD_ENV, timef_format_version=99, ) @@ -309,11 +326,12 @@ def test_dataset_id_must_match_metadata(): dataset_id="other/dataset", metadata=_manifest().metadata, files=_manifest().files, + build_env=_BUILD_ENV, timef_format_version=1, ) -@pytest.mark.parametrize("missing", ["timef_format_version", "dataset_id", "metadata", "files"]) +@pytest.mark.parametrize("missing", ["timef_format_version", "dataset_id", "metadata", "files", "build_env"]) def test_model_validate_requires_core_blocks(missing): d = _manifest().to_dict() del d[missing] @@ -457,6 +475,7 @@ def test_codec_roundtrip_property(version, records, task_counts): ), counts=ManifestCounts(records=records, tasks=task_counts), files=ManifestFiles(control=_CONTROL), + build_env=_BUILD_ENV, timef_format_version=1, ) assert Manifest.model_validate_json(manifest.to_json()) == manifest @@ -483,8 +502,12 @@ def test_bad_dict_block_names_itself(block): Manifest.model_validate(d) -def test_build_env_defaults_to_empty(): - assert _manifest().build_env == {} +def test_build_env_captures_the_running_python_and_the_installed_packages(): + build_env = BuildEnvironment.capture(git=_BUILD_ENV.git) + assert build_env.python == ".".join(str(part) for part in sys.version_info[:3]) + assert build_env.packages["timenet"] + assert list(build_env.packages) == sorted(build_env.packages) + assert build_env.git == _BUILD_ENV.git def test_build_env_round_trips(): @@ -496,9 +519,19 @@ def test_build_env_round_trips(): @pytest.mark.parametrize( "change", [ - {"build_env": {"python": 313}}, - {"build_env": {"git": {"timenet": {"sha": "4818e37f", "contains_local_changes": False}}}}, - {"build_env": {"git": {"timenet": {"sha": "a" * 40, "contains_local_changes": "no"}}}}, + {"build_env": {**_BUILD_ENV.model_dump(), "python": 313}}, + { + "build_env": { + **_BUILD_ENV.model_dump(), + "git": {"timenet": {"sha": "4818e37f", "contains_local_changes": False}}, + } + }, + { + "build_env": { + **_BUILD_ENV.model_dump(), + "git": {"timenet": {"sha": "a" * 40, "contains_local_changes": "no"}}, + } + }, {"value_encoding": {"ecg_lead": "zip"}}, ], ) @@ -508,17 +541,16 @@ def test_direct_manifest_rejects_invalid_provenance_blocks(change): def test_to_dict_copies_build_env(): - build_env = {"python": "3.11.9", "packages": {"timenet": "0.1.0"}} - d = _replace(_manifest(), build_env=build_env).to_dict() - d["build_env"]["python"] = "2.7.0" - assert build_env["python"] == "3.11.9" + manifest = _manifest() + manifest.to_dict()["build_env"]["packages"]["timenet"] = "2.7.0" + assert manifest.build_env.packages["timenet"] == "0.1.0" @pytest.mark.parametrize("method", ["to_dict", "to_json"]) @pytest.mark.parametrize( "mutate", [ - lambda manifest: manifest.build_env.update(python=313), + lambda manifest: manifest.build_env.packages.update(timenet=313), lambda manifest: manifest.counts.tasks.update(classification=-1), ], ) diff --git a/packages/timenet/tests/manifest/test_manifest_schema.py b/packages/timenet/tests/manifest/test_manifest_schema.py index c07b21545..bdfe30c9b 100644 --- a/packages/timenet/tests/manifest/test_manifest_schema.py +++ b/packages/timenet/tests/manifest/test_manifest_schema.py @@ -1,7 +1,7 @@ import jsonschema import pytest -from timenet.manifest import FilePart, Manifest, ManifestCounts, ManifestFiles +from timenet.manifest import BuildEnvironment, FilePart, GitSource, Manifest, ManifestCounts, ManifestFiles from timenet.schemas import MANIFEST_SCHEMA from timenet.types import ( AnnotationDescriptor, @@ -18,6 +18,13 @@ ) +_BUILD_ENV = BuildEnvironment( + python="3.13.1", + packages={"timenet": "0.1.0"}, + git={"timenet": GitSource(sha="a" * 40, contains_local_changes=False)}, +) + + def _manifest() -> Manifest: spec = TimeSeriesSpec( spec_type="ecg", @@ -72,6 +79,7 @@ def _manifest() -> Manifest: ), ), ), + build_env=_BUILD_ENV, timef_format_version=1, ) diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 2996ca0b8..504cc1889 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -4,23 +4,17 @@ from pathlib import Path import shutil import subprocess -import sys import pytest from timenet.errors import TimeNetBuildError from timenet.manifest import GitSource -from timenet.provenance import GitSnapshot, TimeNetDistribution, build_env +from timenet.provenance import InstalledDistribution, PackageSnapshot, TimeNetSnapshot +from timenet.registry import LocalRegistry +from timenet.testing import make_dataset -def test_build_env_records_the_running_python(): - assert build_env()["python"] == ".".join(str(part) for part in sys.version_info[:3]) - - -def test_build_env_records_installed_packages(): - packages = build_env()["packages"] - assert packages["timenet"] - assert packages == dict(sorted(packages.items())) +_CLEAN = GitSource(sha="a" * 40, contains_local_changes=False) @pytest.fixture @@ -95,7 +89,8 @@ def _editable(directory: Path) -> dict: def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, installed): installed(direct_url=_editable(checkout)) head = _run(git, checkout, "rev-parse", "HEAD") - assert build_env()["git"] == {"timenet": GitSource(sha=head, contains_local_changes=False)} + # timenet-connectors is not installed, so the snapshot has no entry for it. + assert TimeNetSnapshot.read().to_source_map() == {"timenet": GitSource(sha=head, contains_local_changes=False)} @pytest.mark.real_git @@ -103,7 +98,7 @@ def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, i def test_a_change_in_the_package_directory_is_a_local_change(checkout, installed, change): installed(direct_url=_editable(checkout)) (checkout / change).write_text("x = 2\n") - assert GitSnapshot.read().recorded_sources()["timenet"].contains_local_changes is True + assert PackageSnapshot.read("timenet").source.contains_local_changes is True @pytest.mark.real_git @@ -111,15 +106,24 @@ def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout, installed(direct_url=_editable(checkout)) (checkout.parents[1] / "README.md").write_text("edited\n") (checkout.parents[1] / "scratch.ipynb").write_text("{}\n") - assert GitSnapshot.read().recorded_sources()["timenet"].contains_local_changes is False + assert PackageSnapshot.read("timenet").source.contains_local_changes is False @pytest.mark.real_git -def test_an_editable_install_outside_git_has_no_commit(git, tmp_path, installed): +def test_an_editable_install_outside_git_fails_with_the_git_error(git, tmp_path, installed): outside = tmp_path / "outside" outside.mkdir() installed(direct_url=_editable(outside)) - assert GitSnapshot.read().recorded_sources() == {} + with pytest.raises(TimeNetBuildError, match=r"cannot find the git commit of timenet.*not a git repository"): + PackageSnapshot.read("timenet") + + +@pytest.mark.real_git +def test_an_editable_install_without_git_fails(checkout, installed, monkeypatch): + installed(direct_url=_editable(checkout)) + monkeypatch.setenv("PATH", "") + with pytest.raises(TimeNetBuildError, match="git is not on PATH"): + PackageSnapshot.read("timenet") @pytest.mark.real_git @@ -130,39 +134,38 @@ def test_an_install_of_a_subdirectory_finds_the_package_directory(checkout, inst installed( direct_url={"url": repository.as_uri(), "dir_info": {"editable": editable}, "subdirectory": "packages/pkg"} ) - (distribution,) = TimeNetDistribution.installed() - assert distribution.path_to_ondisk_project_directory() == checkout + assert InstalledDistribution.from_name("timenet").path_to_ondisk_project_directory() == checkout if editable: # A change outside the package directory does not count. (repository / "README.md").write_text("edited\n") - assert GitSnapshot.read().recorded_sources()["timenet"].contains_local_changes is False + assert PackageSnapshot.read("timenet").source.contains_local_changes is False @pytest.mark.real_git def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): - info = GitSource(sha="a" * 40, contains_local_changes=False) - installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=info.model_dump_json()) - assert GitSnapshot.read().recorded_sources() == {"timenet": info} + installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=_CLEAN.model_dump_json()) + assert PackageSnapshot.read("timenet").source == _CLEAN @pytest.mark.real_git def test_a_git_install_records_the_commit_that_pip_installed(installed): sha = "b" * 40 installed(direct_url={"url": "https://example.org/timenet.git", "vcs_info": {"vcs": "git", "commit_id": sha}}) - assert GitSnapshot.read().recorded_sources() == {"timenet": GitSource(sha=sha, contains_local_changes=False)} + assert PackageSnapshot.read("timenet").source == GitSource(sha=sha, contains_local_changes=False) @pytest.mark.real_git def test_a_wheel_reads_its_commit_from_the_build_info(installed): info = GitSource(sha="a" * 40, contains_local_changes=True) installed(build_info=info.model_dump_json()) - assert GitSnapshot.read().recorded_sources() == {"timenet": info} + assert PackageSnapshot.read("timenet").source == info @pytest.mark.real_git -def test_a_wheel_without_build_info_has_no_commit(installed): +def test_a_wheel_without_build_info_fails_and_says_how_to_build_one(installed): installed() - assert GitSnapshot.read().recorded_sources() == {} + with pytest.raises(TimeNetBuildError, match=r"carries no _build_info\.json\. Build it with `make build`"): + PackageSnapshot.read("timenet") @pytest.mark.real_git @@ -178,4 +181,27 @@ def test_a_wheel_without_build_info_has_no_commit(installed): def test_a_wheel_with_invalid_build_info_fails(installed, build_info): installed(build_info=build_info) with pytest.raises(TimeNetBuildError, match=r"_build_info\.json"): - GitSnapshot.read().recorded_sources() + PackageSnapshot.read("timenet") + + +@pytest.mark.real_git +def test_a_local_build_fails_if_a_commit_is_unknown(installed, tmp_path): + installed() + with pytest.raises(TimeNetBuildError, match="cannot find the git commit of timenet"): + LocalRegistry(tmp_path / "local").store(make_dataset()) + + +def test_a_snapshot_reads_back_from_the_commits_that_a_manifest_records(): + snapshot = TimeNetSnapshot.read() + assert TimeNetSnapshot.from_source_map(snapshot.to_source_map()) == snapshot + + +def test_a_manifest_without_timenet_connectors_reads_as_not_installed(): + snapshot = TimeNetSnapshot.from_source_map({"timenet": _CLEAN}) + assert snapshot.timenet_connectors is None + assert snapshot.to_source_map() == {"timenet": _CLEAN} + + +def test_a_manifest_without_the_commit_of_timenet_is_refused(): + with pytest.raises(TimeNetBuildError, match="the manifest records no git commit for timenet"): + TimeNetSnapshot.from_source_map({"timenet-connectors": _CLEAN}) diff --git a/scripts/build_dists.py b/scripts/build_dists.py index 29dad87d8..8df4ecb46 100644 --- a/scripts/build_dists.py +++ b/scripts/build_dists.py @@ -20,7 +20,7 @@ ROOT / "packages" / "timenet": "timenet", ROOT / "packages" / "timenet-connectors": "timenet_connectors", } -# Keep in step with timenet.provenance.TimeNetDistribution.BUILD_INFO_FILE, which reads the file. +# Keep in step with timenet.provenance.BuiltPackage.BUILD_INFO_FILE, which reads the file. BUILD_INFO_FILE = "_build_info.json" From f08dec8103683c5a0e7bda36ac41219705c67faa Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 14:52:14 +0100 Subject: [PATCH 10/19] refactor(provenance): drop GroupSnapshot and check each snapshot field TimeNetSnapshot now derives directly from Snapshot. It owns read, from_source_map and to_source_map. The GroupSnapshot layer is gone. A validator makes each PackageSnapshot field hold the commit of its own distribution. The timenet field must hold timenet, and the timenet-connectors field must hold timenet-connectors. BuiltPackage now reads its _build_info.json file. GitSource only validates the JSON, so load_from_build_info is gone. The pytest marker real_git is now real_provenance, because it turns off the stub for every install type, wheels included. The autouse fixture is now _stub_installed_sources. Part of TNET-165 --- packages/conftest.py | 11 +-- .../timenet/src/timenet/manifest/manifest.py | 5 -- packages/timenet/src/timenet/provenance.py | 76 +++++++++---------- packages/timenet/tests/test_provenance.py | 31 +++++--- pyproject.toml | 2 +- 5 files changed, 62 insertions(+), 63 deletions(-) diff --git a/packages/conftest.py b/packages/conftest.py index 46d82ae56..00b9af5cd 100644 --- a/packages/conftest.py +++ b/packages/conftest.py @@ -1,8 +1,9 @@ """Shared pytest configuration for every workspace package's tests. -Each manifest write asks git for the commit of the TimeNet distributions. Real git calls would make -a test's result depend on the state of this checkout, and they cost seconds across the suite. So -every installed distribution reports a clean commit, unless the test carries the ``real_git`` marker. +Each manifest write reads the commit of the TimeNet distributions. For an editable install this +asks git, so the result would depend on the state of this checkout, and the calls cost seconds +across the suite. So every installed distribution reports a clean commit, whatever its install +type, unless the test carries the ``real_provenance`` marker. """ import pytest @@ -15,6 +16,6 @@ @pytest.fixture(autouse=True) -def _committed_checkout(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: - if request.node.get_closest_marker("real_git") is None: +def _stub_installed_sources(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + if request.node.get_closest_marker("real_provenance") is None: monkeypatch.setattr(InstalledDistribution, "read_git_source", lambda _self: _CLEAN) diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index 584ef4920..57ad90b90 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -33,11 +33,6 @@ class GitSource(TimeFModel): contains_local_changes: StrictBool """Whether the package directory had uncommitted changes, untracked files included.""" - @classmethod - def load_from_build_info(cls, path: Path) -> Self: - """Read the record that ``scripts/build_dists.py`` wrote into a built package.""" - return cls.model_validate_json(path.read_text(encoding="utf-8")) - DistributionName = StrictStr """The name of an installed distribution, for example ``timenet-connectors``.""" diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index a4552c08a..758741f6f 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,8 +1,8 @@ """Read the git commit of the installed TimeNet code, for the manifest's ``build_env.git`` block. Two layers do this. :class:`InstalledDistribution` and its :class:`InstallSource` read the commit -of any one installed package. The snapshot types (:class:`PackageSnapshot`, :class:`GroupSnapshot`, -and :class:`TimeNetSnapshot`) are typed views of the commits that a manifest records. +of any one installed package. The snapshot types (:class:`PackageSnapshot` and +:class:`TimeNetSnapshot`) are typed views of the commits that a manifest records. """ from abc import ABC, abstractmethod @@ -16,7 +16,7 @@ from urllib.parse import urlparse from urllib.request import url2pathname -from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, ValidationError +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, ValidationError, model_validator from timenet.errors import TimeNetBuildError from timenet.manifest import DistributionName, GitSource, GitSourceMap @@ -99,7 +99,7 @@ def read_git_source(self) -> GitSource: f"scripts/build_dists.py, or upgrade {self.distribution} to a release that records its commit" ) try: - return GitSource.load_from_build_info(self.build_info) + return GitSource.model_validate_json(self.build_info.read_text(encoding="utf-8")) except ValidationError as exc: raise TimeNetBuildError( f"{self.distribution} records an invalid commit in {self.build_info}: {exc}" @@ -203,23 +203,40 @@ def to_source_map(self) -> GitSourceMap: return {self.distribution: self.source} -class GroupSnapshot(Snapshot, ABC): - """The commits of a fixed group of distributions, one :class:`PackageSnapshot` field each. +class TimeNetSnapshot(Snapshot): + """The commit of each installed TimeNet distribution, read at one point in time. - A field takes its name from its distribution, or carries the distribution name as its alias. A - field that may be ``None`` is a distribution that need not be installed. + Each build records it in its manifest. Its fields are the one list of the distributions that + count as TimeNet code. Each field carries the name of its distribution, as its alias or as its + own name. """ - @classmethod - @abstractmethod - def read(cls) -> Self: - """Read the commit of each distribution in the group.""" + timenet: PackageSnapshot + timenet_connectors: PackageSnapshot | None = Field(default=None, alias="timenet-connectors") + """``None`` if timenet-connectors is not installed.""" + + @model_validator(mode="after") + def _each_field_holds_its_distribution(self) -> Self: + for name, field in type(self).model_fields.items(): + package = getattr(self, name) + distribution = field.alias or name + if package is not None and package.distribution != distribution: + raise ValueError(f"the field for {distribution} holds the commit of {package.distribution}") + return self @classmethod def distributions(cls) -> tuple[DistributionName, ...]: - """Return the name of each distribution in the group.""" + """Return the name of each TimeNet distribution, in field order.""" return tuple(field.alias or name for name, field in cls.model_fields.items()) + @classmethod + def read(cls) -> Self: + """Read the commit of each installed TimeNet distribution. Raises ``TimeNetBuildError`` if one is unknown.""" + return cls( + timenet=PackageSnapshot.read("timenet"), + timenet_connectors=PackageSnapshot.read_if_installed("timenet-connectors"), + ) + @classmethod def from_source_map(cls, sources: GitSourceMap) -> Self: """Return the typed view of the commits that a manifest records.""" @@ -232,36 +249,15 @@ def from_source_map(cls, sources: GitSourceMap) -> Self: raise TimeNetBuildError(f"the manifest records no git commit for {missing}") from exc def to_source_map(self) -> GitSourceMap: - """Return the commit of each installed distribution in the group.""" + """Return the commit of each installed TimeNet distribution.""" sources: GitSourceMap = {} - for snapshot in self._snapshots(): - sources.update(snapshot.to_source_map()) + for package in self._installed(): + sources.update(package.to_source_map()) return sources - def _snapshots(self) -> list[Snapshot]: - """Return the snapshot of each installed distribution in the group.""" - children = (getattr(self, name) for name in type(self).model_fields) - return [child for child in children if child is not None] - - -class TimeNetSnapshot(GroupSnapshot): - """The commit of each installed TimeNet distribution, read at one point in time. - - Each build records it in its manifest. This class is the one list of the distributions that - count as TimeNet code. - """ - - timenet: PackageSnapshot - timenet_connectors: PackageSnapshot | None = Field(default=None, alias="timenet-connectors") - """``None`` if timenet-connectors is not installed.""" - - @classmethod - def read(cls) -> Self: - """Read the commit of each installed TimeNet distribution. Raises ``TimeNetBuildError`` if one is unknown.""" - return cls( - timenet=PackageSnapshot.read("timenet"), - timenet_connectors=PackageSnapshot.read_if_installed("timenet-connectors"), - ) + def _installed(self) -> list[PackageSnapshot]: + """Return the snapshot of each installed TimeNet distribution.""" + return [package for package in (self.timenet, self.timenet_connectors) if package is not None] class _DirInfo(BaseModel): diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 504cc1889..43b100246 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -5,6 +5,7 @@ import shutil import subprocess +from pydantic import ValidationError import pytest from timenet.errors import TimeNetBuildError @@ -85,7 +86,7 @@ def _editable(directory: Path) -> dict: return {"url": directory.as_uri(), "dir_info": {"editable": True}} -@pytest.mark.real_git +@pytest.mark.real_provenance def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, installed): installed(direct_url=_editable(checkout)) head = _run(git, checkout, "rev-parse", "HEAD") @@ -93,7 +94,7 @@ def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, i assert TimeNetSnapshot.read().to_source_map() == {"timenet": GitSource(sha=head, contains_local_changes=False)} -@pytest.mark.real_git +@pytest.mark.real_provenance @pytest.mark.parametrize("change", ["module.py", "untracked.py"]) def test_a_change_in_the_package_directory_is_a_local_change(checkout, installed, change): installed(direct_url=_editable(checkout)) @@ -101,7 +102,7 @@ def test_a_change_in_the_package_directory_is_a_local_change(checkout, installed assert PackageSnapshot.read("timenet").source.contains_local_changes is True -@pytest.mark.real_git +@pytest.mark.real_provenance def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout, installed): installed(direct_url=_editable(checkout)) (checkout.parents[1] / "README.md").write_text("edited\n") @@ -109,7 +110,7 @@ def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout, assert PackageSnapshot.read("timenet").source.contains_local_changes is False -@pytest.mark.real_git +@pytest.mark.real_provenance def test_an_editable_install_outside_git_fails_with_the_git_error(git, tmp_path, installed): outside = tmp_path / "outside" outside.mkdir() @@ -118,7 +119,7 @@ def test_an_editable_install_outside_git_fails_with_the_git_error(git, tmp_path, PackageSnapshot.read("timenet") -@pytest.mark.real_git +@pytest.mark.real_provenance def test_an_editable_install_without_git_fails(checkout, installed, monkeypatch): installed(direct_url=_editable(checkout)) monkeypatch.setenv("PATH", "") @@ -126,7 +127,7 @@ def test_an_editable_install_without_git_fails(checkout, installed, monkeypatch) PackageSnapshot.read("timenet") -@pytest.mark.real_git +@pytest.mark.real_provenance @pytest.mark.parametrize("editable", [True, False]) def test_an_install_of_a_subdirectory_finds_the_package_directory(checkout, installed, editable): # direct_url.json may name the repository and put the package in "subdirectory". @@ -141,34 +142,34 @@ def test_an_install_of_a_subdirectory_finds_the_package_directory(checkout, inst assert PackageSnapshot.read("timenet").source.contains_local_changes is False -@pytest.mark.real_git +@pytest.mark.real_provenance def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=_CLEAN.model_dump_json()) assert PackageSnapshot.read("timenet").source == _CLEAN -@pytest.mark.real_git +@pytest.mark.real_provenance def test_a_git_install_records_the_commit_that_pip_installed(installed): sha = "b" * 40 installed(direct_url={"url": "https://example.org/timenet.git", "vcs_info": {"vcs": "git", "commit_id": sha}}) assert PackageSnapshot.read("timenet").source == GitSource(sha=sha, contains_local_changes=False) -@pytest.mark.real_git +@pytest.mark.real_provenance def test_a_wheel_reads_its_commit_from_the_build_info(installed): info = GitSource(sha="a" * 40, contains_local_changes=True) installed(build_info=info.model_dump_json()) assert PackageSnapshot.read("timenet").source == info -@pytest.mark.real_git +@pytest.mark.real_provenance def test_a_wheel_without_build_info_fails_and_says_how_to_build_one(installed): installed() with pytest.raises(TimeNetBuildError, match=r"carries no _build_info\.json\. Build it with `make build`"): PackageSnapshot.read("timenet") -@pytest.mark.real_git +@pytest.mark.real_provenance @pytest.mark.parametrize( "build_info", [ @@ -184,7 +185,7 @@ def test_a_wheel_with_invalid_build_info_fails(installed, build_info): PackageSnapshot.read("timenet") -@pytest.mark.real_git +@pytest.mark.real_provenance def test_a_local_build_fails_if_a_commit_is_unknown(installed, tmp_path): installed() with pytest.raises(TimeNetBuildError, match="cannot find the git commit of timenet"): @@ -205,3 +206,9 @@ def test_a_manifest_without_timenet_connectors_reads_as_not_installed(): def test_a_manifest_without_the_commit_of_timenet_is_refused(): with pytest.raises(TimeNetBuildError, match="the manifest records no git commit for timenet"): TimeNetSnapshot.from_source_map({"timenet-connectors": _CLEAN}) + + +def test_a_field_must_hold_the_commit_of_its_own_distribution(): + connectors = PackageSnapshot(distribution="timenet-connectors", source=_CLEAN) + with pytest.raises(ValidationError, match="the field for timenet holds the commit of timenet-connectors"): + TimeNetSnapshot(timenet=connectors) diff --git a/pyproject.toml b/pyproject.toml index 58723e1fc..89287ffd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,7 +148,7 @@ testpaths = ["packages", "benchmarks/end_to_end/tests"] markers = [ "slow: full-disk round-trip and property tests (deselect with '-m \"not slow\"')", "value_dtypes: scalar value-dtype round-trip tests (run with '-m value_dtypes')", - "real_git: run real git for provenance instead of the clean commit that packages/conftest.py reports", + "real_provenance: read the real commit of every install type instead of the stub in packages/conftest.py", ] [tool.ty.src] From cbc19fa22afafc421f62e8769757d955776ef68a Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 16:06:31 +0100 Subject: [PATCH 11/19] refactor(provenance): name packages, not distributions, in snapshots The snapshot and manifest layer now says package: PackageName, PackageSnapshot.package_name and TimeNetSnapshot.package_names(). "Distribution" also reads as a data distribution in TimeNet, and the classes are already called PackageSnapshot. InstalledDistribution and InstallSource keep the packaging term, because they wrap importlib.metadata.Distribution. Part of TNET-165 --- .../src/timenet_connectors/builder/env.py | 2 +- .../tests/test_build_env.py | 4 +- .../timenet/src/timenet/manifest/__init__.py | 4 +- .../timenet/src/timenet/manifest/manifest.py | 16 ++--- packages/timenet/src/timenet/provenance.py | 60 +++++++++---------- .../src/timenet/schemas/manifest.schema.json | 2 +- packages/timenet/tests/test_provenance.py | 2 +- 7 files changed, 45 insertions(+), 45 deletions(-) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index 3a125fcfa..02e8ebf59 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -202,7 +202,7 @@ def _base_args() -> tuple[str, ...]: # Pin TimeNet to the parent's own versions, so the code that writes the TimeF bytes is the code # that reads them back. timenet-connectors depends on timenet[build], so the card-YAML # dependencies arrive transitively and the extra never needs naming here. - for name in TimeNetSnapshot.distributions(): + for name in TimeNetSnapshot.package_names(): try: installed = InstalledDistribution.from_name(name) except PackageNotFoundError as exc: diff --git a/packages/timenet-connectors/tests/test_build_env.py b/packages/timenet-connectors/tests/test_build_env.py index 4ba053da5..239dc5942 100644 --- a/packages/timenet-connectors/tests/test_build_env.py +++ b/packages/timenet-connectors/tests/test_build_env.py @@ -34,7 +34,7 @@ def read_text(self, name): def _base_args_for(monkeypatch, dist): - monkeypatch.setattr(TimeNetSnapshot, "distributions", classmethod(lambda cls: ("timenet",))) + monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet",))) monkeypatch.setattr(env_module.Distribution, "from_name", lambda name: dist) return env_module._base_args() @@ -150,7 +150,7 @@ def test_run_build_captures_stderr_and_the_exit_code(): def test_env_spec_reports_a_base_distribution_that_is_not_installed(monkeypatch): - monkeypatch.setattr(TimeNetSnapshot, "distributions", classmethod(lambda cls: ("timenet", "timenet-not-installed"))) + monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet", "timenet-not-installed"))) with pytest.raises(TimeNetBuildError, match="timenet-not-installed"): env_spec("timenet/hello-world") diff --git a/packages/timenet/src/timenet/manifest/__init__.py b/packages/timenet/src/timenet/manifest/__init__.py index e7b6d5f3f..c4dc1b58c 100644 --- a/packages/timenet/src/timenet/manifest/__init__.py +++ b/packages/timenet/src/timenet/manifest/__init__.py @@ -4,10 +4,10 @@ from timenet.manifest.files import FilePart, ManifestFiles from timenet.manifest.manifest import ( BuildEnvironment, - DistributionName, GitSource, GitSourceMap, Manifest, + PackageName, parse_manifest, read_manifest, ) @@ -16,13 +16,13 @@ __all__ = [ "BuildEnvironment", - "DistributionName", "FilePart", "GitSource", "GitSourceMap", "Manifest", "ManifestCounts", "ManifestFiles", + "PackageName", "PayloadSource", "parse_manifest", "read_manifest", diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index 57ad90b90..8bbd58140 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -26,7 +26,7 @@ class GitSource(TimeFModel): - """The git commit that one TimeNet distribution was built from.""" + """The git commit that one TimeNet package was built from.""" sha: GitSHA """The commit of the checkout, or of the checkout that built the wheel.""" @@ -34,11 +34,11 @@ class GitSource(TimeFModel): """Whether the package directory had uncommitted changes, untracked files included.""" -DistributionName = StrictStr -"""The name of an installed distribution, for example ``timenet-connectors``.""" +PackageName = StrictStr +"""The name pip installs, for example ``timenet-connectors`` (not the import name ``timenet_connectors``).""" -GitSourceMap = dict[DistributionName, GitSource] -"""The commit of each installed TimeNet distribution, keyed by distribution name.""" +GitSourceMap = dict[PackageName, GitSource] +"""The commit of each installed TimeNet package, keyed by package name.""" class BuildEnvironment(TimeFModel): @@ -46,10 +46,10 @@ class BuildEnvironment(TimeFModel): python: StrictStr """The interpreter version, for example ``3.13.1``.""" - packages: dict[DistributionName, StrictStr] - """Each installed distribution and its version, sorted by name.""" + packages: dict[PackageName, StrictStr] + """Each installed package and its version, sorted by name.""" git: GitSourceMap - """The commit of each installed TimeNet distribution. A distribution that is not installed is absent.""" + """The commit of each installed TimeNet package. A package that is not installed is absent.""" @classmethod def capture(cls, git: GitSourceMap) -> Self: diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 758741f6f..1e3a7605a 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, ValidationError, model_validator from timenet.errors import TimeNetBuildError -from timenet.manifest import DistributionName, GitSource, GitSourceMap +from timenet.manifest import GitSource, GitSourceMap, PackageName _GIT_TIMEOUT_S = 30 @@ -29,7 +29,7 @@ class InstallSource(ABC): """How one distribution was installed. The install type decides where its commit is recorded.""" - distribution: DistributionName + distribution: PackageName @abstractmethod def read_git_source(self) -> GitSource: @@ -113,12 +113,12 @@ def __init__(self, distribution: Distribution) -> None: self._distribution = distribution @classmethod - def from_name(cls, name: DistributionName) -> Self: + def from_name(cls, name: PackageName) -> Self: """Find an installed distribution. Raises ``PackageNotFoundError`` if it is not installed.""" return cls(Distribution.from_name(name)) @property - def name(self) -> DistributionName: + def name(self) -> PackageName: """The distribution name, for example ``timenet-connectors``.""" return self._distribution.metadata["Name"] @@ -174,41 +174,41 @@ def to_source_map(self) -> GitSourceMap: class PackageSnapshot(Snapshot): - """The commit of one installed distribution.""" + """The commit of one installed package.""" - distribution: DistributionName + package_name: PackageName + """The name pip installs, for example ``timenet-connectors``.""" source: GitSource @classmethod - def read(cls, distribution: DistributionName) -> Self: - """Read the commit of an installed distribution. Raises ``TimeNetBuildError`` if it is not installed.""" - snapshot = cls.read_if_installed(distribution) + def read(cls, package_name: PackageName) -> Self: + """Read the commit of an installed package. Raises ``TimeNetBuildError`` if it is not installed.""" + snapshot = cls.read_if_installed(package_name) if snapshot is None: raise TimeNetBuildError( - f"cannot find the git commit of {distribution}: it is not installed in {sys.executable}" + f"cannot find the git commit of {package_name}: it is not installed in {sys.executable}" ) return snapshot @classmethod - def read_if_installed(cls, distribution: DistributionName) -> Self | None: - """Read the commit of a distribution, or return ``None`` if it is not installed.""" + def read_if_installed(cls, package_name: PackageName) -> Self | None: + """Read the commit of a package, or return ``None`` if it is not installed.""" try: - installed = InstalledDistribution.from_name(distribution) + installed = InstalledDistribution.from_name(package_name) except PackageNotFoundError: return None - return cls(distribution=distribution, source=installed.read_git_source()) + return cls(package_name=package_name, source=installed.read_git_source()) def to_source_map(self) -> GitSourceMap: """Return this package's commit in the form that a manifest records.""" - return {self.distribution: self.source} + return {self.package_name: self.source} class TimeNetSnapshot(Snapshot): - """The commit of each installed TimeNet distribution, read at one point in time. + """The commit of each installed TimeNet package, read at one point in time. - Each build records it in its manifest. Its fields are the one list of the distributions that - count as TimeNet code. Each field carries the name of its distribution, as its alias or as its - own name. + Each build records it in its manifest. Its fields are the one list of the packages that count + as TimeNet code. Each field carries the name of its package, as its alias or as its own name. """ timenet: PackageSnapshot @@ -216,22 +216,22 @@ class TimeNetSnapshot(Snapshot): """``None`` if timenet-connectors is not installed.""" @model_validator(mode="after") - def _each_field_holds_its_distribution(self) -> Self: + def _each_field_holds_its_package(self) -> Self: for name, field in type(self).model_fields.items(): package = getattr(self, name) - distribution = field.alias or name - if package is not None and package.distribution != distribution: - raise ValueError(f"the field for {distribution} holds the commit of {package.distribution}") + package_name = field.alias or name + if package is not None and package.package_name != package_name: + raise ValueError(f"the field for {package_name} holds the commit of {package.package_name}") return self @classmethod - def distributions(cls) -> tuple[DistributionName, ...]: - """Return the name of each TimeNet distribution, in field order.""" + def package_names(cls) -> tuple[PackageName, ...]: + """Return the name of each TimeNet package, in field order.""" return tuple(field.alias or name for name, field in cls.model_fields.items()) @classmethod def read(cls) -> Self: - """Read the commit of each installed TimeNet distribution. Raises ``TimeNetBuildError`` if one is unknown.""" + """Read the commit of each installed TimeNet package. Raises ``TimeNetBuildError`` if one is unknown.""" return cls( timenet=PackageSnapshot.read("timenet"), timenet_connectors=PackageSnapshot.read_if_installed("timenet-connectors"), @@ -240,23 +240,23 @@ def read(cls) -> Self: @classmethod def from_source_map(cls, sources: GitSourceMap) -> Self: """Return the typed view of the commits that a manifest records.""" - packages = {name: PackageSnapshot(distribution=name, source=source) for name, source in sources.items()} + packages = {name: PackageSnapshot(package_name=name, source=source) for name, source in sources.items()} try: - # The field aliases map each distribution name to its field. + # The field aliases map each package name to its field. return cls.model_validate(packages) except ValidationError as exc: missing = ", ".join(str(error["loc"][0]) for error in exc.errors()) raise TimeNetBuildError(f"the manifest records no git commit for {missing}") from exc def to_source_map(self) -> GitSourceMap: - """Return the commit of each installed TimeNet distribution.""" + """Return the commit of each installed TimeNet package.""" sources: GitSourceMap = {} for package in self._installed(): sources.update(package.to_source_map()) return sources def _installed(self) -> list[PackageSnapshot]: - """Return the snapshot of each installed TimeNet distribution.""" + """Return the snapshot of each installed TimeNet package.""" return [package for package in (self.timenet, self.timenet_connectors) if package is not None] diff --git a/packages/timenet/src/timenet/schemas/manifest.schema.json b/packages/timenet/src/timenet/schemas/manifest.schema.json index ab4f926aa..c707cafb0 100644 --- a/packages/timenet/src/timenet/schemas/manifest.schema.json +++ b/packages/timenet/src/timenet/schemas/manifest.schema.json @@ -377,7 +377,7 @@ }, "GitSource": { "additionalProperties": false, - "description": "The git commit that one TimeNet distribution was built from.", + "description": "The git commit that one TimeNet package was built from.", "properties": { "sha": { "pattern": "^[0-9a-f]{40}$", diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 43b100246..88f97b9af 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -209,6 +209,6 @@ def test_a_manifest_without_the_commit_of_timenet_is_refused(): def test_a_field_must_hold_the_commit_of_its_own_distribution(): - connectors = PackageSnapshot(distribution="timenet-connectors", source=_CLEAN) + connectors = PackageSnapshot(package_name="timenet-connectors", source=_CLEAN) with pytest.raises(ValidationError, match="the field for timenet holds the commit of timenet-connectors"): TimeNetSnapshot(timenet=connectors) From 3e554573f00bffcc9656ee41627976a0f45f3e46 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 16:38:54 +0100 Subject: [PATCH 12/19] docs(provenance): shorten the module docstring Part of TNET-165 --- packages/timenet/src/timenet/provenance.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 1e3a7605a..fbf4a6fd8 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,9 +1,4 @@ -"""Read the git commit of the installed TimeNet code, for the manifest's ``build_env.git`` block. - -Two layers do this. :class:`InstalledDistribution` and its :class:`InstallSource` read the commit -of any one installed package. The snapshot types (:class:`PackageSnapshot` and -:class:`TimeNetSnapshot`) are typed views of the commits that a manifest records. -""" +"""Read the git commit of the installed TimeNet code, for the manifest's ``build_env.git`` block.""" from abc import ABC, abstractmethod from dataclasses import dataclass From bc9db9f155db98ae6a084f87d1636cb22d595343 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 16:42:12 +0100 Subject: [PATCH 13/19] refactor(provenance): name the InstallSource field package_name The field holds a package name, not an importlib Distribution, so it now reads like every other name in the module. Only InstalledDistribution keeps the packaging term, because it wraps importlib.metadata.Distribution. Part of TNET-165 --- packages/timenet/src/timenet/provenance.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index fbf4a6fd8..8e3eff5b3 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -22,9 +22,9 @@ @dataclass(frozen=True) class InstallSource(ABC): - """How one distribution was installed. The install type decides where its commit is recorded.""" + """How one package was installed. The install type decides where its commit is recorded.""" - distribution: PackageName + package_name: PackageName @abstractmethod def read_git_source(self) -> GitSource: @@ -32,7 +32,7 @@ def read_git_source(self) -> GitSource: def _unknown_commit(self, reason: str) -> TimeNetBuildError: return TimeNetBuildError( - f"cannot find the git commit of {self.distribution}, which each build records in its manifest: {reason}" + f"cannot find the git commit of {self.package_name}, which each build records in its manifest: {reason}" ) @@ -48,7 +48,7 @@ def read_git_source(self) -> GitSource: if git is None: raise self._unknown_commit( f"it is installed editable from {self.project_directory}, and git is not on PATH. Install git, " - f"or install {self.distribution} from a wheel that scripts/build_dists.py built" + f"or install {self.package_name} from a wheel that scripts/build_dists.py built" ) try: sha = _run_git(git, self.project_directory, "rev-parse", "HEAD") @@ -74,7 +74,7 @@ def read_git_source(self) -> GitSource: try: return GitSource(sha=self.commit_id, contains_local_changes=False) except ValidationError as exc: - raise TimeNetBuildError(f"{self.distribution} records an invalid commit in direct_url.json: {exc}") from exc + raise TimeNetBuildError(f"{self.package_name} records an invalid commit in direct_url.json: {exc}") from exc @dataclass(frozen=True) @@ -91,13 +91,13 @@ def read_git_source(self) -> GitSource: if not self.build_info.is_file(): raise self._unknown_commit( f"the installed package carries no {self.BUILD_INFO_FILE}. Build it with `make build` or " - f"scripts/build_dists.py, or upgrade {self.distribution} to a release that records its commit" + f"scripts/build_dists.py, or upgrade {self.package_name} to a release that records its commit" ) try: return GitSource.model_validate_json(self.build_info.read_text(encoding="utf-8")) except ValidationError as exc: raise TimeNetBuildError( - f"{self.distribution} records an invalid commit in {self.build_info}: {exc}" + f"{self.package_name} records an invalid commit in {self.build_info}: {exc}" ) from exc @@ -114,7 +114,7 @@ def from_name(cls, name: PackageName) -> Self: @property def name(self) -> PackageName: - """The distribution name, for example ``timenet-connectors``.""" + """The package name that pip installs, for example ``timenet-connectors``.""" return self._distribution.metadata["Name"] @property From 70cdc1a5fcf695acc6128215c55d80924bbf1da2 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 18:43:35 +0100 Subject: [PATCH 14/19] docs(provenance): say package, not distribution, in the manifest docs The docs for build_env.git and one test name still used "distribution" after the rename to "package". They now say "package". The TimeNetSnapshot docstring loses its second paragraph, because the one-line summary is enough. --- docs/manifest.md | 8 ++++---- packages/timenet/src/timenet/provenance.py | 6 +----- packages/timenet/tests/test_provenance.py | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/manifest.md b/docs/manifest.md index 8825713ab..34430a84a 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -58,8 +58,8 @@ every installed package with its version, and the git commit of the TimeNet code `BuildEnvironment.capture` collects this data. Like `value_encoding`, `build_env` is provenance only. No code reads it to interpret the data. -`build_env.git` records the git commit of each installed TimeNet distribution, keyed by -distribution name. A distribution that is not installed has no entry: +`build_env.git` records the git commit of each installed TimeNet package, keyed by package +name. A package that is not installed has no entry: ```json "git": { @@ -68,7 +68,7 @@ distribution name. A distribution that is not installed has no entry: } ``` -Each distribution records the commit of the code that runs: +Each package records the commit of the code that runs: - An editable install, such as a workspace checkout, asks git. `contains_local_changes` is `true` if the package directory has uncommitted changes, untracked files included. A change @@ -80,7 +80,7 @@ Each distribution records the commit of the code that runs: builds the wheel, and records the state of the checkout at that time. `make build` and the publish workflow run the script. -If the commit of an installed distribution is unknown, the build fails and names the reason. For +If the commit of an installed package is unknown, the build fails and names the reason. For example, git is not on `PATH`, git fails in the checkout, or a plain `uv build` produced the wheel, so the package has no `_build_info.json`. diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 8e3eff5b3..6c502f2be 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -200,11 +200,7 @@ def to_source_map(self) -> GitSourceMap: class TimeNetSnapshot(Snapshot): - """The commit of each installed TimeNet package, read at one point in time. - - Each build records it in its manifest. Its fields are the one list of the packages that count - as TimeNet code. Each field carries the name of its package, as its alias or as its own name. - """ + """The commit of each installed TimeNet package, read at one point in time.""" timenet: PackageSnapshot timenet_connectors: PackageSnapshot | None = Field(default=None, alias="timenet-connectors") diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 88f97b9af..9fcd2eff3 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -208,7 +208,7 @@ def test_a_manifest_without_the_commit_of_timenet_is_refused(): TimeNetSnapshot.from_source_map({"timenet-connectors": _CLEAN}) -def test_a_field_must_hold_the_commit_of_its_own_distribution(): +def test_a_field_must_hold_the_commit_of_its_own_package(): connectors = PackageSnapshot(package_name="timenet-connectors", source=_CLEAN) with pytest.raises(ValidationError, match="the field for timenet holds the commit of timenet-connectors"): TimeNetSnapshot(timenet=connectors) From f09f38e831b61381572d45cfe0398ecf35b715ff Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Mon, 9 Nov 2026 20:16:36 +0100 Subject: [PATCH 15/19] fix(provenance): read the commit after the status, and catch a lost build record An editable install read HEAD before git status. A commit between the two reads then looked like the old commit without local changes, so a build whose code changed could still pass the upload check. The lookup now reads the status first, so such a commit shows as a new commit. scripts/build_dists.py writes and removes the records of both packages. Two builds at the same time in one checkout could remove each other's records, and git status cannot see that, because git ignores the files. The script now makes sure that each record still holds what it wrote, and fails otherwise. --- packages/timenet/src/timenet/provenance.py | 4 +++- scripts/build_dists.py | 25 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 6c502f2be..22a1c241f 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -51,10 +51,12 @@ def read_git_source(self) -> GitSource: f"or install {self.package_name} from a wheel that scripts/build_dists.py built" ) try: - sha = _run_git(git, self.project_directory, "rev-parse", "HEAD") # The pathspec "." scopes the status to the package directory, untracked files included. A # scratch file elsewhere in the repository does not count as a local change. status = _run_git(git, self.project_directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + # Read the commit after the status. A commit between the two reads then shows as a new + # commit, not as the old commit without local changes. + sha = _run_git(git, self.project_directory, "rev-parse", "HEAD") except subprocess.CalledProcessError as exc: detail = (exc.stderr or "").strip() or str(exc) raise self._unknown_commit(f"git failed in {self.project_directory}: {detail}") from exc diff --git a/scripts/build_dists.py b/scripts/build_dists.py index 8df4ecb46..0712ba9a3 100644 --- a/scripts/build_dists.py +++ b/scripts/build_dists.py @@ -44,13 +44,14 @@ def git(directory: Path, *args: str) -> str: def state(directory: Path) -> tuple[str, str]: """Return the commit of a package directory and its ``git status`` output.""" + # Read the commit after the status, as timenet.provenance.EditableCheckout does. status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") return git(directory, "rev-parse", "HEAD"), status def main(build_args: list[str]) -> int: """Write the commit records, build, and remove the records again, also on failure.""" - written: list[Path] = [] + written: dict[Path, str] = {} before = {directory: state(directory) for directory in PACKAGES} try: for directory, module in PACKAGES.items(): @@ -58,9 +59,10 @@ def main(build_args: list[str]) -> int: # Keep the keys in step with timenet.manifest.GitSource. info = {"sha": sha, "contains_local_changes": bool(status)} target = directory / "src" / module / BUILD_INFO_FILE + text = json.dumps(info) + "\n" # Register the file before the write, so a write that fails halfway is removed too. - written.append(target) - target.write_text(json.dumps(info) + "\n", encoding="utf-8") + written[target] = text + target.write_text(text, encoding="utf-8") print(f"{target.relative_to(ROOT)}: {info}", file=sys.stderr) # uv run sets UV to its own executable. uv = os.environ.get("UV") or executable("uv") @@ -74,11 +76,28 @@ def main(build_args: list[str]) -> int: file=sys.stderr, ) return 1 + # git ignores the records, so the status above cannot see one that another build removed. + lost = [str(path.relative_to(ROOT)) for path, text in written.items() if not _holds(path, text)] + if lost: + print( + f"error: {', '.join(lost)} changed during the build, so the built artifacts can lack their " + "commit records. Do not run two builds in one checkout at the same time. Build again.", + file=sys.stderr, + ) + return 1 return returncode finally: for path in written: path.unlink(missing_ok=True) +def _holds(path: Path, text: str) -> bool: + """Return whether ``path`` exists and holds ``text``.""" + try: + return path.read_text(encoding="utf-8") == text + except FileNotFoundError: + return False + + if __name__ == "__main__": sys.exit(main(sys.argv[1:])) From 807c88984495e82559955a53fc366aec4ef39a09 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 10 Nov 2026 16:38:48 +0100 Subject: [PATCH 16/19] refactor(provenance): name the install types, and move git into _GitCheckout Review follow-ups for the provenance types: - EditableCheckout is now EditableInstall, and BuiltPackage is now WheelInstall, to match GitInstall. Their docstrings say why each reads its commit from a different place. - A private _GitCheckout class runs git. It owns the PATH lookup, the timeout and the error handling, and replaces _run_git. - The docstrings of InstallSource, PackageSnapshot and InstalledDistribution.from_name give examples and name the cases. --- packages/timenet/src/timenet/provenance.py | 127 ++++++++++++++------- scripts/build_dists.py | 4 +- 2 files changed, 90 insertions(+), 41 deletions(-) diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 22a1c241f..a6a028cdf 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -22,7 +22,10 @@ @dataclass(frozen=True) class InstallSource(ABC): - """How one package was installed. The install type decides where its commit is recorded.""" + """Where one package was installed from: a git checkout, a git URL, or a wheel. + + Each kind records its git provenance in a different place. + """ package_name: PackageName @@ -37,32 +40,29 @@ def _unknown_commit(self, reason: str) -> TimeNetBuildError: @dataclass(frozen=True) -class EditableCheckout(InstallSource): - """An editable install of a git checkout. Git reports the commit and the local changes.""" +class EditableInstall(InstallSource): + """An editable install (``pip install -e``, or a uv workspace member). + + Python imports the code from the checkout itself, so git reports its commit and local changes. + """ project_directory: Path def read_git_source(self) -> GitSource: """Ask git for the commit of the checkout and whether the package directory has changes.""" - git = shutil.which("git") - if git is None: - raise self._unknown_commit( - f"it is installed editable from {self.project_directory}, and git is not on PATH. Install git, " - f"or install {self.package_name} from a wheel that scripts/build_dists.py built" - ) + checkout = _GitCheckout(self.project_directory) try: - # The pathspec "." scopes the status to the package directory, untracked files included. A - # scratch file elsewhere in the repository does not count as a local change. - status = _run_git(git, self.project_directory, "status", "--porcelain", "--untracked-files=all", "--", ".") # Read the commit after the status. A commit between the two reads then shows as a new # commit, not as the old commit without local changes. - sha = _run_git(git, self.project_directory, "rev-parse", "HEAD") - except subprocess.CalledProcessError as exc: - detail = (exc.stderr or "").strip() or str(exc) - raise self._unknown_commit(f"git failed in {self.project_directory}: {detail}") from exc - except (OSError, subprocess.TimeoutExpired) as exc: - raise self._unknown_commit(f"git failed in {self.project_directory}: {exc}") from exc - return GitSource(sha=sha, contains_local_changes=bool(status)) + changed = checkout.has_local_changes() + return GitSource(sha=checkout.head_commit(), contains_local_changes=changed) + except _GitNotFoundError as exc: + raise self._unknown_commit( + f"it is installed editable from {self.project_directory}, and {exc}. Install git, " + f"or install {self.package_name} from a wheel that scripts/build_dists.py built" + ) from exc + except _GitError as exc: + raise self._unknown_commit(str(exc)) from exc @dataclass(frozen=True) @@ -80,11 +80,15 @@ def read_git_source(self) -> GitSource: @dataclass(frozen=True) -class BuiltPackage(InstallSource): - """A wheel, or a path install that is not editable. The package carries its commit record.""" +class WheelInstall(InstallSource): + """An install from a wheel, including a path install that is not editable (pip builds a wheel first). + + The installed code is a copy, so a checkout can differ from it. The wheel carries the git + provenance record baked into it at build time (``_build_info.json``). + """ BUILD_INFO_FILE: ClassVar[str] = "_build_info.json" - """The commit record in each built package. ``scripts/build_dists.py`` writes it.""" + """The commit record in each wheel. ``scripts/build_dists.py`` writes it.""" build_info: Path @@ -111,7 +115,16 @@ def __init__(self, distribution: Distribution) -> None: @classmethod def from_name(cls, name: PackageName) -> Self: - """Find an installed distribution. Raises ``PackageNotFoundError`` if it is not installed.""" + """Find an installed package by name, such as ``timenet``. + + Note: An extra like ``timenet[zarr]`` is not a name, and ``from_name`` would not find it. + + Returns: + The installed package. + + Raises: + PackageNotFoundError: If it is not installed. + """ # ruff: ignore[docstring-extraneous-exception] - raised by Distribution.from_name return cls(Distribution.from_name(name)) @property @@ -133,12 +146,12 @@ def install_source(self) -> InstallSource: return GitInstall(self.name, commit_id) project_directory = direct_url.local_directory() if direct_url.is_editable() and project_directory is not None: - return EditableCheckout(self.name, project_directory) + return EditableInstall(self.name, project_directory) # An install from an index has no direct_url.json. A path install that is not editable # copied the package, so the live checkout can differ from the code that runs. module = self.name.replace("-", "_") - build_info = self._distribution.locate_file(f"{module}/{BuiltPackage.BUILD_INFO_FILE}") - return BuiltPackage(self.name, Path(str(build_info))) + build_info = self._distribution.locate_file(f"{module}/{WheelInstall.BUILD_INFO_FILE}") + return WheelInstall(self.name, Path(str(build_info))) def read_git_source(self) -> GitSource: """Return the commit of the code that this distribution runs. Raises ``TimeNetBuildError`` if it is unknown.""" @@ -171,7 +184,10 @@ def to_source_map(self) -> GitSourceMap: class PackageSnapshot(Snapshot): - """The commit of one installed package.""" + """The commit of one package that pip installed, such as ``timenet`` or ``timenet-connectors``. + + Note: Not a subpackage such as ``timenet.registry``. + """ package_name: PackageName """The name pip installs, for example ``timenet-connectors``.""" @@ -300,15 +316,48 @@ def _is_file_url(self) -> bool: return urlparse(self.url).scheme == "file" -def _run_git(git: str, directory: Path, *args: str) -> str: - """Run one git command in ``directory`` and return its stripped stdout.""" - result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] - # --no-optional-locks keeps `git status` from refreshing the index, so a build never takes - # the index lock while the user runs git. - [git, "--no-optional-locks", "-C", str(directory), *args], - capture_output=True, - text=True, - check=True, - timeout=_GIT_TIMEOUT_S, - ) - return result.stdout.strip() +class _GitError(Exception): + """Git could not read a checkout. The message gives the reason.""" + + +class _GitNotFoundError(_GitError): + """No git executable is on PATH.""" + + +@dataclass(frozen=True) +class _GitCheckout: + """A directory inside a git checkout, read with the git on PATH.""" + + directory: Path + + def head_commit(self) -> str: + """Return the commit that the checkout is at.""" + return self._run("rev-parse", "HEAD") + + def has_local_changes(self) -> bool: + """Whether the directory has uncommitted changes, untracked files included.""" + # The pathspec "." scopes the status to the directory. A scratch file elsewhere in the + # repository does not count as a local change. + return bool(self._run("status", "--porcelain", "--untracked-files=all", "--", ".")) + + def _run(self, *args: str) -> str: + """Run one git command in the directory and return its stripped stdout.""" + git = shutil.which("git") + if git is None: + raise _GitNotFoundError("git is not on PATH") + try: + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + # --no-optional-locks keeps `git status` from refreshing the index, so a build never + # takes the index lock while the user runs git. + [git, "--no-optional-locks", "-C", str(self.directory), *args], + capture_output=True, + text=True, + check=True, + timeout=_GIT_TIMEOUT_S, + ) + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or "").strip() or str(exc) + raise _GitError(f"git failed in {self.directory}: {detail}") from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise _GitError(f"git failed in {self.directory}: {exc}") from exc + return result.stdout.strip() diff --git a/scripts/build_dists.py b/scripts/build_dists.py index 0712ba9a3..810c16cee 100644 --- a/scripts/build_dists.py +++ b/scripts/build_dists.py @@ -20,7 +20,7 @@ ROOT / "packages" / "timenet": "timenet", ROOT / "packages" / "timenet-connectors": "timenet_connectors", } -# Keep in step with timenet.provenance.BuiltPackage.BUILD_INFO_FILE, which reads the file. +# Keep in step with timenet.provenance.WheelInstall.BUILD_INFO_FILE, which reads the file. BUILD_INFO_FILE = "_build_info.json" @@ -44,7 +44,7 @@ def git(directory: Path, *args: str) -> str: def state(directory: Path) -> tuple[str, str]: """Return the commit of a package directory and its ``git status`` output.""" - # Read the commit after the status, as timenet.provenance.EditableCheckout does. + # Read the commit after the status, as timenet.provenance.EditableInstall does. status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") return git(directory, "rev-parse", "HEAD"), status From 03a1677f60be54a630f74600ebde1b5d29dbacf0 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 10 Nov 2026 19:38:37 +0100 Subject: [PATCH 17/19] test: give the new tests of main a build environment Main added tests that build a Manifest without build_env, call LocalRegistry.store(), or patch env.BASE_DISTRIBUTIONS. This branch makes build_env required and removes the other two names. The tests now pass a full build_env, call put_dataset(), and patch TimeNetSnapshot.package_names. Part of TNET-165 --- packages/timenet-connectors/tests/test_build_plan.py | 3 ++- packages/timenet/tests/format/test_control_cache.py | 1 + packages/timenet/tests/registry/test_copy.py | 2 +- packages/timenet/tests/test_provenance.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index 453662bd8..f1939b1c3 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -28,6 +28,7 @@ TimeNetMissingParentError, TimeNetVersionConflictError, ) +from timenet.provenance import TimeNetSnapshot from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, WritableRegistry, open_local_version from timenet.registry._paths import release_dir @@ -893,7 +894,7 @@ def copy_the_saved_release(release, store_root, *, force=False, **_): def test_an_environment_that_cannot_be_made_names_the_build_in_this_interpreter(tmp_path, monkeypatch, entry, hint): cards = _install_cards(monkeypatch, parent={}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet-not-installed",)) + monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet-not-installed",))) local_registry = tmp_path / "store" if entry == "cli": diff --git a/packages/timenet/tests/format/test_control_cache.py b/packages/timenet/tests/format/test_control_cache.py index acbc0242a..3f05cd908 100644 --- a/packages/timenet/tests/format/test_control_cache.py +++ b/packages/timenet/tests/format/test_control_cache.py @@ -46,6 +46,7 @@ def test_a_remote_control_database_lands_in_a_control_folder_with_the_marker(tmp control=(FilePart(path=path.name, checksum=file_checksum(path), size=path.stat().st_size),), ), timef_format_version=1, + build_env=BuildEnvironment(python="3.13.1", packages={}, git={}), ) # Any file system but the local one counts as remote, so the database is copied into the cache. remote = pafs.SubTreeFileSystem(str(path.parent), pafs.LocalFileSystem()) diff --git a/packages/timenet/tests/registry/test_copy.py b/packages/timenet/tests/registry/test_copy.py index db04faaef..4360495a2 100644 --- a/packages/timenet/tests/registry/test_copy.py +++ b/packages/timenet/tests/registry/test_copy.py @@ -223,7 +223,7 @@ def test_copy_release_refuses_a_release_with_the_same_files_and_another_card(sou def test_copy_release_ignores_the_build_environment(source, tmp_path): dest = LocalRegistry(tmp_path / "dest") copy_release(child_ref, source, dest) - _rewrite_child_manifest(dest, build_env={"python": "0.0.0"}) + _rewrite_child_manifest(dest, build_env={"python": "0.0.0", "packages": {}, "git": {}}) events: list[ProgressEvent] = [] copy_release(child_ref, source, LocalRegistry(dest.root), progress=events.append) assert events == [] diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 9fcd2eff3..72f5d9553 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -189,7 +189,7 @@ def test_a_wheel_with_invalid_build_info_fails(installed, build_info): def test_a_local_build_fails_if_a_commit_is_unknown(installed, tmp_path): installed() with pytest.raises(TimeNetBuildError, match="cannot find the git commit of timenet"): - LocalRegistry(tmp_path / "local").store(make_dataset()) + LocalRegistry(tmp_path / "local").put_dataset(make_dataset()) def test_a_snapshot_reads_back_from_the_commits_that_a_manifest_records(): From db88bfcdcd27048876f165d5f8abaaa17e6a3fa2 Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 10 Nov 2026 19:43:07 +0100 Subject: [PATCH 18/19] fix(writer): read the commit before the data is written The writer read the TimeNet commit only when it wrote the manifest, after every data file. An unknown commit then failed after the whole write. A commit or checkout during the write also changed the record. The writer now reads the commit when it starts, and the manifest records that commit. Part of TNET-165 --- packages/timenet/src/timenet/writer/writer.py | 8 ++++++-- packages/timenet/tests/test_provenance.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/timenet/src/timenet/writer/writer.py b/packages/timenet/src/timenet/writer/writer.py index 95540b11e..4056eb061 100644 --- a/packages/timenet/src/timenet/writer/writer.py +++ b/packages/timenet/src/timenet/writer/writer.py @@ -165,9 +165,13 @@ def __enter__(self) -> "TimeFWriter": Raises: FileExistsError: If a committed ``manifest.json`` already exists at the version directory and ``replace`` is not set. - """ + TimeNetBuildError: If the commit of an installed TimeNet package is unknown. + """ # ruff: ignore[docstring-extraneous-exception] - raised by TimeNetSnapshot.read if not self._replace and (self._final_dir / MANIFEST_FILE).exists(): raise FileExistsError(f"a committed version already exists at {self._final_dir}") + # Read the commit before any data is written. An unknown commit then fails before the work, + # and a commit or checkout during the write does not change the record. + self._snapshot = TimeNetSnapshot.read() self._sweep_stale_staging() self._staging_dir.mkdir(parents=True, exist_ok=True) return self @@ -475,7 +479,7 @@ def _write_manifest(self) -> Manifest: payload_sources=self._storage_sources.sources, parent_cards=import_state.parent_cards, lineage=tuple(import_state.lineage.values()), - build_env=BuildEnvironment.capture(git=TimeNetSnapshot.read().to_source_map()), + build_env=BuildEnvironment.capture(git=self._snapshot.to_source_map()), timef_format_version=1, ) (self._staging_dir / MANIFEST_FILE).write_text(manifest.to_json()) diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 72f5d9553..dfb244d05 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -13,6 +13,7 @@ from timenet.provenance import InstalledDistribution, PackageSnapshot, TimeNetSnapshot from timenet.registry import LocalRegistry from timenet.testing import make_dataset +from timenet.writer import TimeFWriter _CLEAN = GitSource(sha="a" * 40, contains_local_changes=False) @@ -190,6 +191,25 @@ def test_a_local_build_fails_if_a_commit_is_unknown(installed, tmp_path): installed() with pytest.raises(TimeNetBuildError, match="cannot find the git commit of timenet"): LocalRegistry(tmp_path / "local").put_dataset(make_dataset()) + # The writer reads the commit before it writes any data. + assert not (tmp_path / "local" / make_dataset().metadata.dataset_id).exists() + + +def test_the_manifest_records_the_commit_from_when_the_write_started(tmp_path, monkeypatch): + original_write = TimeFWriter.write + + def write_then_commit(writer: TimeFWriter) -> None: + original_write(writer) + later = GitSource(sha="b" * 40, contains_local_changes=False) + monkeypatch.setattr(InstalledDistribution, "read_git_source", lambda _self: later) + + monkeypatch.setattr(TimeFWriter, "write", write_then_commit) + dataset = make_dataset() + registry = LocalRegistry(tmp_path / "local") + registry.put_dataset(dataset) + + manifest = registry.get_manifest(dataset.metadata.dataset_id) + assert manifest.build_env.git["timenet"].sha == "0" * 40 def test_a_snapshot_reads_back_from_the_commits_that_a_manifest_records(): From b15a0d5aedc2a77efb4f0a3eddc3b653d7ee397f Mon Sep 17 00:00:00 2001 From: Adrian Blumer Date: Tue, 10 Nov 2026 20:18:54 +0100 Subject: [PATCH 19/19] fix(build): remove the commit records when the build is killed build_dists.py removed the _build_info.json files only in a finally. A kill or a closed terminal ended the process before the finally ran. The files stayed, and git ignores them, so a later plain uv build packed a record of an old commit. SIGTERM and SIGHUP now raise SystemExit, so the finally removes the files. Only SIGKILL still leaves them. Part of TNET-165 --- scripts/build_dists.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/build_dists.py b/scripts/build_dists.py index 810c16cee..3272465d6 100644 --- a/scripts/build_dists.py +++ b/scripts/build_dists.py @@ -11,6 +11,7 @@ import os from pathlib import Path import shutil +import signal import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors import sys @@ -99,5 +100,15 @@ def _holds(path: Path, text: str) -> bool: return False +def _exit_on_signal(signum: int, _frame: object) -> None: + """Raise ``SystemExit``, so that the ``finally`` in ``main`` removes the commit records.""" + raise SystemExit(128 + signum) + + if __name__ == "__main__": + # Ctrl-C already raises KeyboardInterrupt. Without a handler, a kill or a closed terminal ends the + # process before the finally runs, and a later plain uv build would pack the stale records. + for name in ("SIGTERM", "SIGHUP"): + if hasattr(signal, name): + signal.signal(getattr(signal, name), _exit_on_signal) sys.exit(main(sys.argv[1:]))