diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 27c4f721c..38c57d279 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -52,15 +52,17 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v7 - name: Build + # build_dists.py records the commit inside each package, so a build can name its code. run: | if [ "${{ inputs.package }}" = "both" ]; then - uv build --all-packages + uv run scripts/build_dists.py --all-packages else - uv build --package "${{ inputs.package }}" + uv run scripts/build_dists.py --package "${{ inputs.package }}" fi - name: Smoke test built artifacts # Import each built wheel and sdist in an isolated environment to catch missing files or - # broken metadata before uploading. timenet-connectors depends on timenet, so hand it the + # broken metadata before uploading. Each artifact must also report this commit, clean, as + # the code it was built from. timenet-connectors depends on timenet, so hand it the # wheel we just built when it is present (a `both` run); otherwise timenet resolves from the # default index, which requires it to be published already. run: | @@ -68,20 +70,27 @@ jobs: shopt -s nullglob timenet_whl=(dist/timenet-*.whl) smoke() { - local module="$1"; shift + local module="$1" dist="$2"; shift 2 local args=() artifact for artifact in "$@"; do args+=(--with "$artifact"); done echo "::group::import $module" uv run --isolated --no-project "${args[@]}" \ python -c "import $module; print('smoke ok:', $module.__name__)" + uv run --isolated --no-project "${args[@]}" python -c " + from timenet.manifest import GitSource + from timenet.provenance import PackageSnapshot + commit = PackageSnapshot.read('$dist').source + expected = GitSource(sha='$GITHUB_SHA', contains_local_changes=False) + assert commit == expected, f'$dist records {commit}, expected {expected}' + print('commit ok:', commit)" echo "::endgroup::" } for artifact in dist/*; do case "$artifact" in dist/timenet-*.whl | dist/timenet-*.tar.gz) - smoke timenet "$artifact" ;; + smoke timenet timenet "$artifact" ;; dist/timenet_connectors-*.whl | dist/timenet_connectors-*.tar.gz) - smoke timenet_connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;; + smoke timenet_connectors timenet-connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;; esac done - name: Publish diff --git a/.gitignore b/.gitignore index 284e19e23..785af413f 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ docs/api/ # Local superpowers planning artifacts, not part of the repo docs/superpowers/ + +# commit record written into each import package before a wheel build (scripts/build_dists.py removes it after the build) +packages/*/src/*/_build_info.json diff --git a/AGENTS.md b/AGENTS.md index d9e1d389c..65da0ba29 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,9 @@ workspace definition and the shared ruff/ty/pytest config; per-package workspace extras (`timenet[cli,torch]`). It does not install connector dependencies. Each connector declares its own in a `requirements.txt`, and `make test-connectors` runs that connector's tests and type-check in an environment built from it. -- Build distributables with `make build` (`uv build --package ` per member). +- Build distributables with `make build`. It runs the build through a proxy script, which records + the git commit inside each package for provenance, before running `uv build --package ` + per member. - For one-off scripts, use inline `uv` metadata and run with `uv run `. Never `pip install`. - Keep `uv.lock` committed; the `uv-lock` pre-commit hook enforces freshness. diff --git a/Makefile b/Makefile index b007f42ff..4c688e34d 100644 --- a/Makefile +++ b/Makefile @@ -55,9 +55,10 @@ license-check: --fail-on="GPL;LGPL;AGPL;SSPL;EUPL;CDDL;OSL" \ --format=markdown +# Builds through scripts/build_dists.py, so each wheel records the commit it was built from. build: - uv build --package timenet - uv build --package timenet-connectors + uv run scripts/build_dists.py --package timenet + uv run scripts/build_dists.py --package timenet-connectors check: schemas-check uv run ruff format . diff --git a/docs/build.md b/docs/build.md index 4b33def97..e9a45f817 100644 --- a/docs/build.md +++ b/docs/build.md @@ -48,8 +48,8 @@ variables (see the [settings table](client.md#settings)). For example, `$TIMENET_LOCAL_REGISTRY` & `TIMENET_ISOLATION=off`. These apply to every CLI command, and the Python SDK also honors them. -For each build, the used Python and package and TimeNet versions are recorded to the build's -manifest. +The manifest records the Python and package versions used for the build, and the git commit of +`timenet` and `timenet-connectors`. See [`build_env`](manifest.md) for the fields. ## Pipeline diff --git a/docs/manifest.md b/docs/manifest.md index 10eaf3ebc..34430a84a 100644 --- a/docs/manifest.md +++ b/docs/manifest.md @@ -39,7 +39,7 @@ Manifest( counts=counts, # ManifestCounts (default: empty) values_backend="parquet", # "parquet" (default) or "zarr" value_encoding={}, # spec_type -> the encoding its shards carry - build_env={}, # {"python", "packages"} (default: empty) + build_env=build_env, # BuildEnvironment (required) timef_format_version=1, # required; validated against the supported set {1} ) ``` @@ -53,9 +53,36 @@ of each spec type. No code reads this field. Parquet records the applied encodin each file, so the reader does not need it. The field lets a builder see which encoding a build selected. The field is empty for a backend that has no such choice. -`build_env` records the environment that produced the version. It gives the interpreter version and -every installed package with its version. `timenet.provenance.build_env` collects this data. Like -`value_encoding`, `build_env` is provenance only. No code reads it to interpret the data. +`build_env` records the environment that produced the version. It gives the interpreter version, +every installed package with its version, and the git commit of the TimeNet code. +`BuildEnvironment.capture` collects this data. Like `value_encoding`, `build_env` is provenance +only. No code reads it to interpret the data. + +`build_env.git` records the git commit of each installed TimeNet package, keyed by package +name. A package that is not installed has no entry: + +```json +"git": { + "timenet": {"sha": "4818e37f...", "contains_local_changes": false}, + "timenet-connectors": {"sha": "4818e37f...", "contains_local_changes": false} +} +``` + +Each package records the commit of the code that runs: + +- An editable install, such as a workspace checkout, asks git. `contains_local_changes` is + `true` if the package directory has uncommitted changes, untracked files included. A change + elsewhere in the repository does not set it. +- A git install (`pip install git+https://...`) records the commit that pip installed, with + `contains_local_changes` set to `false`. +- Any other install, such as a wheel or `pip install ./packages/timenet`, reads the + `_build_info.json` file in the package. `scripts/build_dists.py` writes that file while it + builds the wheel, and records the state of the checkout at that time. `make build` and the + publish workflow run the script. + +If the commit of an installed package is unknown, the build fails and names the reason. For +example, git is not on `PATH`, git fails in the checkout, or a plain `uv build` produced the wheel, +so the package has no `_build_info.json`. The values locator is backend-neutral. One schema covers both scalar and multidimensional specs. A multidimensional spec records its shape in `value_shape` and `dimension_names`. It does not need a diff --git a/packages/conftest.py b/packages/conftest.py new file mode 100644 index 000000000..00b9af5cd --- /dev/null +++ b/packages/conftest.py @@ -0,0 +1,21 @@ +"""Shared pytest configuration for every workspace package's tests. + +Each manifest write reads the commit of the TimeNet distributions. For an editable install this +asks git, so the result would depend on the state of this checkout, and the calls cost seconds +across the suite. So every installed distribution reports a clean commit, whatever its install +type, unless the test carries the ``real_provenance`` marker. +""" + +import pytest + +from timenet.manifest import GitSource +from timenet.provenance import InstalledDistribution + + +_CLEAN = GitSource(sha="0" * 40, contains_local_changes=False) + + +@pytest.fixture(autouse=True) +def _stub_installed_sources(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + if request.node.get_closest_marker("real_provenance") is None: + monkeypatch.setattr(InstalledDistribution, "read_git_source", lambda _self: _CLEAN) diff --git a/packages/timenet-connectors/src/timenet_connectors/builder/env.py b/packages/timenet-connectors/src/timenet_connectors/builder/env.py index c325c035b..02e8ebf59 100644 --- a/packages/timenet-connectors/src/timenet_connectors/builder/env.py +++ b/packages/timenet-connectors/src/timenet_connectors/builder/env.py @@ -18,23 +18,16 @@ import subprocess # ruff: ignore[suspicious-subprocess-import] - the command is a fixed argument vector import sys import tempfile -from urllib.parse import urlparse -from urllib.request import url2pathname from uv import find_uv_bin from timenet.errors import TimeNetBuildError, TimeNetIsolationError from timenet.manifest import PayloadSource +from timenet.provenance import InstalledDistribution, TimeNetSnapshot from timenet.types.references import DatasetRef from timenet_connectors.discovery import requirements_for -BASE_DISTRIBUTIONS = ("timenet", "timenet-connectors") -"""Pinned to the parent's own versions, so the code that writes the TimeF bytes is the code that -reads them back. ``timenet-connectors`` depends on ``timenet[build]``, so the card-YAML -dependencies arrive transitively and the extra never needs naming here.""" - - @dataclass(frozen=True) class EnvSpec: """What one connector's environment needs.""" @@ -206,44 +199,19 @@ def _base_args() -> tuple[str, ...]: TimeNetIsolationError: If a base distribution is not installed in this interpreter. """ args: list[str] = [] - for name in BASE_DISTRIBUTIONS: + # Pin TimeNet to the parent's own versions, so the code that writes the TimeF bytes is the code + # that reads them back. timenet-connectors depends on timenet[build], so the card-YAML + # dependencies arrive transitively and the extra never needs naming here. + for name in TimeNetSnapshot.package_names(): try: - dist = Distribution.from_name(name) + installed = InstalledDistribution.from_name(name) except PackageNotFoundError as exc: raise TimeNetIsolationError( f"cannot make a build environment: {name} is not installed in {sys.executable}." ) from exc - source = _local_source_path(dist) + source = installed.path_to_ondisk_project_directory() if source is not None: args += ["--with-editable", str(source)] else: - args += ["--with", f"{name}=={dist.version}"] + args += ["--with", f"{name}=={installed.version}"] return tuple(args) - - -def _local_source_path(dist: Distribution) -> Path | None: - """Return the on-disk project directory an editable or local path install points at. - - ``direct_url.json`` records how a distribution was installed. An editable install carries - ``dir_info.editable``. A plain path install (``pip install ./packages/timenet``) records a - ``file://`` URL to the same source tree without that flag. Both let the child build against the - parent's on-disk source, so both return the directory. An index or wheel install has no - ``file://`` directory and returns ``None``, leaving the caller to pin the exact version. - - Args: - dist: The installed distribution. - - Returns: - The source directory for an editable or local path install, else ``None``. - """ - raw = dist.read_text("direct_url.json") - if raw is None: - return None - info = json.loads(raw) - parsed = urlparse(info.get("url", "")) - if parsed.scheme != "file": - return None - path = Path(url2pathname(parsed.path)) - if info.get("dir_info", {}).get("editable") or path.is_dir(): - return path - return None diff --git a/packages/timenet-connectors/tests/test_build_env.py b/packages/timenet-connectors/tests/test_build_env.py index f42bb2038..239dc5942 100644 --- a/packages/timenet-connectors/tests/test_build_env.py +++ b/packages/timenet-connectors/tests/test_build_env.py @@ -9,6 +9,7 @@ from timenet.errors import TimeNetBuildError from timenet.manifest import PayloadSource from timenet.manifest.files import FilePart +from timenet.provenance import TimeNetSnapshot from timenet.types.references import DatasetRef from timenet_connectors.builder import env as env_module from timenet_connectors.builder.env import EnvSpec, env_spec, run_isolated_build, uv_command @@ -33,7 +34,7 @@ def read_text(self, name): def _base_args_for(monkeypatch, dist): - monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet",)) + monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet",))) monkeypatch.setattr(env_module.Distribution, "from_name", lambda name: dist) return env_module._base_args() @@ -149,7 +150,7 @@ def test_run_build_captures_stderr_and_the_exit_code(): def test_env_spec_reports_a_base_distribution_that_is_not_installed(monkeypatch): - monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet", "timenet-not-installed")) + monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet", "timenet-not-installed"))) with pytest.raises(TimeNetBuildError, match="timenet-not-installed"): env_spec("timenet/hello-world") diff --git a/packages/timenet-connectors/tests/test_build_plan.py b/packages/timenet-connectors/tests/test_build_plan.py index 453662bd8..f1939b1c3 100644 --- a/packages/timenet-connectors/tests/test_build_plan.py +++ b/packages/timenet-connectors/tests/test_build_plan.py @@ -28,6 +28,7 @@ TimeNetMissingParentError, TimeNetVersionConflictError, ) +from timenet.provenance import TimeNetSnapshot from timenet.reader import TimeFReader from timenet.registry import LocalRegistry, WritableRegistry, open_local_version from timenet.registry._paths import release_dir @@ -893,7 +894,7 @@ def copy_the_saved_release(release, store_root, *, force=False, **_): def test_an_environment_that_cannot_be_made_names_the_build_in_this_interpreter(tmp_path, monkeypatch, entry, hint): cards = _install_cards(monkeypatch, parent={}) monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards)) - monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet-not-installed",)) + monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet-not-installed",))) local_registry = tmp_path / "store" if entry == "cli": diff --git a/packages/timenet-connectors/tests/test_end_to_end.py b/packages/timenet-connectors/tests/test_end_to_end.py index 7066215fa..32abc908f 100644 --- a/packages/timenet-connectors/tests/test_end_to_end.py +++ b/packages/timenet-connectors/tests/test_end_to_end.py @@ -6,7 +6,7 @@ from timenet.client import TimeNet from timenet.config import settings -from timenet.provenance import build_env +from timenet.manifest import BuildEnvironment from timenet.registry import LocalRegistry from timenet.sources import InstalledConnectors from timenet.testing import assert_datasets_equal @@ -113,5 +113,5 @@ def test_an_isolated_build_records_its_environment(tmp_path): # pytest runs this process and is a runtime dependency of neither package, so it separates a # real isolated build from one that quietly inherited the parent's site-packages. Assert both # directions: without the first the second would also pass on a machine that simply lacks it. - assert "pytest" in build_env()["packages"] + assert "pytest" in BuildEnvironment.capture(git={}).packages assert "pytest" not in packages diff --git a/packages/timenet/src/timenet/manifest/__init__.py b/packages/timenet/src/timenet/manifest/__init__.py index 480bdf788..c4dc1b58c 100644 --- a/packages/timenet/src/timenet/manifest/__init__.py +++ b/packages/timenet/src/timenet/manifest/__init__.py @@ -2,16 +2,27 @@ from timenet.manifest.counts import ManifestCounts from timenet.manifest.files import FilePart, ManifestFiles -from timenet.manifest.manifest import BuildEnvironment, Manifest, parse_manifest, read_manifest +from timenet.manifest.manifest import ( + BuildEnvironment, + GitSource, + GitSourceMap, + Manifest, + PackageName, + parse_manifest, + read_manifest, +) from timenet.manifest.payload_source import PayloadSource __all__ = [ "BuildEnvironment", "FilePart", + "GitSource", + "GitSourceMap", "Manifest", "ManifestCounts", "ManifestFiles", + "PackageName", "PayloadSource", "parse_manifest", "read_manifest", diff --git a/packages/timenet/src/timenet/manifest/manifest.py b/packages/timenet/src/timenet/manifest/manifest.py index c2b099509..8bbd58140 100644 --- a/packages/timenet/src/timenet/manifest/manifest.py +++ b/packages/timenet/src/timenet/manifest/manifest.py @@ -1,10 +1,11 @@ """The :class:`Manifest`: the compiled ``manifest.json`` file and its JSON codec.""" +from importlib.metadata import distributions from pathlib import Path +import platform from typing import Annotated, Any, Self -from pydantic import Field, StrictStr, ValidationError, model_validator -from typing_extensions import TypedDict +from pydantic import Field, StrictBool, StrictStr, StringConstraints, ValidationError, model_validator from timenet.errors import ReleaseDifference, TimeFFormatError, TimeNetManifestError from timenet.manifest.counts import ManifestCounts @@ -20,11 +21,44 @@ from timenet.values_backends import ValuesBackend -class BuildEnvironment(TypedDict, total=False): - """Build-provenance fields recorded in a manifest.""" +GitSHA = Annotated[str, StringConstraints(strict=True, pattern=r"^[0-9a-f]{40}$")] +"""A full SHA-1 git commit id: 40 lowercase hex digits.""" + + +class GitSource(TimeFModel): + """The git commit that one TimeNet package was built from.""" + + sha: GitSHA + """The commit of the checkout, or of the checkout that built the wheel.""" + contains_local_changes: StrictBool + """Whether the package directory had uncommitted changes, untracked files included.""" + + +PackageName = StrictStr +"""The name pip installs, for example ``timenet-connectors`` (not the import name ``timenet_connectors``).""" + +GitSourceMap = dict[PackageName, GitSource] +"""The commit of each installed TimeNet package, keyed by package name.""" + + +class BuildEnvironment(TimeFModel): + """The Python version, the installed packages, and the TimeNet commits that produced a version.""" python: StrictStr - packages: dict[StrictStr, StrictStr] + """The interpreter version, for example ``3.13.1``.""" + packages: dict[PackageName, StrictStr] + """Each installed package and its version, sorted by name.""" + git: GitSourceMap + """The commit of each installed TimeNet package. A package that is not installed is absent.""" + + @classmethod + def capture(cls, git: GitSourceMap) -> Self: + """Describe this interpreter, with ``git`` from ``TimeNetSnapshot.to_source_map()``.""" + # Under an isolated build the packages are exactly the base, the connector's requirements, + # and their transitives. Under --no-isolation they are the whole ambient environment, which + # is noisier but still the truth about what produced the dataset. + packages = {dist.metadata["Name"]: dist.version for dist in distributions() if dist.metadata["Name"]} + return cls(python=platform.python_version(), packages=dict(sorted(packages.items())), git=git) class Manifest(TimeFModel): @@ -52,8 +86,8 @@ class Manifest(TimeFModel): """Every release that stores a payload file this release reads, by source key, pinned to those files. The writer flattens them, so a source's own sources are not listed unless this release reads their files too.""" - build_env: BuildEnvironment = Field(default_factory=dict) - """The Python version and package set that produced this version.""" + build_env: BuildEnvironment + """The Python version, the package set, and the TimeNet commits that produced this version.""" timef_format_version: Annotated[int, Field(strict=True, ge=1, le=1)] """The TimeF manifest format version.""" diff --git a/packages/timenet/src/timenet/provenance.py b/packages/timenet/src/timenet/provenance.py index 3da28946d..a6a028cdf 100644 --- a/packages/timenet/src/timenet/provenance.py +++ b/packages/timenet/src/timenet/provenance.py @@ -1,24 +1,363 @@ -"""Describe the environment a build ran in, for the manifest's provenance block.""" +"""Read the git commit of the installed TimeNet code, for the manifest's ``build_env.git`` block.""" -from importlib.metadata import distributions -import platform +from abc import ABC, abstractmethod +from dataclasses import dataclass +from importlib.metadata import Distribution, PackageNotFoundError +from pathlib import Path +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors +import sys +from typing import ClassVar, Self +from urllib.parse import urlparse +from urllib.request import url2pathname -from timenet.manifest import BuildEnvironment +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, ValidationError, model_validator +from timenet.errors import TimeNetBuildError +from timenet.manifest import GitSource, GitSourceMap, PackageName -def build_env() -> BuildEnvironment: - """Describe this process's Python version and installed packages. - Under an isolated build the recorded set is exactly the base, the connector's requirements, and - their transitives. Under ``--no-isolation`` it is the whole ambient environment, which is noisier - but still the truth about what produced the dataset. +_GIT_TIMEOUT_S = 30 - Returns: - A dict with ``python`` (the interpreter version) and ``packages`` (name -> version, sorted). + +@dataclass(frozen=True) +class InstallSource(ABC): + """Where one package was installed from: a git checkout, a git URL, or a wheel. + + Each kind records its git provenance in a different place. + """ + + package_name: PackageName + + @abstractmethod + def read_git_source(self) -> GitSource: + """Return the commit of the code that this install runs. Raises ``TimeNetBuildError`` if it is unknown.""" + + def _unknown_commit(self, reason: str) -> TimeNetBuildError: + return TimeNetBuildError( + f"cannot find the git commit of {self.package_name}, which each build records in its manifest: {reason}" + ) + + +@dataclass(frozen=True) +class EditableInstall(InstallSource): + """An editable install (``pip install -e``, or a uv workspace member). + + Python imports the code from the checkout itself, so git reports its commit and local changes. + """ + + project_directory: Path + + def read_git_source(self) -> GitSource: + """Ask git for the commit of the checkout and whether the package directory has changes.""" + checkout = _GitCheckout(self.project_directory) + try: + # Read the commit after the status. A commit between the two reads then shows as a new + # commit, not as the old commit without local changes. + changed = checkout.has_local_changes() + return GitSource(sha=checkout.head_commit(), contains_local_changes=changed) + except _GitNotFoundError as exc: + raise self._unknown_commit( + f"it is installed editable from {self.project_directory}, and {exc}. Install git, " + f"or install {self.package_name} from a wheel that scripts/build_dists.py built" + ) from exc + except _GitError as exc: + raise self._unknown_commit(str(exc)) from exc + + +@dataclass(frozen=True) +class GitInstall(InstallSource): + """An install from a git URL (``pip install git+https://...``). Pip recorded the commit.""" + + commit_id: str + + def read_git_source(self) -> GitSource: + """Return the commit that pip installed, which has no local changes.""" + try: + return GitSource(sha=self.commit_id, contains_local_changes=False) + except ValidationError as exc: + raise TimeNetBuildError(f"{self.package_name} records an invalid commit in direct_url.json: {exc}") from exc + + +@dataclass(frozen=True) +class WheelInstall(InstallSource): + """An install from a wheel, including a path install that is not editable (pip builds a wheel first). + + The installed code is a copy, so a checkout can differ from it. The wheel carries the git + provenance record baked into it at build time (``_build_info.json``). + """ + + BUILD_INFO_FILE: ClassVar[str] = "_build_info.json" + """The commit record in each wheel. ``scripts/build_dists.py`` writes it.""" + + build_info: Path + + def read_git_source(self) -> GitSource: + """Read the commit that ``scripts/build_dists.py`` wrote into the package.""" + if not self.build_info.is_file(): + raise self._unknown_commit( + f"the installed package carries no {self.BUILD_INFO_FILE}. Build it with `make build` or " + f"scripts/build_dists.py, or upgrade {self.package_name} to a release that records its commit" + ) + try: + return GitSource.model_validate_json(self.build_info.read_text(encoding="utf-8")) + except ValidationError as exc: + raise TimeNetBuildError( + f"{self.package_name} records an invalid commit in {self.build_info}: {exc}" + ) from exc + + +class InstalledDistribution: + """One distribution that is installed in this interpreter.""" + + def __init__(self, distribution: Distribution) -> None: + self._distribution = distribution + + @classmethod + def from_name(cls, name: PackageName) -> Self: + """Find an installed package by name, such as ``timenet``. + + Note: An extra like ``timenet[zarr]`` is not a name, and ``from_name`` would not find it. + + Returns: + The installed package. + + Raises: + PackageNotFoundError: If it is not installed. + """ # ruff: ignore[docstring-extraneous-exception] - raised by Distribution.from_name + return cls(Distribution.from_name(name)) + + @property + def name(self) -> PackageName: + """The package name that pip installs, for example ``timenet-connectors``.""" + return self._distribution.metadata["Name"] + + @property + def version(self) -> str: + """The installed version.""" + return self._distribution.version + + def install_source(self) -> InstallSource: + """Return how this distribution was installed, which decides where its commit is recorded.""" + direct_url = self._read_direct_url() + if direct_url is not None: + commit_id = direct_url.installed_git_commit() + if commit_id is not None: + return GitInstall(self.name, commit_id) + project_directory = direct_url.local_directory() + if direct_url.is_editable() and project_directory is not None: + return EditableInstall(self.name, project_directory) + # An install from an index has no direct_url.json. A path install that is not editable + # copied the package, so the live checkout can differ from the code that runs. + module = self.name.replace("-", "_") + build_info = self._distribution.locate_file(f"{module}/{WheelInstall.BUILD_INFO_FILE}") + return WheelInstall(self.name, Path(str(build_info))) + + def read_git_source(self) -> GitSource: + """Return the commit of the code that this distribution runs. Raises ``TimeNetBuildError`` if it is unknown.""" + return self.install_source().read_git_source() + + def path_to_ondisk_project_directory(self) -> Path | None: + """Return the project directory of an editable or local path install, else ``None``.""" + direct_url = self._read_direct_url() + return None if direct_url is None else direct_url.local_directory() + + def _read_direct_url(self) -> "_DirectUrl | None": + """Read the PEP 610 ``direct_url.json``, or return ``None`` for an install from an index.""" + raw = self._distribution.read_text("direct_url.json") + if raw is None: + return None + try: + return _DirectUrl.model_validate_json(raw) + except ValidationError as exc: + raise TimeNetBuildError(f"{self.name} has a malformed direct_url.json: {exc}") from exc + + +class Snapshot(BaseModel, ABC): + """The commit of some installed code, read at one point in time.""" + + model_config = ConfigDict(frozen=True, validate_by_name=True) + + @abstractmethod + def to_source_map(self) -> GitSourceMap: + """Return the commits in the form that a manifest records.""" + + +class PackageSnapshot(Snapshot): + """The commit of one package that pip installed, such as ``timenet`` or ``timenet-connectors``. + + Note: Not a subpackage such as ``timenet.registry``. """ - packages: dict[str, str] = {} - for dist in distributions(): - name = dist.metadata["Name"] - if name: - packages[name] = dist.version - return {"python": platform.python_version(), "packages": dict(sorted(packages.items()))} + + package_name: PackageName + """The name pip installs, for example ``timenet-connectors``.""" + source: GitSource + + @classmethod + def read(cls, package_name: PackageName) -> Self: + """Read the commit of an installed package. Raises ``TimeNetBuildError`` if it is not installed.""" + snapshot = cls.read_if_installed(package_name) + if snapshot is None: + raise TimeNetBuildError( + f"cannot find the git commit of {package_name}: it is not installed in {sys.executable}" + ) + return snapshot + + @classmethod + def read_if_installed(cls, package_name: PackageName) -> Self | None: + """Read the commit of a package, or return ``None`` if it is not installed.""" + try: + installed = InstalledDistribution.from_name(package_name) + except PackageNotFoundError: + return None + return cls(package_name=package_name, source=installed.read_git_source()) + + def to_source_map(self) -> GitSourceMap: + """Return this package's commit in the form that a manifest records.""" + return {self.package_name: self.source} + + +class TimeNetSnapshot(Snapshot): + """The commit of each installed TimeNet package, read at one point in time.""" + + timenet: PackageSnapshot + timenet_connectors: PackageSnapshot | None = Field(default=None, alias="timenet-connectors") + """``None`` if timenet-connectors is not installed.""" + + @model_validator(mode="after") + def _each_field_holds_its_package(self) -> Self: + for name, field in type(self).model_fields.items(): + package = getattr(self, name) + package_name = field.alias or name + if package is not None and package.package_name != package_name: + raise ValueError(f"the field for {package_name} holds the commit of {package.package_name}") + return self + + @classmethod + def package_names(cls) -> tuple[PackageName, ...]: + """Return the name of each TimeNet package, in field order.""" + return tuple(field.alias or name for name, field in cls.model_fields.items()) + + @classmethod + def read(cls) -> Self: + """Read the commit of each installed TimeNet package. Raises ``TimeNetBuildError`` if one is unknown.""" + return cls( + timenet=PackageSnapshot.read("timenet"), + timenet_connectors=PackageSnapshot.read_if_installed("timenet-connectors"), + ) + + @classmethod + def from_source_map(cls, sources: GitSourceMap) -> Self: + """Return the typed view of the commits that a manifest records.""" + packages = {name: PackageSnapshot(package_name=name, source=source) for name, source in sources.items()} + try: + # The field aliases map each package name to its field. + return cls.model_validate(packages) + except ValidationError as exc: + missing = ", ".join(str(error["loc"][0]) for error in exc.errors()) + raise TimeNetBuildError(f"the manifest records no git commit for {missing}") from exc + + def to_source_map(self) -> GitSourceMap: + """Return the commit of each installed TimeNet package.""" + sources: GitSourceMap = {} + for package in self._installed(): + sources.update(package.to_source_map()) + return sources + + def _installed(self) -> list[PackageSnapshot]: + """Return the snapshot of each installed TimeNet package.""" + return [package for package in (self.timenet, self.timenet_connectors) if package is not None] + + +class _DirInfo(BaseModel): + model_config = ConfigDict(frozen=True) + + editable: StrictBool = False + + +class _VcsInfo(BaseModel): + model_config = ConfigDict(frozen=True) + + vcs: StrictStr + commit_id: StrictStr + + +class _DirectUrl(BaseModel): + """The PEP 610 record of how a distribution was installed. Keys this module does not read are ignored.""" + + model_config = ConfigDict(frozen=True) + + url: StrictStr + dir_info: _DirInfo | None = None + vcs_info: _VcsInfo | None = None + subdirectory: StrictStr | None = None + """The project's folder inside ``url``, for a project that does not sit at its root.""" + + def installed_git_commit(self) -> str | None: + """Return the commit that pip installed from a git repository, else ``None``.""" + if self.vcs_info is None or self.vcs_info.vcs != "git": + return None + return self.vcs_info.commit_id + + def is_editable(self) -> bool: + """Whether the distribution was installed editable.""" + return self.dir_info is not None and self.dir_info.editable + + def local_directory(self) -> Path | None: + """Return the directory of an editable or local path install, else ``None``.""" + if not self._is_file_url(): + return None + path = Path(url2pathname(urlparse(self.url).path)) + if self.subdirectory is not None: + path /= self.subdirectory + return path if self.is_editable() or path.is_dir() else None + + def _is_file_url(self) -> bool: + return urlparse(self.url).scheme == "file" + + +class _GitError(Exception): + """Git could not read a checkout. The message gives the reason.""" + + +class _GitNotFoundError(_GitError): + """No git executable is on PATH.""" + + +@dataclass(frozen=True) +class _GitCheckout: + """A directory inside a git checkout, read with the git on PATH.""" + + directory: Path + + def head_commit(self) -> str: + """Return the commit that the checkout is at.""" + return self._run("rev-parse", "HEAD") + + def has_local_changes(self) -> bool: + """Whether the directory has uncommitted changes, untracked files included.""" + # The pathspec "." scopes the status to the directory. A scratch file elsewhere in the + # repository does not count as a local change. + return bool(self._run("status", "--porcelain", "--untracked-files=all", "--", ".")) + + def _run(self, *args: str) -> str: + """Run one git command in the directory and return its stripped stdout.""" + git = shutil.which("git") + if git is None: + raise _GitNotFoundError("git is not on PATH") + try: + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + # --no-optional-locks keeps `git status` from refreshing the index, so a build never + # takes the index lock while the user runs git. + [git, "--no-optional-locks", "-C", str(self.directory), *args], + capture_output=True, + text=True, + check=True, + timeout=_GIT_TIMEOUT_S, + ) + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or "").strip() or str(exc) + raise _GitError(f"git failed in {self.directory}: {detail}") from exc + except (OSError, subprocess.TimeoutExpired) as exc: + raise _GitError(f"git failed in {self.directory}: {exc}") from exc + return result.stdout.strip() diff --git a/packages/timenet/src/timenet/registry/s3.py b/packages/timenet/src/timenet/registry/s3.py index 1209c028f..19c654e1a 100644 --- a/packages/timenet/src/timenet/registry/s3.py +++ b/packages/timenet/src/timenet/registry/s3.py @@ -244,7 +244,6 @@ def _reaching(self) -> Generator[None]: raise TimeNetUnreachableError(f"cannot reach {self._display_root()}: {error}") from error def _key(self, *parts: str) -> str: - # ruff: ignore[docstring-missing-returns] """Build a bucket-relative key: the prefix, then ``datasets/``, then the path parts.""" segments = [self._prefix, _DATASETS_PREFIX, *parts] return "/".join(segment.strip("/") for segment in segments if segment) diff --git a/packages/timenet/src/timenet/schemas/manifest.schema.json b/packages/timenet/src/timenet/schemas/manifest.schema.json index f75f97d41..c707cafb0 100644 --- a/packages/timenet/src/timenet/schemas/manifest.schema.json +++ b/packages/timenet/src/timenet/schemas/manifest.schema.json @@ -86,7 +86,7 @@ }, "BuildEnvironment": { "additionalProperties": false, - "description": "Build-provenance fields recorded in a manifest.", + "description": "The Python version, the installed packages, and the TimeNet commits that produced a version.", "properties": { "python": { "title": "Python", @@ -98,8 +98,20 @@ }, "title": "Packages", "type": "object" + }, + "git": { + "additionalProperties": { + "$ref": "#/$defs/GitSource" + }, + "title": "Git", + "type": "object" } }, + "required": [ + "python", + "packages", + "git" + ], "title": "BuildEnvironment", "type": "object" }, @@ -363,6 +375,27 @@ "title": "FilePart", "type": "object" }, + "GitSource": { + "additionalProperties": false, + "description": "The git commit that one TimeNet package was built from.", + "properties": { + "sha": { + "pattern": "^[0-9a-f]{40}$", + "title": "Sha", + "type": "string" + }, + "contains_local_changes": { + "title": "Contains Local Changes", + "type": "boolean" + } + }, + "required": [ + "sha", + "contains_local_changes" + ], + "title": "GitSource", + "type": "object" + }, "InputModality": { "description": "A kind of input supplied to a task.", "enum": [ @@ -751,6 +784,7 @@ "dataset_id", "metadata", "files", + "build_env", "timef_format_version" ], "title": "Manifest", diff --git a/packages/timenet/src/timenet/writer/writer.py b/packages/timenet/src/timenet/writer/writer.py index f8e094d92..4056eb061 100644 --- a/packages/timenet/src/timenet/writer/writer.py +++ b/packages/timenet/src/timenet/writer/writer.py @@ -27,8 +27,8 @@ from timenet.format.storage import StorageSources from timenet.format.stored_array import StoredArray from timenet.format.stored_values import PackedValueWriter -from timenet.manifest import FilePart, Manifest, ManifestCounts, ManifestFiles, PayloadSource -from timenet.provenance import build_env +from timenet.manifest import BuildEnvironment, FilePart, Manifest, ManifestCounts, ManifestFiles, PayloadSource +from timenet.provenance import TimeNetSnapshot from timenet.types.ids import EntityId from timenet.types.reference_keys import SourceKey from timenet.types.references import DatasetRef @@ -165,9 +165,13 @@ def __enter__(self) -> "TimeFWriter": Raises: FileExistsError: If a committed ``manifest.json`` already exists at the version directory and ``replace`` is not set. - """ + TimeNetBuildError: If the commit of an installed TimeNet package is unknown. + """ # ruff: ignore[docstring-extraneous-exception] - raised by TimeNetSnapshot.read if not self._replace and (self._final_dir / MANIFEST_FILE).exists(): raise FileExistsError(f"a committed version already exists at {self._final_dir}") + # Read the commit before any data is written. An unknown commit then fails before the work, + # and a commit or checkout during the write does not change the record. + self._snapshot = TimeNetSnapshot.read() self._sweep_stale_staging() self._staging_dir.mkdir(parents=True, exist_ok=True) return self @@ -475,7 +479,7 @@ def _write_manifest(self) -> Manifest: payload_sources=self._storage_sources.sources, parent_cards=import_state.parent_cards, lineage=tuple(import_state.lineage.values()), - build_env=build_env(), + build_env=BuildEnvironment.capture(git=self._snapshot.to_source_map()), timef_format_version=1, ) (self._staging_dir / MANIFEST_FILE).write_text(manifest.to_json()) diff --git a/packages/timenet/tests/format/test_control_cache.py b/packages/timenet/tests/format/test_control_cache.py index 18746c4ac..3f05cd908 100644 --- a/packages/timenet/tests/format/test_control_cache.py +++ b/packages/timenet/tests/format/test_control_cache.py @@ -6,7 +6,7 @@ from timenet.format.control_writer import DuckDBControlWriter from timenet.format.storage import StorageSources from timenet.format.stored_values import PackedValueWriter -from timenet.manifest import FilePart, Manifest, ManifestFiles +from timenet.manifest import BuildEnvironment, FilePart, Manifest, ManifestFiles from timenet.registry.version import DatasetVersion from .test_control_writer import _dataset @@ -22,6 +22,7 @@ def test_local_control_database_uses_its_existing_path(tmp_path): files=ManifestFiles( control=(FilePart(path=path.name, checksum=file_checksum(path), size=path.stat().st_size),), ), + build_env=BuildEnvironment(python="3.13.1", packages={}, git={}), timef_format_version=1, ) version = DatasetVersion( @@ -45,6 +46,7 @@ def test_a_remote_control_database_lands_in_a_control_folder_with_the_marker(tmp control=(FilePart(path=path.name, checksum=file_checksum(path), size=path.stat().st_size),), ), timef_format_version=1, + build_env=BuildEnvironment(python="3.13.1", packages={}, git={}), ) # Any file system but the local one counts as remote, so the database is copied into the cache. remote = pafs.SubTreeFileSystem(str(path.parent), pafs.LocalFileSystem()) diff --git a/packages/timenet/tests/manifest/test_manifest.py b/packages/timenet/tests/manifest/test_manifest.py index 1f9acc0ac..3eaa40631 100644 --- a/packages/timenet/tests/manifest/test_manifest.py +++ b/packages/timenet/tests/manifest/test_manifest.py @@ -1,3 +1,4 @@ +import sys from typing import Any, TypeVar, cast from hypothesis import given, strategies as st @@ -6,7 +7,15 @@ from pydantic import BaseModel, ValidationError import pytest -from timenet.manifest import FilePart, Manifest, ManifestCounts, ManifestFiles, PayloadSource +from timenet.manifest import ( + BuildEnvironment, + FilePart, + GitSource, + Manifest, + ManifestCounts, + ManifestFiles, + PayloadSource, +) from timenet.schemas import MANIFEST_SCHEMA from timenet.types import ( AnnotationDescriptor, @@ -31,6 +40,11 @@ _CONTROL = (FilePart(path="control.duckdb", checksum="sha256:" + "0" * 64, size=5),) +_BUILD_ENV = BuildEnvironment( + python="3.13.1", + packages={"timenet": "0.1.0"}, + git={"timenet": GitSource(sha="a" * 40, contains_local_changes=False)}, +) _ModelT = TypeVar("_ModelT", bound=BaseModel) @@ -89,6 +103,7 @@ def _manifest(*, values_backend: str = ValuesBackend.PARQUET) -> Manifest: ), ), values_backend=cast("ValuesBackend", values_backend), + build_env=_BUILD_ENV, timef_format_version=1, ) @@ -161,6 +176,7 @@ def test_missing_values_round_trip_at_format_version_1(): metadata=base.metadata, files=base.files, dataset_schema=_replace(base.dataset_schema, time_series_specs=(spec,)), + build_env=_BUILD_ENV, timef_format_version=1, ) assert manifest.timef_format_version == 1 @@ -299,6 +315,7 @@ def test_direct_construction_validates_format_version(): dataset_id="x", metadata=_manifest().metadata, files=_manifest().files, + build_env=_BUILD_ENV, timef_format_version=99, ) @@ -309,11 +326,12 @@ def test_dataset_id_must_match_metadata(): dataset_id="other/dataset", metadata=_manifest().metadata, files=_manifest().files, + build_env=_BUILD_ENV, timef_format_version=1, ) -@pytest.mark.parametrize("missing", ["timef_format_version", "dataset_id", "metadata", "files"]) +@pytest.mark.parametrize("missing", ["timef_format_version", "dataset_id", "metadata", "files", "build_env"]) def test_model_validate_requires_core_blocks(missing): d = _manifest().to_dict() del d[missing] @@ -457,6 +475,7 @@ def test_codec_roundtrip_property(version, records, task_counts): ), counts=ManifestCounts(records=records, tasks=task_counts), files=ManifestFiles(control=_CONTROL), + build_env=_BUILD_ENV, timef_format_version=1, ) assert Manifest.model_validate_json(manifest.to_json()) == manifest @@ -483,19 +502,36 @@ def test_bad_dict_block_names_itself(block): Manifest.model_validate(d) -def test_build_env_defaults_to_empty(): - assert _manifest().build_env == {} +def test_build_env_captures_the_running_python_and_the_installed_packages(): + build_env = BuildEnvironment.capture(git=_BUILD_ENV.git) + assert build_env.python == ".".join(str(part) for part in sys.version_info[:3]) + assert build_env.packages["timenet"] + assert list(build_env.packages) == sorted(build_env.packages) + assert build_env.git == _BUILD_ENV.git def test_build_env_round_trips(): - m = _replace(_manifest(), build_env={"python": "3.11.9", "packages": {"timenet": "0.1.0"}}) + git = {"timenet": {"sha": "a" * 40, "contains_local_changes": False}} + m = _replace(_manifest(), build_env={"python": "3.11.9", "packages": {"timenet": "0.1.0"}, "git": git}) assert Manifest.model_validate(m.to_dict()).build_env == m.build_env @pytest.mark.parametrize( "change", [ - {"build_env": {"python": 313}}, + {"build_env": {**_BUILD_ENV.model_dump(), "python": 313}}, + { + "build_env": { + **_BUILD_ENV.model_dump(), + "git": {"timenet": {"sha": "4818e37f", "contains_local_changes": False}}, + } + }, + { + "build_env": { + **_BUILD_ENV.model_dump(), + "git": {"timenet": {"sha": "a" * 40, "contains_local_changes": "no"}}, + } + }, {"value_encoding": {"ecg_lead": "zip"}}, ], ) @@ -505,17 +541,16 @@ def test_direct_manifest_rejects_invalid_provenance_blocks(change): def test_to_dict_copies_build_env(): - build_env = {"python": "3.11.9", "packages": {"timenet": "0.1.0"}} - d = _replace(_manifest(), build_env=build_env).to_dict() - d["build_env"]["python"] = "2.7.0" - assert build_env["python"] == "3.11.9" + manifest = _manifest() + manifest.to_dict()["build_env"]["packages"]["timenet"] = "2.7.0" + assert manifest.build_env.packages["timenet"] == "0.1.0" @pytest.mark.parametrize("method", ["to_dict", "to_json"]) @pytest.mark.parametrize( "mutate", [ - lambda manifest: manifest.build_env.update(python=313), + lambda manifest: manifest.build_env.packages.update(timenet=313), lambda manifest: manifest.counts.tasks.update(classification=-1), ], ) diff --git a/packages/timenet/tests/manifest/test_manifest_schema.py b/packages/timenet/tests/manifest/test_manifest_schema.py index c07b21545..bdfe30c9b 100644 --- a/packages/timenet/tests/manifest/test_manifest_schema.py +++ b/packages/timenet/tests/manifest/test_manifest_schema.py @@ -1,7 +1,7 @@ import jsonschema import pytest -from timenet.manifest import FilePart, Manifest, ManifestCounts, ManifestFiles +from timenet.manifest import BuildEnvironment, FilePart, GitSource, Manifest, ManifestCounts, ManifestFiles from timenet.schemas import MANIFEST_SCHEMA from timenet.types import ( AnnotationDescriptor, @@ -18,6 +18,13 @@ ) +_BUILD_ENV = BuildEnvironment( + python="3.13.1", + packages={"timenet": "0.1.0"}, + git={"timenet": GitSource(sha="a" * 40, contains_local_changes=False)}, +) + + def _manifest() -> Manifest: spec = TimeSeriesSpec( spec_type="ecg", @@ -72,6 +79,7 @@ def _manifest() -> Manifest: ), ), ), + build_env=_BUILD_ENV, timef_format_version=1, ) diff --git a/packages/timenet/tests/registry/test_copy.py b/packages/timenet/tests/registry/test_copy.py index db04faaef..4360495a2 100644 --- a/packages/timenet/tests/registry/test_copy.py +++ b/packages/timenet/tests/registry/test_copy.py @@ -223,7 +223,7 @@ def test_copy_release_refuses_a_release_with_the_same_files_and_another_card(sou def test_copy_release_ignores_the_build_environment(source, tmp_path): dest = LocalRegistry(tmp_path / "dest") copy_release(child_ref, source, dest) - _rewrite_child_manifest(dest, build_env={"python": "0.0.0"}) + _rewrite_child_manifest(dest, build_env={"python": "0.0.0", "packages": {}, "git": {}}) events: list[ProgressEvent] = [] copy_release(child_ref, source, LocalRegistry(dest.root), progress=events.append) assert events == [] diff --git a/packages/timenet/tests/test_provenance.py b/packages/timenet/tests/test_provenance.py index 074d144fa..dfb244d05 100644 --- a/packages/timenet/tests/test_provenance.py +++ b/packages/timenet/tests/test_provenance.py @@ -1,13 +1,234 @@ -import sys +from importlib.metadata import Distribution, PackageNotFoundError, PathDistribution +import json +import os +from pathlib import Path +import shutil +import subprocess -from timenet.provenance import build_env +from pydantic import ValidationError +import pytest +from timenet.errors import TimeNetBuildError +from timenet.manifest import GitSource +from timenet.provenance import InstalledDistribution, PackageSnapshot, TimeNetSnapshot +from timenet.registry import LocalRegistry +from timenet.testing import make_dataset +from timenet.writer import TimeFWriter -def test_build_env_records_the_running_python(): - assert build_env()["python"] == ".".join(str(part) for part in sys.version_info[:3]) +_CLEAN = GitSource(sha="a" * 40, contains_local_changes=False) -def test_build_env_records_installed_packages(): - packages = build_env()["packages"] - assert packages["timenet"] - assert packages == dict(sorted(packages.items())) + +@pytest.fixture +def git(monkeypatch: pytest.MonkeyPatch) -> str: + """The git executable, with the user's and the system's git configuration switched off.""" + executable = shutil.which("git") + if executable is None: + pytest.skip("git is not on PATH") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + return executable + + +def _run(git: str, directory: Path, *args: str) -> str: + return subprocess.run([git, "-C", str(directory), *args], capture_output=True, text=True, check=True).stdout.strip() + + +@pytest.fixture +def checkout(tmp_path: Path, git: str) -> Path: + """A committed repository with one package directory, ``packages/pkg``.""" + repo = tmp_path / "repo" + package = repo / "packages" / "pkg" + package.mkdir(parents=True) + (package / "module.py").write_text("x = 1\n") + (repo / "README.md").write_text("readme\n") + _run(git, repo, "init", "-q") + # Hide untracked files from a plain status, so the lookup must ask for them itself. + _run(git, repo, "config", "status.showUntrackedFiles", "no") + _run(git, repo, "add", ".") + _run(git, repo, "-c", "user.name=t", "-c", "user.email=t@example.org", "commit", "-qm", "init") + return package + + +def _install(site: Path, *, direct_url: dict | None = None, build_info: str | None = None) -> PathDistribution: + """Lay out an installed ``timenet`` with an optional ``direct_url.json`` and build info.""" + dist_info = site / "timenet-1.0.dist-info" + dist_info.mkdir(parents=True) + (site / "timenet").mkdir() + record = ["timenet/__init__.py,,", "timenet-1.0.dist-info/RECORD,,"] + if build_info is not None: + (site / "timenet" / "_build_info.json").write_text(build_info) + record.append("timenet/_build_info.json,,") + if direct_url is not None: + (dist_info / "direct_url.json").write_text(json.dumps(direct_url)) + (dist_info / "METADATA").write_text("Metadata-Version: 2.1\nName: timenet\nVersion: 1.0\n") + (dist_info / "RECORD").write_text("\n".join(record) + "\n") + return PathDistribution(dist_info) + + +@pytest.fixture +def installed(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Make a fake ``timenet`` the only installed TimeNet distribution. Call it to lay one out.""" + + def install(**kwargs) -> None: + dist = _install(tmp_path / "site", **kwargs) + + def from_name(name: str) -> Distribution: + if name != "timenet": + raise PackageNotFoundError(name) + return dist + + monkeypatch.setattr(Distribution, "from_name", staticmethod(from_name)) + + return install + + +def _editable(directory: Path) -> dict: + return {"url": directory.as_uri(), "dir_info": {"editable": True}} + + +@pytest.mark.real_provenance +def test_an_editable_install_records_the_commit_of_its_checkout(git, checkout, installed): + installed(direct_url=_editable(checkout)) + head = _run(git, checkout, "rev-parse", "HEAD") + # timenet-connectors is not installed, so the snapshot has no entry for it. + assert TimeNetSnapshot.read().to_source_map() == {"timenet": GitSource(sha=head, contains_local_changes=False)} + + +@pytest.mark.real_provenance +@pytest.mark.parametrize("change", ["module.py", "untracked.py"]) +def test_a_change_in_the_package_directory_is_a_local_change(checkout, installed, change): + installed(direct_url=_editable(checkout)) + (checkout / change).write_text("x = 2\n") + assert PackageSnapshot.read("timenet").source.contains_local_changes is True + + +@pytest.mark.real_provenance +def test_a_change_outside_the_package_directory_is_not_a_local_change(checkout, installed): + installed(direct_url=_editable(checkout)) + (checkout.parents[1] / "README.md").write_text("edited\n") + (checkout.parents[1] / "scratch.ipynb").write_text("{}\n") + assert PackageSnapshot.read("timenet").source.contains_local_changes is False + + +@pytest.mark.real_provenance +def test_an_editable_install_outside_git_fails_with_the_git_error(git, tmp_path, installed): + outside = tmp_path / "outside" + outside.mkdir() + installed(direct_url=_editable(outside)) + with pytest.raises(TimeNetBuildError, match=r"cannot find the git commit of timenet.*not a git repository"): + PackageSnapshot.read("timenet") + + +@pytest.mark.real_provenance +def test_an_editable_install_without_git_fails(checkout, installed, monkeypatch): + installed(direct_url=_editable(checkout)) + monkeypatch.setenv("PATH", "") + with pytest.raises(TimeNetBuildError, match="git is not on PATH"): + PackageSnapshot.read("timenet") + + +@pytest.mark.real_provenance +@pytest.mark.parametrize("editable", [True, False]) +def test_an_install_of_a_subdirectory_finds_the_package_directory(checkout, installed, editable): + # direct_url.json may name the repository and put the package in "subdirectory". + repository = checkout.parents[1] + installed( + direct_url={"url": repository.as_uri(), "dir_info": {"editable": editable}, "subdirectory": "packages/pkg"} + ) + assert InstalledDistribution.from_name("timenet").path_to_ondisk_project_directory() == checkout + if editable: + # A change outside the package directory does not count. + (repository / "README.md").write_text("edited\n") + assert PackageSnapshot.read("timenet").source.contains_local_changes is False + + +@pytest.mark.real_provenance +def test_a_path_install_reads_its_build_info_and_not_the_live_checkout(checkout, installed): + installed(direct_url={"url": checkout.as_uri(), "dir_info": {}}, build_info=_CLEAN.model_dump_json()) + assert PackageSnapshot.read("timenet").source == _CLEAN + + +@pytest.mark.real_provenance +def test_a_git_install_records_the_commit_that_pip_installed(installed): + sha = "b" * 40 + installed(direct_url={"url": "https://example.org/timenet.git", "vcs_info": {"vcs": "git", "commit_id": sha}}) + assert PackageSnapshot.read("timenet").source == GitSource(sha=sha, contains_local_changes=False) + + +@pytest.mark.real_provenance +def test_a_wheel_reads_its_commit_from_the_build_info(installed): + info = GitSource(sha="a" * 40, contains_local_changes=True) + installed(build_info=info.model_dump_json()) + assert PackageSnapshot.read("timenet").source == info + + +@pytest.mark.real_provenance +def test_a_wheel_without_build_info_fails_and_says_how_to_build_one(installed): + installed() + with pytest.raises(TimeNetBuildError, match=r"carries no _build_info\.json\. Build it with `make build`"): + PackageSnapshot.read("timenet") + + +@pytest.mark.real_provenance +@pytest.mark.parametrize( + "build_info", + [ + # The manifest forbids extra keys for every nested block, but this standalone check must + # forbid them itself. + json.dumps({"sha": "a" * 40, "contains_local_changes": False, "branch": "main"}), + "not json", + ], +) +def test_a_wheel_with_invalid_build_info_fails(installed, build_info): + installed(build_info=build_info) + with pytest.raises(TimeNetBuildError, match=r"_build_info\.json"): + PackageSnapshot.read("timenet") + + +@pytest.mark.real_provenance +def test_a_local_build_fails_if_a_commit_is_unknown(installed, tmp_path): + installed() + with pytest.raises(TimeNetBuildError, match="cannot find the git commit of timenet"): + LocalRegistry(tmp_path / "local").put_dataset(make_dataset()) + # The writer reads the commit before it writes any data. + assert not (tmp_path / "local" / make_dataset().metadata.dataset_id).exists() + + +def test_the_manifest_records_the_commit_from_when_the_write_started(tmp_path, monkeypatch): + original_write = TimeFWriter.write + + def write_then_commit(writer: TimeFWriter) -> None: + original_write(writer) + later = GitSource(sha="b" * 40, contains_local_changes=False) + monkeypatch.setattr(InstalledDistribution, "read_git_source", lambda _self: later) + + monkeypatch.setattr(TimeFWriter, "write", write_then_commit) + dataset = make_dataset() + registry = LocalRegistry(tmp_path / "local") + registry.put_dataset(dataset) + + manifest = registry.get_manifest(dataset.metadata.dataset_id) + assert manifest.build_env.git["timenet"].sha == "0" * 40 + + +def test_a_snapshot_reads_back_from_the_commits_that_a_manifest_records(): + snapshot = TimeNetSnapshot.read() + assert TimeNetSnapshot.from_source_map(snapshot.to_source_map()) == snapshot + + +def test_a_manifest_without_timenet_connectors_reads_as_not_installed(): + snapshot = TimeNetSnapshot.from_source_map({"timenet": _CLEAN}) + assert snapshot.timenet_connectors is None + assert snapshot.to_source_map() == {"timenet": _CLEAN} + + +def test_a_manifest_without_the_commit_of_timenet_is_refused(): + with pytest.raises(TimeNetBuildError, match="the manifest records no git commit for timenet"): + TimeNetSnapshot.from_source_map({"timenet-connectors": _CLEAN}) + + +def test_a_field_must_hold_the_commit_of_its_own_package(): + connectors = PackageSnapshot(package_name="timenet-connectors", source=_CLEAN) + with pytest.raises(ValidationError, match="the field for timenet holds the commit of timenet-connectors"): + TimeNetSnapshot(timenet=connectors) diff --git a/pyproject.toml b/pyproject.toml index 84360d028..89287ffd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,10 @@ banned-module-level-imports = ["typer", "timenet.cli", "timenet_connectors.build # timenet-connectors depends on timenet, so timenet never imports it and works without it. "timenet_connectors".msg = "timenet must work without timenet-connectors" +[tool.ruff.lint.pydoclint] +# A one-line docstring of a small helper does not need Returns or Raises sections. +ignore-one-line-docstrings = true + [tool.ruff.lint.flake8-bugbear] # Typer expects Option/Argument descriptors as parameter defaults. extend-immutable-calls = ["typer.Option", "typer.Argument"] @@ -144,6 +148,7 @@ testpaths = ["packages", "benchmarks/end_to_end/tests"] markers = [ "slow: full-disk round-trip and property tests (deselect with '-m \"not slow\"')", "value_dtypes: scalar value-dtype round-trip tests (run with '-m value_dtypes')", + "real_provenance: read the real commit of every install type instead of the stub in packages/conftest.py", ] [tool.ty.src] diff --git a/scripts/build_dists.py b/scripts/build_dists.py new file mode 100644 index 000000000..3272465d6 --- /dev/null +++ b/scripts/build_dists.py @@ -0,0 +1,114 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Run ``uv build`` with the git commit of each package recorded in its ``_build_info.json``. + +Usage: ``uv run scripts/build_dists.py --all-packages`` (any ``uv build`` arguments work). +""" + +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess # ruff: ignore[suspicious-subprocess-import] - the commands are fixed argument vectors +import sys + + +ROOT = Path(__file__).resolve().parents[1] +PACKAGES = { + ROOT / "packages" / "timenet": "timenet", + ROOT / "packages" / "timenet-connectors": "timenet_connectors", +} +# Keep in step with timenet.provenance.WheelInstall.BUILD_INFO_FILE, which reads the file. +BUILD_INFO_FILE = "_build_info.json" + + +def executable(name: str) -> str: + """Return the absolute path of a command on ``PATH``, or exit if it is missing.""" + path = shutil.which(name) + if path is None: + raise SystemExit(f"{name} is not on PATH") + return path + + +def git(directory: Path, *args: str) -> str: + """Run one git command in ``directory`` and return its stripped stdout, or exit if it fails.""" + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] + [executable("git"), "-C", str(directory), *args], capture_output=True, text=True, check=False + ) + if result.returncode != 0: + raise SystemExit(f"git {' '.join(args)} failed in {directory}: {result.stderr.strip()}") + return result.stdout.strip() + + +def state(directory: Path) -> tuple[str, str]: + """Return the commit of a package directory and its ``git status`` output.""" + # Read the commit after the status, as timenet.provenance.EditableInstall does. + status = git(directory, "status", "--porcelain", "--untracked-files=all", "--", ".") + return git(directory, "rev-parse", "HEAD"), status + + +def main(build_args: list[str]) -> int: + """Write the commit records, build, and remove the records again, also on failure.""" + written: dict[Path, str] = {} + before = {directory: state(directory) for directory in PACKAGES} + try: + for directory, module in PACKAGES.items(): + sha, status = before[directory] + # Keep the keys in step with timenet.manifest.GitSource. + info = {"sha": sha, "contains_local_changes": bool(status)} + target = directory / "src" / module / BUILD_INFO_FILE + text = json.dumps(info) + "\n" + # Register the file before the write, so a write that fails halfway is removed too. + written[target] = text + target.write_text(text, encoding="utf-8") + print(f"{target.relative_to(ROOT)}: {info}", file=sys.stderr) + # uv run sets UV to its own executable. + uv = os.environ.get("UV") or executable("uv") + returncode = subprocess.run([uv, "build", *build_args], cwd=ROOT, check=False).returncode # ruff: ignore[subprocess-without-shell-equals-true] + # A file that changed while uv built would ship under a record that does not describe it. + changed = [str(directory.relative_to(ROOT)) for directory in PACKAGES if state(directory) != before[directory]] + if changed: + print( + f"error: {', '.join(changed)} changed during the build, so the built artifacts do not " + "match their commit records. Build again.", + file=sys.stderr, + ) + return 1 + # git ignores the records, so the status above cannot see one that another build removed. + lost = [str(path.relative_to(ROOT)) for path, text in written.items() if not _holds(path, text)] + if lost: + print( + f"error: {', '.join(lost)} changed during the build, so the built artifacts can lack their " + "commit records. Do not run two builds in one checkout at the same time. Build again.", + file=sys.stderr, + ) + return 1 + return returncode + finally: + for path in written: + path.unlink(missing_ok=True) + + +def _holds(path: Path, text: str) -> bool: + """Return whether ``path`` exists and holds ``text``.""" + try: + return path.read_text(encoding="utf-8") == text + except FileNotFoundError: + return False + + +def _exit_on_signal(signum: int, _frame: object) -> None: + """Raise ``SystemExit``, so that the ``finally`` in ``main`` removes the commit records.""" + raise SystemExit(128 + signum) + + +if __name__ == "__main__": + # Ctrl-C already raises KeyboardInterrupt. Without a handler, a kill or a closed terminal ends the + # process before the finally runs, and a later plain uv build would pack the stale records. + for name in ("SIGTERM", "SIGHUP"): + if hasattr(signal, name): + signal.signal(getattr(signal, name), _exit_on_signal) + sys.exit(main(sys.argv[1:]))