feat(manifest)!: record TimeNet git provenance in each written dataset

.github/workflows/publish.yml

@@ -52,36 +52,45 @@ jobs:
       - name: Set up uv
         uses: astral-sh/setup-uv@v7
       - name: Build
+        # build_dists.py records the commit inside each package, so a build can name its code.
         run: |
           if [ "${{ inputs.package }}" = "both" ]; then
-            uv build --all-packages
+            uv run scripts/build_dists.py --all-packages
           else
-            uv build --package "${{ inputs.package }}"
+            uv run scripts/build_dists.py --package "${{ inputs.package }}"
           fi
       - name: Smoke test built artifacts
         # Import each built wheel and sdist in an isolated environment to catch missing files or
-        # broken metadata before uploading. timenet-connectors depends on timenet, so hand it the
+        # broken metadata before uploading. Each artifact must also report this commit, clean, as
+        # the code it was built from. timenet-connectors depends on timenet, so hand it the
         # wheel we just built when it is present (a `both` run); otherwise timenet resolves from the
         # default index, which requires it to be published already.
         run: |
           set -euo pipefail
           shopt -s nullglob
           timenet_whl=(dist/timenet-*.whl)
           smoke() {
-            local module="$1"; shift
+            local module="$1" dist="$2"; shift 2
             local args=() artifact
             for artifact in "$@"; do args+=(--with "$artifact"); done
             echo "::group::import $module"
             uv run --isolated --no-project "${args[@]}" \
               python -c "import $module; print('smoke ok:', $module.__name__)"
+            uv run --isolated --no-project "${args[@]}" python -c "
+          from timenet.manifest import GitSource
+          from timenet.provenance import PackageSnapshot
+          commit = PackageSnapshot.read('$dist').source
+          expected = GitSource(sha='$GITHUB_SHA', contains_local_changes=False)
+          assert commit == expected, f'$dist records {commit}, expected {expected}'
+          print('commit ok:', commit)"
             echo "::endgroup::"
           }
           for artifact in dist/*; do
             case "$artifact" in
               dist/timenet-*.whl | dist/timenet-*.tar.gz)
-                smoke timenet "$artifact" ;;
+                smoke timenet timenet "$artifact" ;;
               dist/timenet_connectors-*.whl | dist/timenet_connectors-*.tar.gz)
-                smoke timenet_connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;;
+                smoke timenet_connectors timenet-connectors ${timenet_whl[@]+"${timenet_whl[@]}"} "$artifact" ;;
             esac
           done
       - name: Publish

.gitignore

@@ -47,3 +47,6 @@ docs/api/
 
 # Local superpowers planning artifacts, not part of the repo
 docs/superpowers/
+
+# commit record written into each import package before a wheel build (scripts/build_dists.py removes it after the build)
+packages/*/src/*/_build_info.json

AGENTS.md

@@ -60,7 +60,9 @@ workspace definition and the shared ruff/ty/pytest config; per-package
   workspace extras (`timenet[cli,torch]`). It does not install connector dependencies. Each
   connector declares its own in a `requirements.txt`, and `make test-connectors` runs that
   connector's tests and type-check in an environment built from it.
-- Build distributables with `make build` (`uv build --package <name>` per member).
+- Build distributables with `make build`. It runs the build through a proxy script, which records
+  the git commit inside each package for provenance, before running `uv build --package <name>`
+  per member.
 - For one-off scripts, use inline `uv` metadata and run with `uv run <script.py>`. Never `pip install`.
 - Keep `uv.lock` committed; the `uv-lock` pre-commit hook enforces freshness.
 

Makefile

@@ -55,9 +55,10 @@ license-check:
 		--fail-on="GPL;LGPL;AGPL;SSPL;EUPL;CDDL;OSL" \
 		--format=markdown
 
+# Builds through scripts/build_dists.py, so each wheel records the commit it was built from.
 build:
-	uv build --package timenet
-	uv build --package timenet-connectors
+	uv run scripts/build_dists.py --package timenet
+	uv run scripts/build_dists.py --package timenet-connectors
 
 check: schemas-check
 	uv run ruff format .

docs/build.md

@@ -48,8 +48,8 @@ variables (see the [settings table](client.md#settings)). For example,
 `$TIMENET_LOCAL_REGISTRY` & `TIMENET_ISOLATION=off`. These apply to every CLI command, and the
 Python SDK also honors them.
 
-For each build, the used Python and package and TimeNet versions are recorded to the build's
-manifest.
+The manifest records the Python and package versions used for the build, and the git commit of
+`timenet` and `timenet-connectors`. See [`build_env`](manifest.md) for the fields.
 
 ## Pipeline
 

docs/manifest.md

@@ -39,7 +39,7 @@ Manifest(
     counts=counts,              # ManifestCounts (default: empty)
     values_backend="parquet",   # "parquet" (default) or "zarr"
     value_encoding={},          # spec_type -> the encoding its shards carry
-    build_env={},               # {"python", "packages"} (default: empty)
+    build_env=build_env,        # BuildEnvironment (required)
     timef_format_version=1,     # required; validated against the supported set {1}
 )
 ```
@@ -53,9 +53,36 @@ of each spec type. No code reads this field. Parquet records the applied encodin
 each file, so the reader does not need it. The field lets a builder see which encoding a build
 selected. The field is empty for a backend that has no such choice.
 
-`build_env` records the environment that produced the version. It gives the interpreter version and
-every installed package with its version. `timenet.provenance.build_env` collects this data. Like
-`value_encoding`, `build_env` is provenance only. No code reads it to interpret the data.
+`build_env` records the environment that produced the version. It gives the interpreter version,
+every installed package with its version, and the git commit of the TimeNet code.
+`BuildEnvironment.capture` collects this data. Like `value_encoding`, `build_env` is provenance
+only. No code reads it to interpret the data.
+
+`build_env.git` records the git commit of each installed TimeNet package, keyed by package
+name. A package that is not installed has no entry:
+
+```json
+"git": {
+  "timenet": {"sha": "4818e37f...", "contains_local_changes": false},
+  "timenet-connectors": {"sha": "4818e37f...", "contains_local_changes": false}
+}
+```
+
+Each package records the commit of the code that runs:
+
+- An editable install, such as a workspace checkout, asks git. `contains_local_changes` is
+  `true` if the package directory has uncommitted changes, untracked files included. A change
+  elsewhere in the repository does not set it.
+- A git install (`pip install git+https://...`) records the commit that pip installed, with
+  `contains_local_changes` set to `false`.
+- Any other install, such as a wheel or `pip install ./packages/timenet`, reads the
+  `_build_info.json` file in the package. `scripts/build_dists.py` writes that file while it
+  builds the wheel, and records the state of the checkout at that time. `make build` and the
+  publish workflow run the script.
+
+If the commit of an installed package is unknown, the build fails and names the reason. For
+example, git is not on `PATH`, git fails in the checkout, or a plain `uv build` produced the wheel,
+so the package has no `_build_info.json`.
 
 The values locator is backend-neutral. One schema covers both scalar and multidimensional specs. A
 multidimensional spec records its shape in `value_shape` and `dimension_names`. It does not need a

packages/conftest.py

@@ -0,0 +1,21 @@
+"""Shared pytest configuration for every workspace package's tests.
+
+Each manifest write reads the commit of the TimeNet distributions. For an editable install this
+asks git, so the result would depend on the state of this checkout, and the calls cost seconds
+across the suite. So every installed distribution reports a clean commit, whatever its install
+type, unless the test carries the ``real_provenance`` marker.
+"""
+
+import pytest
+
+from timenet.manifest import GitSource
+from timenet.provenance import InstalledDistribution
+
+
+_CLEAN = GitSource(sha="0" * 40, contains_local_changes=False)
+
+
+@pytest.fixture(autouse=True)
+def _stub_installed_sources(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None:
+    if request.node.get_closest_marker("real_provenance") is None:
+        monkeypatch.setattr(InstalledDistribution, "read_git_source", lambda _self: _CLEAN)

packages/timenet-connectors/src/timenet_connectors/builder/env.py

@@ -18,23 +18,16 @@
 import subprocess  # ruff: ignore[suspicious-subprocess-import] - the command is a fixed argument vector
 import sys
 import tempfile
-from urllib.parse import urlparse
-from urllib.request import url2pathname
 
 from uv import find_uv_bin
 
 from timenet.errors import TimeNetBuildError, TimeNetIsolationError
 from timenet.manifest import PayloadSource
+from timenet.provenance import InstalledDistribution, TimeNetSnapshot
 from timenet.types.references import DatasetRef
 from timenet_connectors.discovery import requirements_for
 
 
-BASE_DISTRIBUTIONS = ("timenet", "timenet-connectors")
-"""Pinned to the parent's own versions, so the code that writes the TimeF bytes is the code that
-reads them back. ``timenet-connectors`` depends on ``timenet[build]``, so the card-YAML
-dependencies arrive transitively and the extra never needs naming here."""
-
-
 @dataclass(frozen=True)
 class EnvSpec:
     """What one connector's environment needs."""
@@ -206,44 +199,19 @@ def _base_args() -> tuple[str, ...]:
         TimeNetIsolationError: If a base distribution is not installed in this interpreter.
     """
     args: list[str] = []
-    for name in BASE_DISTRIBUTIONS:
+    # Pin TimeNet to the parent's own versions, so the code that writes the TimeF bytes is the code
+    # that reads them back. timenet-connectors depends on timenet[build], so the card-YAML
+    # dependencies arrive transitively and the extra never needs naming here.
+    for name in TimeNetSnapshot.package_names():
         try:
-            dist = Distribution.from_name(name)
+            installed = InstalledDistribution.from_name(name)
         except PackageNotFoundError as exc:
             raise TimeNetIsolationError(
                 f"cannot make a build environment: {name} is not installed in {sys.executable}."
             ) from exc
-        source = _local_source_path(dist)
+        source = installed.path_to_ondisk_project_directory()
         if source is not None:
             args += ["--with-editable", str(source)]
         else:
-            args += ["--with", f"{name}=={dist.version}"]
+            args += ["--with", f"{name}=={installed.version}"]
     return tuple(args)
-
-
-def _local_source_path(dist: Distribution) -> Path | None:
-    """Return the on-disk project directory an editable or local path install points at.
-
-    ``direct_url.json`` records how a distribution was installed. An editable install carries
-    ``dir_info.editable``. A plain path install (``pip install ./packages/timenet``) records a
-    ``file://`` URL to the same source tree without that flag. Both let the child build against the
-    parent's on-disk source, so both return the directory. An index or wheel install has no
-    ``file://`` directory and returns ``None``, leaving the caller to pin the exact version.
-
-    Args:
-        dist: The installed distribution.
-
-    Returns:
-        The source directory for an editable or local path install, else ``None``.
-    """
-    raw = dist.read_text("direct_url.json")
-    if raw is None:
-        return None
-    info = json.loads(raw)
-    parsed = urlparse(info.get("url", ""))
-    if parsed.scheme != "file":
-        return None
-    path = Path(url2pathname(parsed.path))
-    if info.get("dir_info", {}).get("editable") or path.is_dir():
-        return path
-    return None

packages/timenet-connectors/tests/test_build_env.py

@@ -9,6 +9,7 @@
 from timenet.errors import TimeNetBuildError
 from timenet.manifest import PayloadSource
 from timenet.manifest.files import FilePart
+from timenet.provenance import TimeNetSnapshot
 from timenet.types.references import DatasetRef
 from timenet_connectors.builder import env as env_module
 from timenet_connectors.builder.env import EnvSpec, env_spec, run_isolated_build, uv_command
@@ -33,7 +34,7 @@ def read_text(self, name):
 
 
 def _base_args_for(monkeypatch, dist):
-    monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet",))
+    monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet",)))
     monkeypatch.setattr(env_module.Distribution, "from_name", lambda name: dist)
     return env_module._base_args()
 
@@ -149,7 +150,7 @@ def test_run_build_captures_stderr_and_the_exit_code():
 
 
 def test_env_spec_reports_a_base_distribution_that_is_not_installed(monkeypatch):
-    monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet", "timenet-not-installed"))
+    monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet", "timenet-not-installed")))
 
     with pytest.raises(TimeNetBuildError, match="timenet-not-installed"):
         env_spec("timenet/hello-world")

packages/timenet-connectors/tests/test_build_plan.py

@@ -28,6 +28,7 @@
     TimeNetMissingParentError,
     TimeNetVersionConflictError,
 )
+from timenet.provenance import TimeNetSnapshot
 from timenet.reader import TimeFReader
 from timenet.registry import LocalRegistry, WritableRegistry, open_local_version
 from timenet.registry._paths import release_dir
@@ -893,7 +894,7 @@ def copy_the_saved_release(release, store_root, *, force=False, **_):
 def test_an_environment_that_cannot_be_made_names_the_build_in_this_interpreter(tmp_path, monkeypatch, entry, hint):
     cards = _install_cards(monkeypatch, parent={})
     monkeypatch.setattr(cli_module, "known_ids", lambda: list(cards))
-    monkeypatch.setattr(env_module, "BASE_DISTRIBUTIONS", ("timenet-not-installed",))
+    monkeypatch.setattr(TimeNetSnapshot, "package_names", classmethod(lambda cls: ("timenet-not-installed",)))
     local_registry = tmp_path / "store"
 
     if entry == "cli":

packages/timenet-connectors/tests/test_end_to_end.py

@@ -6,7 +6,7 @@
 
 from timenet.client import TimeNet
 from timenet.config import settings
-from timenet.provenance import build_env
+from timenet.manifest import BuildEnvironment
 from timenet.registry import LocalRegistry
 from timenet.sources import InstalledConnectors
 from timenet.testing import assert_datasets_equal
@@ -113,5 +113,5 @@ def test_an_isolated_build_records_its_environment(tmp_path):
     # pytest runs this process and is a runtime dependency of neither package, so it separates a
     # real isolated build from one that quietly inherited the parent's site-packages. Assert both
     # directions: without the first the second would also pass on a machine that simply lacks it.
-    assert "pytest" in build_env()["packages"]
+    assert "pytest" in BuildEnvironment.capture(git={}).packages
     assert "pytest" not in packages

packages/timenet/src/timenet/manifest/__init__.py

@@ -2,16 +2,27 @@
 
 from timenet.manifest.counts import ManifestCounts
 from timenet.manifest.files import FilePart, ManifestFiles
-from timenet.manifest.manifest import BuildEnvironment, Manifest, parse_manifest, read_manifest
+from timenet.manifest.manifest import (
+    BuildEnvironment,
+    GitSource,
+    GitSourceMap,
+    Manifest,
+    PackageName,
+    parse_manifest,
+    read_manifest,
+)
 from timenet.manifest.payload_source import PayloadSource
 
 
 __all__ = [
     "BuildEnvironment",
     "FilePart",
+    "GitSource",
+    "GitSourceMap",
     "Manifest",
     "ManifestCounts",
     "ManifestFiles",
+    "PackageName",
     "PayloadSource",
     "parse_manifest",
     "read_manifest",