feat: scaffold OpenTofu and pull-request file synchronization

.github/scripts/discover.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# /.github/scripts/discover.py
+
+"""Discover existing GitHub objects; write local OpenTofu and file-sync inputs."""
+
+import argparse
+import json
+import os
+from pathlib import Path, PurePosixPath
+import re
+import subprocess
+import sys
+import tempfile
+
+
+ORGANIZATION = "pheaz"
+SOURCE_REPOSITORY = f"{ORGANIZATION}/.github"
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class DiscoveryError(Exception):
+    """A failed or ambiguous read that must prevent configuration generation."""
+
+
+class GitHub:
+    def read(self, route, *, paginate=False, missing_ok=False):
+        arguments = ["gh", "api", "--method", "GET"]
+        if paginate:
+            arguments.extend(["--paginate", "--slurp"])
+        result = subprocess.run(
+            [*arguments, route], capture_output=True, text=True, check=False
+        )
+        if result.returncode != 0:
+            # Distinguish a missing path from authentication/rate-limit failures.
+            status = re.search(r"\(HTTP (\d{3})\)", result.stderr)
+            if missing_ok and status and status.group(1) == "404":
+                return None
+            raise DiscoveryError(f"GitHub read failed for {route}: {result.stderr.strip()}")
+        try:
+            return json.loads(result.stdout)
+        except json.JSONDecodeError as error:
+            raise DiscoveryError(f"Invalid GitHub JSON for {route}") from error
+
+
+def load_files(root):
+    config = json.loads((root / ".github/sync-files.json").read_text(encoding="utf-8"))
+    if not isinstance(config, dict) or set(config) != {"agents", "workflows"}:
+        raise DiscoveryError("File policy must contain exactly agents and workflows groups")
+    for group, files in config.items():
+        if not isinstance(files, list) or not files:
+            raise DiscoveryError(f"File policy group {group} must be a nonempty list")
+        for file in files:
+            if not isinstance(file, dict) or set(file) != {"source", "dest"}:
+                raise DiscoveryError("File policy entries require only source and dest")
+            for value in file.values():
+                if not isinstance(value, str) or not value:
+                    raise DiscoveryError("File paths must be nonempty strings")
+                path = PurePosixPath(value)
+                if path.is_absolute() or ".." in path.parts:
+                    raise DiscoveryError(f"File path must remain inside its repository: {value}")
+            if not (root / file["source"]).is_file():
+                raise DiscoveryError(f"Canonical source does not exist: {file['source']}")
+    return config
+
+
+def mask_private_repositories(repositories):
+    if os.environ.get("GITHUB_ACTIONS") != "true":
+        return
+    # Public source-repository logs must not disclose private target names.
+    for repository in repositories:
+        if repository["visibility"] != "public":
+            for value in (repository["full_name"], repository["name"]):
+                print(f"::add-mask::{value}", flush=True)
+
+
+def discover(github, files, sync_repositories=""):
+    pages = github.read(
+        f"orgs/{ORGANIZATION}/repos?type=all&per_page=100", paginate=True
+    )
+    if not isinstance(pages, list) or not all(isinstance(page, list) for page in pages):
+        raise DiscoveryError("Repository discovery did not return paginated lists")
+    repositories = [repository for page in pages for repository in page]
+    if not repositories:
+        raise DiscoveryError("Empty organization inventory; refusing to generate configuration")
+    seen = set()
+    for repository in repositories:
+        if not isinstance(repository, dict):
+            raise DiscoveryError("Invalid repository inventory entry")
+        name = repository.get("name")
+        if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,100}", name):
+            raise DiscoveryError("Invalid repository name in inventory")
+        if repository.get("full_name") != f"{ORGANIZATION}/{name}" or name in seen:
+            raise DiscoveryError("Duplicate repository or unexpected organization in inventory")
+        seen.add(name)
+        if type(repository.get("id")) is not int or repository["id"] <= 0:
+            raise DiscoveryError("Repository ID must be a positive integer")
+        if any(type(repository.get(key)) is not bool for key in ("fork", "archived")):
+            raise DiscoveryError("Repository fork/archive flags must be booleans")
+        if repository.get("visibility") not in {"public", "private", "internal"}:
+            raise DiscoveryError("Repository visibility is missing or invalid")
+        branch = repository.get("default_branch")
+        if branch is not None and not isinstance(branch, str):
+            raise DiscoveryError("Invalid default branch in repository inventory")
+    if SOURCE_REPOSITORY not in {repository["full_name"] for repository in repositories}:
+        raise DiscoveryError("Policy repository is missing; verify organization read access")
+    mask_private_repositories(repositories)
+
+    inventory = {}
+    sync = {}
+    requested = {name.strip() for name in sync_repositories.split(",") if name.strip()}
+    eligible = set()
+    for repository in sorted(repositories, key=lambda item: item["name"]):
+        if repository["fork"]:
+            continue
+        name = repository["name"]
+        full_name = repository["full_name"]
+        inventory[name] = {"id": repository["id"], "archived": repository["archived"]}
+        if repository["archived"] or not repository.get("default_branch"):
+            continue
+        if full_name == SOURCE_REPOSITORY:
+            continue
+        eligible.add(full_name)
+        if requested and full_name not in requested:
+            continue
+        # Do not pin a branch name: repo-files-sync resolves the current default.
+        mappings = list(files["workflows"])
+        directory = github.read(f"repos/{full_name}/contents/.github", missing_ok=True)
+        if isinstance(directory, list):
+            mappings = [*files["agents"], *mappings]
+        elif directory is not None and not isinstance(directory, dict):
+            raise DiscoveryError(f"Unexpected .github response for {full_name}")
+        sync[full_name] = mappings
+    if not inventory:
+        raise DiscoveryError("No non-fork repositories found")
+    if requested - eligible:
+        raise DiscoveryError("Requested sync repositories are not eligible organization targets")
+    return {"repositories": inventory}, sync
+
+
+def write_if_changed(path, content):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.is_file() and path.read_text(encoding="utf-8") == content:
+        path.chmod(0o600)
+        return
+    with tempfile.NamedTemporaryFile(
+        mode="w", encoding="utf-8", dir=path.parent, delete=False
+    ) as temporary:
+        temporary.write(content)
+        temporary_path = Path(temporary.name)
+    try:
+        temporary_path.replace(path)
+    finally:
+        temporary_path.unlink(missing_ok=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--sync-repositories", default="", help="Comma-separated owner/name pilot targets; empty means all"
+    )
+    parser.add_argument("--github-output", type=Path, help="GitHub Actions step output file")
+    arguments = parser.parse_args()
+    try:
+        inventory, sync = discover(GitHub(), load_files(ROOT), arguments.sync_repositories)
+        write_if_changed(
+            ROOT / "tofu/repositories.auto.tfvars.json", json.dumps(inventory, indent=2) + "\n"
+        )
+        # JSON is valid YAML and avoids a runtime YAML package dependency.
+        write_if_changed(
+            ROOT / ".github/sync.yml",
+            "# /.github/sync.yml\n# Generated by discover.py; MUST NOT be committed.\n"
+            + json.dumps(sync, indent=2) + "\n",
+        )
+        if arguments.github_output:
+            names = sorted({".github", *(name.split("/", 1)[1] for name in sync)})
+            with arguments.github_output.open("a", encoding="utf-8") as output:
+                output.write(f"has_targets={'true' if sync else 'false'}\n")
+                output.write(f"token_repositories={','.join(names)}\n")
+        print(
+            f"Discovered {len(inventory['repositories'])} non-fork repositories; "
+            f"generated {len(sync)} file-sync targets."
+        )
+        return 0
+    except (DiscoveryError, OSError, ValueError) as error:
+        print(str(error), file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/sync-files.json

@@ -0,0 +1,22 @@
+{
+  "agents": [
+    {
+      "source": ".github/AGENTS.md",
+      "dest": ".github/AGENTS.md"
+    },
+    {
+      "source": ".github/actions/AGENTS.md",
+      "dest": ".github/actions/AGENTS.md"
+    },
+    {
+      "source": ".github/workflows/AGENTS.md",
+      "dest": ".github/workflows/AGENTS.md"
+    }
+  ],
+  "workflows": [
+    {
+      "source": ".github/workflows/lint-pr.yml",
+      "dest": ".github/workflows/lint-pr.yml"
+    }
+  ]
+}

.github/tests/test_discover.py

@@ -0,0 +1,148 @@
+# /.github/tests/test_discover.py
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("discovery", ROOT / ".github/scripts/discover.py")
+discovery = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(discovery)
+
+
+def repository(name, *, fork=False, archived=False, branch="main", visibility="public"):
+    return {
+        "name": name,
+        "full_name": f"pheaz/{name}",
+        "id": 100 + sum(map(ord, name)),
+        "fork": fork,
+        "archived": archived,
+        "default_branch": branch,
+        "visibility": visibility,
+    }
+
+
+class FakeGitHub:
+    def __init__(self, pages, directories=None):
+        self.pages = pages
+        self.directories = directories or {}
+        self.calls = []
+
+    def read(self, route, *, paginate=False, missing_ok=False):
+        self.calls.append((route, paginate, missing_ok))
+        if route.startswith("orgs/"):
+            return self.pages
+        response = self.directories[route.split("/")[2]]
+        if isinstance(response, Exception):
+            raise response
+        return response
+
+
+class DiscoveryTests(unittest.TestCase):
+    def setUp(self):
+        self.files = discovery.load_files(ROOT)
+        self.source = repository(".github")
+
+    def test_preserves_different_sync_scopes_across_pages(self):
+        github = FakeGitHub(
+            [
+                [self.source, repository("existing"), repository("fork", fork=True)],
+                [repository("missing"), repository("file"), repository("archived", archived=True), repository("empty", branch=None)],
+            ],
+            {"existing": [], "missing": None, "file": {"type": "file"}},
+        )
+        inventory, sync = discovery.discover(github, self.files)
+        self.assertEqual(set(inventory["repositories"]), {".github", "existing", "missing", "file", "archived", "empty"})
+        self.assertTrue(inventory["repositories"]["archived"]["archived"])
+        self.assertEqual(set(sync), {"pheaz/existing", "pheaz/missing", "pheaz/file"})
+        self.assertEqual(
+            [file["dest"] for file in sync["pheaz/existing"]],
+            [".github/AGENTS.md", ".github/actions/AGENTS.md", ".github/workflows/AGENTS.md", ".github/workflows/lint-pr.yml"],
+        )
+        self.assertEqual(sync["pheaz/missing"], self.files["workflows"])
+        self.assertEqual(sync["pheaz/file"], self.files["workflows"])
+        self.assertTrue(github.calls[0][1])
+
+    def test_pilot_filters_only_sync_not_iac_inventory(self):
+        github = FakeGitHub([[self.source, repository("first"), repository("second")]], {"first": []})
+        inventory, sync = discovery.discover(github, self.files, "pheaz/first")
+        self.assertEqual(set(inventory["repositories"]), {".github", "first", "second"})
+        self.assertEqual(set(sync), {"pheaz/first"})
+        self.assertEqual(len(github.calls), 2)
+
+    def test_unknown_archived_fork_and_source_pilot_targets_are_rejected(self):
+        pages = [[self.source, repository("archive", archived=True), repository("fork", fork=True)]]
+        for target in ["pheaz/typo", "pheaz/archive", "pheaz/fork", "pheaz/.github"]:
+            with self.subTest(target=target), self.assertRaises(discovery.DiscoveryError):
+                discovery.discover(FakeGitHub(pages), self.files, target)
+
+    def test_permission_error_is_not_treated_as_a_missing_directory(self):
+        github = FakeGitHub([[self.source, repository("restricted")]], {"restricted": discovery.DiscoveryError("HTTP 403")})
+        with self.assertRaisesRegex(discovery.DiscoveryError, "403"):
+            discovery.discover(github, self.files)
+
+    def test_empty_missing_source_duplicate_and_malformed_inventories_fail(self):
+        cases = [[], [[]], [[repository("other")]], [[self.source, self.source]]]
+        for key, value in [("id", True), ("fork", None), ("visibility", None), ("full_name", "other/.github")]:
+            cases.append([[{**self.source, key: value}]])
+        for pages in cases:
+            with self.subTest(pages=pages), self.assertRaises(discovery.DiscoveryError):
+                discovery.discover(FakeGitHub(pages), self.files)
+
+    def test_private_names_are_masked_before_public_action_logs(self):
+        output = io.StringIO()
+        with patch.dict(os.environ, {"GITHUB_ACTIONS": "true"}), contextlib.redirect_stdout(output):
+            discovery.discover(
+                FakeGitHub([[self.source, repository("private-target", visibility="private")]], {"private-target": []}),
+                self.files,
+            )
+        self.assertEqual(output.getvalue(), "::add-mask::pheaz/private-target\n::add-mask::private-target\n")
+
+    def test_output_is_idempotent_and_private(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "nested/sync.yml"
+            discovery.write_if_changed(path, "first\n")
+            timestamp = path.stat().st_mtime_ns
+            discovery.write_if_changed(path, "first\n")
+            self.assertEqual(path.stat().st_mtime_ns, timestamp)
+            self.assertEqual(path.stat().st_mode & 0o777, 0o600)
+            discovery.write_if_changed(path, "second\n")
+            self.assertEqual(path.read_text(), "second\n")
+
+    def test_invalid_source_paths_fail_before_discovery(self):
+        for value in ["../outside", "/outside", "missing"]:
+            with self.subTest(value=value), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                (root / ".github").mkdir()
+                config = {"agents": [{"source": value, "dest": "AGENTS.md"}], "workflows": self.files["workflows"]}
+                (root / ".github/sync-files.json").write_text(json.dumps(config))
+                with self.assertRaises(discovery.DiscoveryError):
+                    discovery.load_files(root)
+
+    def test_cli_reads_use_get_and_slurp_all_pages(self):
+        result = subprocess.CompletedProcess([], 0, stdout="[[]]", stderr="")
+        with patch.object(discovery.subprocess, "run", return_value=result) as run:
+            self.assertEqual(discovery.GitHub().read("orgs/pheaz/repos", paginate=True), [[]])
+        self.assertEqual(run.call_args.args[0], ["gh", "api", "--method", "GET", "--paginate", "--slurp", "orgs/pheaz/repos"])
+
+    def test_cli_only_accepts_404_as_a_missing_path(self):
+        for status in [401, 403, 404, 429, 500]:
+            result = subprocess.CompletedProcess([], 1, stdout="", stderr=f"gh: failed (HTTP {status})")
+            with self.subTest(status=status), patch.object(discovery.subprocess, "run", return_value=result):
+                if status == 404:
+                    self.assertIsNone(discovery.GitHub().read("repos/pheaz/example/contents/.github", missing_ok=True))
+                else:
+                    with self.assertRaises(discovery.DiscoveryError):
+                        discovery.GitHub().read("repos/pheaz/example/contents/.github", missing_ok=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+# /.github/workflows/ci.yml
+
+name: Validate policy automation
+
+on:
+  pull_request:
+    paths:
+      - .github/scripts/discover.py
+      - .github/tests/**
+      - .github/workflows/ci.yml
+  push:
+    branches: [main]
+    paths:
+      - .github/scripts/discover.py
+      - .github/tests/**
+      - .github/workflows/ci.yml
+
+permissions:
+  contents: read
+
+jobs:
+  test-discovery:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Test read-only discovery
+        run: python3 -m unittest discover -s .github/tests -v