From 6f46cb2ce2da44080a5af9e1852fedaec62027a6 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Mon, 28 Sep 2026 18:59:34 +0200 Subject: [PATCH] ci: pin TLS-verified patch hosts on macOS compat legs The Bun, vlt and Poetry compatibility workflows drive real package managers against the production patch service. On GitHub's hosted macOS runners the system resolver intermittently answers patch.socket.dev with EAI_NONAME ("[Errno 8] nodename nor servname provided"; bun: FailedToOpenSocket; Bun 1.3.x workspace installs never exit) for minutes at a time, at job start or mid-job, while the service is up: ubuntu and windows legs of the same run pass, and the same macOS cells pass before and after the window. Over the last 60 Bun runs (69 attempts) 29 macOS native jobs failed this way and no other OS did; the CLI's own API calls in those cells succeeded. A pre-flight wait cannot cover a mid-job window, and the failing processes are bun / vlt / poetry / python rather than the CLI, so a product retry cannot help. The runner's resolver is not under test, so take it out of the path: .github/actions/pin-socket-hosts runs scripts/pin-socket-hosts.py on macOS, which resolves patch.socket.dev and patches-api.socket.dev (system resolver, then DNS-over-HTTPS by IP literal, with bounded backoff), keeps only addresses whose TLS handshake verifies the hostname, and pins them in /etc/hosts. Every cell still hits production over TLS verified for the hostname, so the captures depscan imports stay production captures. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/pin-socket-hosts/action.yml | 42 ++++++ .github/workflows/bun-compatibility.yml | 9 ++ .github/workflows/poetry-compatibility.yml | 8 ++ .github/workflows/vlt-compatibility.yml | 8 ++ docs/testing/bun-compatibility.md | 12 ++ docs/testing/vlt-compatibility.md | 5 +- scripts/pin-socket-hosts.py | 140 ++++++++++++++++++++ scripts/tests/test_pin_socket_hosts.py | 76 +++++++++++ 8 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 .github/actions/pin-socket-hosts/action.yml create mode 100644 scripts/pin-socket-hosts.py create mode 100644 scripts/tests/test_pin_socket_hosts.py diff --git a/.github/actions/pin-socket-hosts/action.yml b/.github/actions/pin-socket-hosts/action.yml new file mode 100644 index 00000000..c7807042 --- /dev/null +++ b/.github/actions/pin-socket-hosts/action.yml @@ -0,0 +1,42 @@ +name: Pin Socket patch hosts +description: >- + On macOS runners, resolve the production patch hosts once (system resolver, + then DNS-over-HTTPS by IP literal), TLS-verify every address for its host, + and pin them in /etc/hosts for the rest of the job +inputs: + hosts: + description: Space-separated hostnames to pin + default: patch.socket.dev patches-api.socket.dev +runs: + using: composite + steps: + # GitHub's hosted macOS runners intermittently answer patch.socket.dev + # with EAI_NONAME ("[Errno 8] nodename nor servname provided", bun's + # `FailedToOpenSocket`) for minutes at a time — at job start or mid-job — + # while the service is up: the ubuntu / windows legs of the same run pass + # and the same macOS cells pass before and after the window. A pre-flight + # wait cannot cover a mid-job window and the failing processes are the + # real package managers, not the CLI, so the job takes the runner's + # resolver out of the path instead. Every request still goes to the + # production service over TLS verified for the hostname. + # scripts/pin-socket-hosts.py documents the resolution and verification. + - name: Pin hosts + if: runner.os == 'macOS' + shell: bash + env: + PIN_HOSTS: ${{ inputs.hosts }} + run: | + set -euo pipefail + # shellcheck disable=SC2086 # PIN_HOSTS is a space-separated list + lines=$(python3 "$GITHUB_WORKSPACE/scripts/pin-socket-hosts.py" $PIN_HOSTS) + printf '%s\n' "$lines" + printf '\n# pinned by .github/actions/pin-socket-hosts\n%s\n' "$lines" | sudo tee -a /etc/hosts >/dev/null + sudo dscacheutil -flushcache + sudo killall -HUP mDNSResponder || true + for host in $PIN_HOSTS; do + got=$(python3 -c 'import socket, sys; print(" ".join(sorted({i[4][0] for i in socket.getaddrinfo(sys.argv[1], 443)})))' "$host" || true) + echo "$host now resolves to: ${got:-nothing}" + if [ -z "$got" ] || ! grep -qE "^(${got// /|}) $host\$" <<<"$lines"; then + echo "::warning::$host does not resolve to its pinned address after pinning (got: ${got:-nothing})" + fi + done diff --git a/.github/workflows/bun-compatibility.yml b/.github/workflows/bun-compatibility.yml index 3f6b325f..9894fe3b 100644 --- a/.github/workflows/bun-compatibility.yml +++ b/.github/workflows/bun-compatibility.yml @@ -17,6 +17,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/bun-compatibility.yml' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' @@ -57,6 +59,8 @@ on: branches: [main] paths: - '.github/workflows/bun-compatibility.yml' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' - 'scripts/bun-historical-shas.json' @@ -187,6 +191,11 @@ jobs: with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts + - name: Download Bun ${{ matrix.bun }} id: bun # Pre-populate the exact directory layout the script's install_tool() diff --git a/.github/workflows/poetry-compatibility.yml b/.github/workflows/poetry-compatibility.yml index c4b07934..585e8378 100644 --- a/.github/workflows/poetry-compatibility.yml +++ b/.github/workflows/poetry-compatibility.yml @@ -15,6 +15,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/poetry-compatibility.yml' - 'scripts/backtest-poetry.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' @@ -29,6 +31,8 @@ on: branches: [main] paths: - 'scripts/backtest-poetry.py' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' - 'crates/socket-patch-core/src/patch/redirect/**' - 'crates/socket-patch-core/src/vendor/pypi*.rs' @@ -91,6 +95,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts # uv bootstraps every pinned Poetry release (and its interpreter) # itself; pinning uv keeps the bootstrap reproducible. - run: python -m pip install uv==0.11.19 diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 284164b7..4eb555a9 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -21,6 +21,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -58,6 +60,8 @@ on: branches: [main] paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -390,6 +394,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts - name: Backtest against production # Every hosted cell probes the artifact first; it records # blocked-by-server-encoding only when that probe saw a non-identity diff --git a/docs/testing/bun-compatibility.md b/docs/testing/bun-compatibility.md index 53ad0feb..eea92e67 100644 --- a/docs/testing/bun-compatibility.md +++ b/docs/testing/bun-compatibility.md @@ -345,6 +345,18 @@ matrix to six concurrent jobs, with three cells per job. Each cell has its own temporary directory so historical Bun processes cannot collide while extracting identically named packages. +On macOS the job first runs `.github/actions/pin-socket-hosts` +(`scripts/pin-socket-hosts.py`): the hosted macOS resolver intermittently +answers `patch.socket.dev` with EAI_NONAME for minutes at a time, at job start +or mid-job, while the service is up, which failed every hosted cell in the +window (`FailedToOpenSocket`, `[Errno 8] nodename nor servname provided`, or a +Bun 1.3.x workspace install that never exits). The action resolves the patch +hosts once (the system resolver, then DNS-over-HTTPS by IP literal), keeps only +addresses whose TLS handshake verifies the hostname, and pins them in +`/etc/hosts`, so cells still reach the production service over verified TLS +without depending on the runner's resolver. The vlt and Poetry workflows run +the same step. + **Pinned versions:** 0.8.1, 1.0.0, 1.0.36, 1.1.0, 1.1.38 (binary lock), 1.1.39 (first text lock, version 0), 1.1.43 (first `--lockfile-only`), 1.1.45 (last version-0 writer), 1.2.0, 1.2.23, 1.3.0 (version 1), 1.3.9 / 1.3.10 diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index 3129c0d3..1a162d62 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -74,7 +74,10 @@ asserted and each named test or row exists. releases, the Node floors 22.22.0 / 22.13.0 / 22.7.0 / 22.0.0 with the collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm` cache root); `native` (the backtest against production, artifacts - `vlt-results--` in depscan's capture `result.json` shape); + `vlt-results--` in depscan's capture `result.json` shape; on macOS + it first pins the TLS-verified patch hosts in `/etc/hosts` through + `.github/actions/pin-socket-hosts`, because the hosted macOS resolver + intermittently loses `patch.socket.dev` for minutes while the service is up); `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux, macOS and Windows); `matrix-coverage` (every era × suite × OS). - **Nightly:** `canary` runs every capstone on `vlt@latest` on 3 OS (only the diff --git a/scripts/pin-socket-hosts.py b/scripts/pin-socket-hosts.py new file mode 100644 index 00000000..df36bb1e --- /dev/null +++ b/scripts/pin-socket-hosts.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Pin the production Socket patch hosts in the hosts file of a CI runner. + +The compatibility workflows drive REAL package managers (bun, vlt, poetry, +...) against the production patch service. On GitHub's hosted macOS runners +the system resolver intermittently answers `patch.socket.dev` with +EAI_NONAME ("[Errno 8] nodename nor servname provided, or not known"; +bun: `FailedToOpenSocket`) for minutes at a time, starting at job start or +mid-job, while the service itself is up (the ubuntu and windows legs of the +same run pass, and the same macOS cells pass before and after the window). +The runner's resolver is not under test, so the workflow takes it out of the +path: this script resolves each host once, verifies every address, and +prints `hosts(5)` lines the workflow appends to /etc/hosts. + +Resolution tries the system resolver first, then DNS-over-HTTPS to IP-literal +endpoints (no DNS needed to reach them), retrying with backoff inside a +bounded window. An address is only pinned after a TLS handshake to it with +SNI = the host verifies the host's certificate, so a pinned address is one +that really serves that name; the package managers still verify TLS for the +hostname on every request. Both families are resolved; an IPv6 address is +pinned only when it verifies too, so a runner without an IPv6 route never gets +an unreachable entry. + +Exit status is non-zero, with nothing printed, when a host cannot be pinned +within the window: the job then fails at this step, naming the host, rather +than in a hundred cells downstream. +""" + +import argparse +import ipaddress +import json +import socket +import ssl +import sys +import time +import urllib.request + +DEFAULT_HOSTS = ['patch.socket.dev', 'patches-api.socket.dev'] +# DoH JSON endpoints reached by IP literal; both serve certificates with the +# IP in the subjectAltName, so they verify without any name resolution. +DOH_ENDPOINTS = [ + 'https://1.1.1.1/dns-query?name={host}&type={rrtype}', + 'https://8.8.8.8/resolve?name={host}&type={rrtype}', +] +RR_TYPES = {'A': 1, 'AAAA': 28} + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def system_resolve(host): + infos = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + return [info[4][0] for info in infos] + + +def doh_resolve(host, template, timeout): + addresses = [] + for rrtype, code in RR_TYPES.items(): + request = urllib.request.Request(template.format(host=host, rrtype=rrtype), + headers={'accept': 'application/dns-json'}) + with urllib.request.urlopen(request, timeout=timeout) as response: + answer = json.loads(response.read()).get('Answer') or [] + # CNAME answers (type 5) precede the address records and are skipped. + addresses += [record['data'] for record in answer if record.get('type') == code] + return addresses + + +def verified(host, address, timeout): + """A TLS handshake to `address` with SNI `host` verifies `host`'s cert.""" + context = ssl.create_default_context() + try: + with socket.create_connection((address, 443), timeout=timeout) as raw: + with context.wrap_socket(raw, server_hostname=host): + return True + except (OSError, ssl.SSLError) as error: + log(f'{host}: {address} failed verification: {error}') + return False + + +def resolve(host, window, timeout): + """Verified addresses of `host` (IPv4 first), or [] once `window` seconds pass.""" + sources = [('system resolver', lambda: system_resolve(host))] + sources += [(template.split('/')[2], lambda t=template: doh_resolve(host, t, timeout)) + for template in DOH_ENDPOINTS] + deadline = time.monotonic() + window + attempt = 0 + fallback = [] + while True: + attempt += 1 + for name, source in sources: + try: + candidates = source() + except Exception as error: # noqa: BLE001 - every source is best effort + log(f'{host}: {name} attempt {attempt} failed: {error}') + continue + addresses = [] + for candidate in dict.fromkeys(candidates): + try: + ipaddress.ip_address(candidate) + except ValueError: + continue + if verified(host, candidate, timeout): + addresses.append(candidate) + addresses.sort(key=lambda a: ipaddress.ip_address(a).version) + # A source that only verified IPv6 is not enough on its own: try + # the next one for an IPv4 address before settling for it. + if any(ipaddress.ip_address(a).version == 4 for a in addresses): + log(f'{host}: pinned {" ".join(addresses)} (from {name}, attempt {attempt})') + return addresses + log(f'{host}: {name} attempt {attempt} gave no verified IPv4 address') + fallback = fallback or addresses + remaining = deadline - time.monotonic() + if remaining <= 0: + return fallback + time.sleep(min(5 * attempt, 30, remaining)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('hosts', nargs='*', default=DEFAULT_HOSTS) + parser.add_argument('--window', type=float, default=300, + help='seconds to keep retrying a host before failing (default 300)') + parser.add_argument('--timeout', type=float, default=10, + help='per-request timeout in seconds (default 10)') + args = parser.parse_args(argv) + lines = [] + for host in args.hosts: + addresses = resolve(host, args.window, args.timeout) + if not addresses: + log(f'::error::could not resolve and verify {host} within {args.window:.0f} s') + return 1 + lines += [f'{address} {host}' for address in addresses] + print('\n'.join(lines)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/tests/test_pin_socket_hosts.py b/scripts/tests/test_pin_socket_hosts.py new file mode 100644 index 00000000..56094000 --- /dev/null +++ b/scripts/tests/test_pin_socket_hosts.py @@ -0,0 +1,76 @@ +"""Hermetic coverage for scripts/pin-socket-hosts.py's resolution fallbacks.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import socket +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location( + 'pin_socket_hosts', Path(__file__).resolve().parents[1] / 'pin-socket-hosts.py') +pin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pin) + +HOST = 'patch.socket.dev' +EAI_NONAME = socket.gaierror(8, 'nodename nor servname provided, or not known') + + +def run(fn): + with contextlib.redirect_stderr(io.StringIO()): + return fn() + + +class PinSocketHostsTests(unittest.TestCase): + def test_system_resolver_answer_is_pinned_when_it_verifies(self): + with patch.object(pin, 'system_resolve', return_value=['172.66.3.58', '172.66.3.58']), \ + patch.object(pin, 'doh_resolve') as doh, \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['172.66.3.58']) + doh.assert_not_called() + + def test_doh_takes_over_when_the_system_resolver_fails(self): + with patch.object(pin, 'system_resolve', side_effect=EAI_NONAME), \ + patch.object(pin, 'doh_resolve', return_value=['2606:4700:7::32d', '162.159.143.62']), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), + ['162.159.143.62', '2606:4700:7::32d']) + + def test_unverified_addresses_are_never_pinned(self): + with patch.object(pin, 'system_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'doh_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'verified', return_value=False): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), []) + + def test_ipv6_only_is_a_last_resort(self): + with patch.object(pin, 'system_resolve', return_value=['2606:4700:7::32d']), \ + patch.object(pin, 'doh_resolve', return_value=[]), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['2606:4700:7::32d']) + + def test_retries_until_the_resolver_recovers(self): + answers = [EAI_NONAME, ['172.66.3.58']] + with patch.object(pin, 'system_resolve', side_effect=answers), \ + patch.object(pin, 'doh_resolve', side_effect=OSError('no route')), \ + patch.object(pin, 'verified', return_value=True), \ + patch.object(pin.time, 'sleep') as sleep: + self.assertEqual(run(lambda: pin.resolve(HOST, 60, 1)), ['172.66.3.58']) + sleep.assert_called_once() + + def test_main_prints_hosts_lines_and_fails_closed(self): + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=['172.66.3.58']), \ + contextlib.redirect_stdout(out): + self.assertEqual(pin.main([HOST]), 0) + self.assertEqual(out.getvalue(), f'172.66.3.58 {HOST}\n') + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=[]), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(pin.main([HOST]), 1) + self.assertEqual(out.getvalue(), '') + + +if __name__ == '__main__': + unittest.main()