diff --git a/.github/actions/pin-socket-hosts/action.yml b/.github/actions/pin-socket-hosts/action.yml new file mode 100644 index 00000000..c7807042 --- /dev/null +++ b/.github/actions/pin-socket-hosts/action.yml @@ -0,0 +1,42 @@ +name: Pin Socket patch hosts +description: >- + On macOS runners, resolve the production patch hosts once (system resolver, + then DNS-over-HTTPS by IP literal), TLS-verify every address for its host, + and pin them in /etc/hosts for the rest of the job +inputs: + hosts: + description: Space-separated hostnames to pin + default: patch.socket.dev patches-api.socket.dev +runs: + using: composite + steps: + # GitHub's hosted macOS runners intermittently answer patch.socket.dev + # with EAI_NONAME ("[Errno 8] nodename nor servname provided", bun's + # `FailedToOpenSocket`) for minutes at a time — at job start or mid-job — + # while the service is up: the ubuntu / windows legs of the same run pass + # and the same macOS cells pass before and after the window. A pre-flight + # wait cannot cover a mid-job window and the failing processes are the + # real package managers, not the CLI, so the job takes the runner's + # resolver out of the path instead. Every request still goes to the + # production service over TLS verified for the hostname. + # scripts/pin-socket-hosts.py documents the resolution and verification. + - name: Pin hosts + if: runner.os == 'macOS' + shell: bash + env: + PIN_HOSTS: ${{ inputs.hosts }} + run: | + set -euo pipefail + # shellcheck disable=SC2086 # PIN_HOSTS is a space-separated list + lines=$(python3 "$GITHUB_WORKSPACE/scripts/pin-socket-hosts.py" $PIN_HOSTS) + printf '%s\n' "$lines" + printf '\n# pinned by .github/actions/pin-socket-hosts\n%s\n' "$lines" | sudo tee -a /etc/hosts >/dev/null + sudo dscacheutil -flushcache + sudo killall -HUP mDNSResponder || true + for host in $PIN_HOSTS; do + got=$(python3 -c 'import socket, sys; print(" ".join(sorted({i[4][0] for i in socket.getaddrinfo(sys.argv[1], 443)})))' "$host" || true) + echo "$host now resolves to: ${got:-nothing}" + if [ -z "$got" ] || ! grep -qE "^(${got// /|}) $host\$" <<<"$lines"; then + echo "::warning::$host does not resolve to its pinned address after pinning (got: ${got:-nothing})" + fi + done diff --git a/.github/workflows/bun-compatibility.yml b/.github/workflows/bun-compatibility.yml index 3f6b325f..9894fe3b 100644 --- a/.github/workflows/bun-compatibility.yml +++ b/.github/workflows/bun-compatibility.yml @@ -17,6 +17,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/bun-compatibility.yml' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' @@ -57,6 +59,8 @@ on: branches: [main] paths: - '.github/workflows/bun-compatibility.yml' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'scripts/backtest-bun*.py' - 'scripts/probe-bun-historical-linux.py' - 'scripts/bun-historical-shas.json' @@ -187,6 +191,11 @@ jobs: with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts + - name: Download Bun ${{ matrix.bun }} id: bun # Pre-populate the exact directory layout the script's install_tool() diff --git a/.github/workflows/poetry-compatibility.yml b/.github/workflows/poetry-compatibility.yml index c4b07934..585e8378 100644 --- a/.github/workflows/poetry-compatibility.yml +++ b/.github/workflows/poetry-compatibility.yml @@ -15,6 +15,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/poetry-compatibility.yml' - 'scripts/backtest-poetry.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' @@ -29,6 +31,8 @@ on: branches: [main] paths: - 'scripts/backtest-poetry.py' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - 'crates/socket-patch-core/src/utils/poetry_lock.rs' - 'crates/socket-patch-core/src/patch/redirect/**' - 'crates/socket-patch-core/src/vendor/pypi*.rs' @@ -91,6 +95,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts # uv bootstraps every pinned Poetry release (and its interpreter) # itself; pinning uv keeps the bootstrap reproducible. - run: python -m pip install uv==0.11.19 diff --git a/.github/workflows/vlt-compatibility.yml b/.github/workflows/vlt-compatibility.yml index 284164b7..4eb555a9 100644 --- a/.github/workflows/vlt-compatibility.yml +++ b/.github/workflows/vlt-compatibility.yml @@ -21,6 +21,8 @@ on: pull_request: paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -58,6 +60,8 @@ on: branches: [main] paths: - '.github/actions/upload-artifact/**' + - '.github/actions/pin-socket-hosts/**' + - 'scripts/pin-socket-hosts.py' - '.github/workflows/vlt-compatibility.yml' - 'Cargo.lock' - 'rust-toolchain.toml' @@ -390,6 +394,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' + - name: Pin the production patch hosts (macOS) + # The hosted macOS resolver intermittently loses patch.socket.dev for + # minutes (EAI_NONAME) while the service is up; see the action. + uses: ./.github/actions/pin-socket-hosts - name: Backtest against production # Every hosted cell probes the artifact first; it records # blocked-by-server-encoding only when that probe saw a non-identity diff --git a/docs/testing/bun-compatibility.md b/docs/testing/bun-compatibility.md index 53ad0feb..eea92e67 100644 --- a/docs/testing/bun-compatibility.md +++ b/docs/testing/bun-compatibility.md @@ -345,6 +345,18 @@ matrix to six concurrent jobs, with three cells per job. Each cell has its own temporary directory so historical Bun processes cannot collide while extracting identically named packages. +On macOS the job first runs `.github/actions/pin-socket-hosts` +(`scripts/pin-socket-hosts.py`): the hosted macOS resolver intermittently +answers `patch.socket.dev` with EAI_NONAME for minutes at a time, at job start +or mid-job, while the service is up, which failed every hosted cell in the +window (`FailedToOpenSocket`, `[Errno 8] nodename nor servname provided`, or a +Bun 1.3.x workspace install that never exits). The action resolves the patch +hosts once (the system resolver, then DNS-over-HTTPS by IP literal), keeps only +addresses whose TLS handshake verifies the hostname, and pins them in +`/etc/hosts`, so cells still reach the production service over verified TLS +without depending on the runner's resolver. The vlt and Poetry workflows run +the same step. + **Pinned versions:** 0.8.1, 1.0.0, 1.0.36, 1.1.0, 1.1.38 (binary lock), 1.1.39 (first text lock, version 0), 1.1.43 (first `--lockfile-only`), 1.1.45 (last version-0 writer), 1.2.0, 1.2.23, 1.3.0 (version 1), 1.3.9 / 1.3.10 diff --git a/docs/testing/vlt-compatibility.md b/docs/testing/vlt-compatibility.md index 3129c0d3..1a162d62 100644 --- a/docs/testing/vlt-compatibility.md +++ b/docs/testing/vlt-compatibility.md @@ -74,7 +74,10 @@ asserted and each named test or row exists. releases, the Node floors 22.22.0 / 22.13.0 / 22.7.0 / 22.0.0 with the collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm` cache root); `native` (the backtest against production, artifacts - `vlt-results--` in depscan's capture `result.json` shape); + `vlt-results--` in depscan's capture `result.json` shape; on macOS + it first pins the TLS-verified patch hosts in `/etc/hosts` through + `.github/actions/pin-socket-hosts`, because the hosted macOS resolver + intermittently loses `patch.socket.dev` for minutes while the service is up); `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux, macOS and Windows); `matrix-coverage` (every era × suite × OS). - **Nightly:** `canary` runs every capstone on `vlt@latest` on 3 OS (only the diff --git a/scripts/pin-socket-hosts.py b/scripts/pin-socket-hosts.py new file mode 100644 index 00000000..df36bb1e --- /dev/null +++ b/scripts/pin-socket-hosts.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Pin the production Socket patch hosts in the hosts file of a CI runner. + +The compatibility workflows drive REAL package managers (bun, vlt, poetry, +...) against the production patch service. On GitHub's hosted macOS runners +the system resolver intermittently answers `patch.socket.dev` with +EAI_NONAME ("[Errno 8] nodename nor servname provided, or not known"; +bun: `FailedToOpenSocket`) for minutes at a time, starting at job start or +mid-job, while the service itself is up (the ubuntu and windows legs of the +same run pass, and the same macOS cells pass before and after the window). +The runner's resolver is not under test, so the workflow takes it out of the +path: this script resolves each host once, verifies every address, and +prints `hosts(5)` lines the workflow appends to /etc/hosts. + +Resolution tries the system resolver first, then DNS-over-HTTPS to IP-literal +endpoints (no DNS needed to reach them), retrying with backoff inside a +bounded window. An address is only pinned after a TLS handshake to it with +SNI = the host verifies the host's certificate, so a pinned address is one +that really serves that name; the package managers still verify TLS for the +hostname on every request. Both families are resolved; an IPv6 address is +pinned only when it verifies too, so a runner without an IPv6 route never gets +an unreachable entry. + +Exit status is non-zero, with nothing printed, when a host cannot be pinned +within the window: the job then fails at this step, naming the host, rather +than in a hundred cells downstream. +""" + +import argparse +import ipaddress +import json +import socket +import ssl +import sys +import time +import urllib.request + +DEFAULT_HOSTS = ['patch.socket.dev', 'patches-api.socket.dev'] +# DoH JSON endpoints reached by IP literal; both serve certificates with the +# IP in the subjectAltName, so they verify without any name resolution. +DOH_ENDPOINTS = [ + 'https://1.1.1.1/dns-query?name={host}&type={rrtype}', + 'https://8.8.8.8/resolve?name={host}&type={rrtype}', +] +RR_TYPES = {'A': 1, 'AAAA': 28} + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def system_resolve(host): + infos = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + return [info[4][0] for info in infos] + + +def doh_resolve(host, template, timeout): + addresses = [] + for rrtype, code in RR_TYPES.items(): + request = urllib.request.Request(template.format(host=host, rrtype=rrtype), + headers={'accept': 'application/dns-json'}) + with urllib.request.urlopen(request, timeout=timeout) as response: + answer = json.loads(response.read()).get('Answer') or [] + # CNAME answers (type 5) precede the address records and are skipped. + addresses += [record['data'] for record in answer if record.get('type') == code] + return addresses + + +def verified(host, address, timeout): + """A TLS handshake to `address` with SNI `host` verifies `host`'s cert.""" + context = ssl.create_default_context() + try: + with socket.create_connection((address, 443), timeout=timeout) as raw: + with context.wrap_socket(raw, server_hostname=host): + return True + except (OSError, ssl.SSLError) as error: + log(f'{host}: {address} failed verification: {error}') + return False + + +def resolve(host, window, timeout): + """Verified addresses of `host` (IPv4 first), or [] once `window` seconds pass.""" + sources = [('system resolver', lambda: system_resolve(host))] + sources += [(template.split('/')[2], lambda t=template: doh_resolve(host, t, timeout)) + for template in DOH_ENDPOINTS] + deadline = time.monotonic() + window + attempt = 0 + fallback = [] + while True: + attempt += 1 + for name, source in sources: + try: + candidates = source() + except Exception as error: # noqa: BLE001 - every source is best effort + log(f'{host}: {name} attempt {attempt} failed: {error}') + continue + addresses = [] + for candidate in dict.fromkeys(candidates): + try: + ipaddress.ip_address(candidate) + except ValueError: + continue + if verified(host, candidate, timeout): + addresses.append(candidate) + addresses.sort(key=lambda a: ipaddress.ip_address(a).version) + # A source that only verified IPv6 is not enough on its own: try + # the next one for an IPv4 address before settling for it. + if any(ipaddress.ip_address(a).version == 4 for a in addresses): + log(f'{host}: pinned {" ".join(addresses)} (from {name}, attempt {attempt})') + return addresses + log(f'{host}: {name} attempt {attempt} gave no verified IPv4 address') + fallback = fallback or addresses + remaining = deadline - time.monotonic() + if remaining <= 0: + return fallback + time.sleep(min(5 * attempt, 30, remaining)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('hosts', nargs='*', default=DEFAULT_HOSTS) + parser.add_argument('--window', type=float, default=300, + help='seconds to keep retrying a host before failing (default 300)') + parser.add_argument('--timeout', type=float, default=10, + help='per-request timeout in seconds (default 10)') + args = parser.parse_args(argv) + lines = [] + for host in args.hosts: + addresses = resolve(host, args.window, args.timeout) + if not addresses: + log(f'::error::could not resolve and verify {host} within {args.window:.0f} s') + return 1 + lines += [f'{address} {host}' for address in addresses] + print('\n'.join(lines)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/tests/test_pin_socket_hosts.py b/scripts/tests/test_pin_socket_hosts.py new file mode 100644 index 00000000..56094000 --- /dev/null +++ b/scripts/tests/test_pin_socket_hosts.py @@ -0,0 +1,76 @@ +"""Hermetic coverage for scripts/pin-socket-hosts.py's resolution fallbacks.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import socket +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location( + 'pin_socket_hosts', Path(__file__).resolve().parents[1] / 'pin-socket-hosts.py') +pin = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pin) + +HOST = 'patch.socket.dev' +EAI_NONAME = socket.gaierror(8, 'nodename nor servname provided, or not known') + + +def run(fn): + with contextlib.redirect_stderr(io.StringIO()): + return fn() + + +class PinSocketHostsTests(unittest.TestCase): + def test_system_resolver_answer_is_pinned_when_it_verifies(self): + with patch.object(pin, 'system_resolve', return_value=['172.66.3.58', '172.66.3.58']), \ + patch.object(pin, 'doh_resolve') as doh, \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['172.66.3.58']) + doh.assert_not_called() + + def test_doh_takes_over_when_the_system_resolver_fails(self): + with patch.object(pin, 'system_resolve', side_effect=EAI_NONAME), \ + patch.object(pin, 'doh_resolve', return_value=['2606:4700:7::32d', '162.159.143.62']), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), + ['162.159.143.62', '2606:4700:7::32d']) + + def test_unverified_addresses_are_never_pinned(self): + with patch.object(pin, 'system_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'doh_resolve', return_value=['140.82.112.3']), \ + patch.object(pin, 'verified', return_value=False): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), []) + + def test_ipv6_only_is_a_last_resort(self): + with patch.object(pin, 'system_resolve', return_value=['2606:4700:7::32d']), \ + patch.object(pin, 'doh_resolve', return_value=[]), \ + patch.object(pin, 'verified', return_value=True): + self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['2606:4700:7::32d']) + + def test_retries_until_the_resolver_recovers(self): + answers = [EAI_NONAME, ['172.66.3.58']] + with patch.object(pin, 'system_resolve', side_effect=answers), \ + patch.object(pin, 'doh_resolve', side_effect=OSError('no route')), \ + patch.object(pin, 'verified', return_value=True), \ + patch.object(pin.time, 'sleep') as sleep: + self.assertEqual(run(lambda: pin.resolve(HOST, 60, 1)), ['172.66.3.58']) + sleep.assert_called_once() + + def test_main_prints_hosts_lines_and_fails_closed(self): + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=['172.66.3.58']), \ + contextlib.redirect_stdout(out): + self.assertEqual(pin.main([HOST]), 0) + self.assertEqual(out.getvalue(), f'172.66.3.58 {HOST}\n') + out = io.StringIO() + with patch.object(pin, 'resolve', return_value=[]), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(pin.main([HOST]), 1) + self.assertEqual(out.getvalue(), '') + + +if __name__ == '__main__': + unittest.main()