ci: pin TLS-verified patch hosts on macOS compat legs (fix hosted DNS flake)

.github/actions/pin-socket-hosts/action.yml

@@ -0,0 +1,42 @@
+name: Pin Socket patch hosts
+description: >-
+  On macOS runners, resolve the production patch hosts once (system resolver,
+  then DNS-over-HTTPS by IP literal), TLS-verify every address for its host,
+  and pin them in /etc/hosts for the rest of the job
+inputs:
+  hosts:
+    description: Space-separated hostnames to pin
+    default: patch.socket.dev patches-api.socket.dev
+runs:
+  using: composite
+  steps:
+    # GitHub's hosted macOS runners intermittently answer patch.socket.dev
+    # with EAI_NONAME ("[Errno 8] nodename nor servname provided", bun's
+    # `FailedToOpenSocket`) for minutes at a time — at job start or mid-job —
+    # while the service is up: the ubuntu / windows legs of the same run pass
+    # and the same macOS cells pass before and after the window. A pre-flight
+    # wait cannot cover a mid-job window and the failing processes are the
+    # real package managers, not the CLI, so the job takes the runner's
+    # resolver out of the path instead. Every request still goes to the
+    # production service over TLS verified for the hostname.
+    # scripts/pin-socket-hosts.py documents the resolution and verification.
+    - name: Pin hosts
+      if: runner.os == 'macOS'
+      shell: bash
+      env:
+        PIN_HOSTS: ${{ inputs.hosts }}
+      run: |
+        set -euo pipefail
+        # shellcheck disable=SC2086 # PIN_HOSTS is a space-separated list
+        lines=$(python3 "$GITHUB_WORKSPACE/scripts/pin-socket-hosts.py" $PIN_HOSTS)
+        printf '%s\n' "$lines"
+        printf '\n# pinned by .github/actions/pin-socket-hosts\n%s\n' "$lines" | sudo tee -a /etc/hosts >/dev/null
+        sudo dscacheutil -flushcache
+        sudo killall -HUP mDNSResponder || true
+        for host in $PIN_HOSTS; do
+          got=$(python3 -c 'import socket, sys; print(" ".join(sorted({i[4][0] for i in socket.getaddrinfo(sys.argv[1], 443)})))' "$host" || true)
+          echo "$host now resolves to: ${got:-nothing}"
+          if [ -z "$got" ] || ! grep -qE "^(${got// /|}) $host\$" <<<"$lines"; then
+            echo "::warning::$host does not resolve to its pinned address after pinning (got: ${got:-nothing})"
+          fi
+        done

.github/workflows/bun-compatibility.yml

@@ -17,6 +17,8 @@ on:
   pull_request:
     paths:
       - '.github/actions/upload-artifact/**'
+      - '.github/actions/pin-socket-hosts/**'
+      - 'scripts/pin-socket-hosts.py'
       - '.github/workflows/bun-compatibility.yml'
       - 'scripts/backtest-bun*.py'
       - 'scripts/probe-bun-historical-linux.py'
@@ -57,6 +59,8 @@ on:
     branches: [main]
     paths:
       - '.github/workflows/bun-compatibility.yml'
+      - '.github/actions/pin-socket-hosts/**'
+      - 'scripts/pin-socket-hosts.py'
       - 'scripts/backtest-bun*.py'
       - 'scripts/probe-bun-historical-linux.py'
       - 'scripts/bun-historical-shas.json'
@@ -187,6 +191,11 @@ jobs:
         with:
           python-version: '3.12'
 
+      - name: Pin the production patch hosts (macOS)
+        # The hosted macOS resolver intermittently loses patch.socket.dev for
+        # minutes (EAI_NONAME) while the service is up; see the action.
+        uses: ./.github/actions/pin-socket-hosts
+
       - name: Download Bun ${{ matrix.bun }}
         id: bun
         # Pre-populate the exact directory layout the script's install_tool()

.github/workflows/poetry-compatibility.yml

@@ -15,6 +15,8 @@ on:
   pull_request:
     paths:
       - '.github/actions/upload-artifact/**'
+      - '.github/actions/pin-socket-hosts/**'
+      - 'scripts/pin-socket-hosts.py'
       - '.github/workflows/poetry-compatibility.yml'
       - 'scripts/backtest-poetry.py'
       - 'crates/socket-patch-core/src/utils/poetry_lock.rs'
@@ -29,6 +31,8 @@ on:
     branches: [main]
     paths:
       - 'scripts/backtest-poetry.py'
+      - '.github/actions/pin-socket-hosts/**'
+      - 'scripts/pin-socket-hosts.py'
       - 'crates/socket-patch-core/src/utils/poetry_lock.rs'
       - 'crates/socket-patch-core/src/patch/redirect/**'
       - 'crates/socket-patch-core/src/vendor/pypi*.rs'
@@ -91,6 +95,10 @@ jobs:
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: '3.12'
+      - name: Pin the production patch hosts (macOS)
+        # The hosted macOS resolver intermittently loses patch.socket.dev for
+        # minutes (EAI_NONAME) while the service is up; see the action.
+        uses: ./.github/actions/pin-socket-hosts
       # uv bootstraps every pinned Poetry release (and its interpreter)
       # itself; pinning uv keeps the bootstrap reproducible.
       - run: python -m pip install uv==0.11.19

.github/workflows/vlt-compatibility.yml

@@ -21,6 +21,8 @@ on:
   pull_request:
     paths:
       - '.github/actions/upload-artifact/**'
+      - '.github/actions/pin-socket-hosts/**'
+      - 'scripts/pin-socket-hosts.py'
       - '.github/workflows/vlt-compatibility.yml'
       - 'Cargo.lock'
       - 'rust-toolchain.toml'
@@ -58,6 +60,8 @@ on:
     branches: [main]
     paths:
       - '.github/actions/upload-artifact/**'
+      - '.github/actions/pin-socket-hosts/**'
+      - 'scripts/pin-socket-hosts.py'
       - '.github/workflows/vlt-compatibility.yml'
       - 'Cargo.lock'
       - 'rust-toolchain.toml'
@@ -390,6 +394,10 @@ jobs:
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: '3.12'
+      - name: Pin the production patch hosts (macOS)
+        # The hosted macOS resolver intermittently loses patch.socket.dev for
+        # minutes (EAI_NONAME) while the service is up; see the action.
+        uses: ./.github/actions/pin-socket-hosts
       - name: Backtest against production
         # Every hosted cell probes the artifact first; it records
         # blocked-by-server-encoding only when that probe saw a non-identity

docs/testing/bun-compatibility.md

@@ -345,6 +345,18 @@ matrix to six concurrent jobs, with three cells per job. Each cell has its own
 temporary directory so historical Bun processes cannot collide while extracting
 identically named packages.
 
+On macOS the job first runs `.github/actions/pin-socket-hosts`
+(`scripts/pin-socket-hosts.py`): the hosted macOS resolver intermittently
+answers `patch.socket.dev` with EAI_NONAME for minutes at a time, at job start
+or mid-job, while the service is up, which failed every hosted cell in the
+window (`FailedToOpenSocket`, `[Errno 8] nodename nor servname provided`, or a
+Bun 1.3.x workspace install that never exits). The action resolves the patch
+hosts once (the system resolver, then DNS-over-HTTPS by IP literal), keeps only
+addresses whose TLS handshake verifies the hostname, and pins them in
+`/etc/hosts`, so cells still reach the production service over verified TLS
+without depending on the runner's resolver. The vlt and Poetry workflows run
+the same step.
+
 **Pinned versions:** 0.8.1, 1.0.0, 1.0.36, 1.1.0, 1.1.38 (binary lock),
 1.1.39 (first text lock, version 0), 1.1.43 (first `--lockfile-only`), 1.1.45
 (last version-0 writer), 1.2.0, 1.2.23, 1.3.0 (version 1), 1.3.9 / 1.3.10

docs/testing/vlt-compatibility.md

@@ -74,7 +74,10 @@ asserted and each named test or row exists.
   releases, the Node floors 22.22.0 / 22.13.0 / 22.7.0 / 22.0.0 with the
   collation golden, and the store linkers auto / hardlink / copy / unpack / a `/dev/shm`
   cache root); `native` (the backtest against production, artifacts
-  `vlt-results-<os>-<vlt>` in depscan's capture `result.json` shape);
+  `vlt-results-<os>-<vlt>` in depscan's capture `result.json` shape; on macOS
+  it first pins the TLS-verified patch hosts in `/etc/hosts` through
+  `.github/actions/pin-socket-hosts`, because the hosted macOS resolver
+  intermittently loses `patch.socket.dev` for minutes while the service is up);
   `lock-diff` (the same cell's `vlt-lock.json` must be byte-identical on Linux,
   macOS and Windows); `matrix-coverage` (every era × suite × OS).
 - **Nightly:** `canary` runs every capstone on `vlt@latest` on 3 OS (only the

scripts/pin-socket-hosts.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Pin the production Socket patch hosts in the hosts file of a CI runner.
+
+The compatibility workflows drive REAL package managers (bun, vlt, poetry,
+...) against the production patch service. On GitHub's hosted macOS runners
+the system resolver intermittently answers `patch.socket.dev` with
+EAI_NONAME ("[Errno 8] nodename nor servname provided, or not known";
+bun: `FailedToOpenSocket`) for minutes at a time, starting at job start or
+mid-job, while the service itself is up (the ubuntu and windows legs of the
+same run pass, and the same macOS cells pass before and after the window).
+The runner's resolver is not under test, so the workflow takes it out of the
+path: this script resolves each host once, verifies every address, and
+prints `hosts(5)` lines the workflow appends to /etc/hosts.
+
+Resolution tries the system resolver first, then DNS-over-HTTPS to IP-literal
+endpoints (no DNS needed to reach them), retrying with backoff inside a
+bounded window. An address is only pinned after a TLS handshake to it with
+SNI = the host verifies the host's certificate, so a pinned address is one
+that really serves that name; the package managers still verify TLS for the
+hostname on every request. Both families are resolved; an IPv6 address is
+pinned only when it verifies too, so a runner without an IPv6 route never gets
+an unreachable entry.
+
+Exit status is non-zero, with nothing printed, when a host cannot be pinned
+within the window: the job then fails at this step, naming the host, rather
+than in a hundred cells downstream.
+"""
+
+import argparse
+import ipaddress
+import json
+import socket
+import ssl
+import sys
+import time
+import urllib.request
+
+DEFAULT_HOSTS = ['patch.socket.dev', 'patches-api.socket.dev']
+# DoH JSON endpoints reached by IP literal; both serve certificates with the
+# IP in the subjectAltName, so they verify without any name resolution.
+DOH_ENDPOINTS = [
+    'https://1.1.1.1/dns-query?name={host}&type={rrtype}',
+    'https://8.8.8.8/resolve?name={host}&type={rrtype}',
+]
+RR_TYPES = {'A': 1, 'AAAA': 28}
+
+
+def log(message):
+    print(message, file=sys.stderr, flush=True)
+
+
+def system_resolve(host):
+    infos = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM)
+    return [info[4][0] for info in infos]
+
+
+def doh_resolve(host, template, timeout):
+    addresses = []
+    for rrtype, code in RR_TYPES.items():
+        request = urllib.request.Request(template.format(host=host, rrtype=rrtype),
+                                         headers={'accept': 'application/dns-json'})
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            answer = json.loads(response.read()).get('Answer') or []
+        # CNAME answers (type 5) precede the address records and are skipped.
+        addresses += [record['data'] for record in answer if record.get('type') == code]
+    return addresses
+
+
+def verified(host, address, timeout):
+    """A TLS handshake to `address` with SNI `host` verifies `host`'s cert."""
+    context = ssl.create_default_context()
+    try:
+        with socket.create_connection((address, 443), timeout=timeout) as raw:
+            with context.wrap_socket(raw, server_hostname=host):
+                return True
+    except (OSError, ssl.SSLError) as error:
+        log(f'{host}: {address} failed verification: {error}')
+        return False
+
+
+def resolve(host, window, timeout):
+    """Verified addresses of `host` (IPv4 first), or [] once `window` seconds pass."""
+    sources = [('system resolver', lambda: system_resolve(host))]
+    sources += [(template.split('/')[2], lambda t=template: doh_resolve(host, t, timeout))
+                for template in DOH_ENDPOINTS]
+    deadline = time.monotonic() + window
+    attempt = 0
+    fallback = []
+    while True:
+        attempt += 1
+        for name, source in sources:
+            try:
+                candidates = source()
+            except Exception as error:  # noqa: BLE001 - every source is best effort
+                log(f'{host}: {name} attempt {attempt} failed: {error}')
+                continue
+            addresses = []
+            for candidate in dict.fromkeys(candidates):
+                try:
+                    ipaddress.ip_address(candidate)
+                except ValueError:
+                    continue
+                if verified(host, candidate, timeout):
+                    addresses.append(candidate)
+            addresses.sort(key=lambda a: ipaddress.ip_address(a).version)
+            # A source that only verified IPv6 is not enough on its own: try
+            # the next one for an IPv4 address before settling for it.
+            if any(ipaddress.ip_address(a).version == 4 for a in addresses):
+                log(f'{host}: pinned {" ".join(addresses)} (from {name}, attempt {attempt})')
+                return addresses
+            log(f'{host}: {name} attempt {attempt} gave no verified IPv4 address')
+            fallback = fallback or addresses
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            return fallback
+        time.sleep(min(5 * attempt, 30, remaining))
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('hosts', nargs='*', default=DEFAULT_HOSTS)
+    parser.add_argument('--window', type=float, default=300,
+                        help='seconds to keep retrying a host before failing (default 300)')
+    parser.add_argument('--timeout', type=float, default=10,
+                        help='per-request timeout in seconds (default 10)')
+    args = parser.parse_args(argv)
+    lines = []
+    for host in args.hosts:
+        addresses = resolve(host, args.window, args.timeout)
+        if not addresses:
+            log(f'::error::could not resolve and verify {host} within {args.window:.0f} s')
+            return 1
+        lines += [f'{address} {host}' for address in addresses]
+    print('\n'.join(lines))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/tests/test_pin_socket_hosts.py

@@ -0,0 +1,76 @@
+"""Hermetic coverage for scripts/pin-socket-hosts.py's resolution fallbacks."""
+
+import contextlib
+import importlib.util
+import io
+from pathlib import Path
+import socket
+import unittest
+from unittest.mock import patch
+
+
+spec = importlib.util.spec_from_file_location(
+    'pin_socket_hosts', Path(__file__).resolve().parents[1] / 'pin-socket-hosts.py')
+pin = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(pin)
+
+HOST = 'patch.socket.dev'
+EAI_NONAME = socket.gaierror(8, 'nodename nor servname provided, or not known')
+
+
+def run(fn):
+    with contextlib.redirect_stderr(io.StringIO()):
+        return fn()
+
+
+class PinSocketHostsTests(unittest.TestCase):
+    def test_system_resolver_answer_is_pinned_when_it_verifies(self):
+        with patch.object(pin, 'system_resolve', return_value=['172.66.3.58', '172.66.3.58']), \
+                patch.object(pin, 'doh_resolve') as doh, \
+                patch.object(pin, 'verified', return_value=True):
+            self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['172.66.3.58'])
+        doh.assert_not_called()
+
+    def test_doh_takes_over_when_the_system_resolver_fails(self):
+        with patch.object(pin, 'system_resolve', side_effect=EAI_NONAME), \
+                patch.object(pin, 'doh_resolve', return_value=['2606:4700:7::32d', '162.159.143.62']), \
+                patch.object(pin, 'verified', return_value=True):
+            self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)),
+                             ['162.159.143.62', '2606:4700:7::32d'])
+
+    def test_unverified_addresses_are_never_pinned(self):
+        with patch.object(pin, 'system_resolve', return_value=['140.82.112.3']), \
+                patch.object(pin, 'doh_resolve', return_value=['140.82.112.3']), \
+                patch.object(pin, 'verified', return_value=False):
+            self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), [])
+
+    def test_ipv6_only_is_a_last_resort(self):
+        with patch.object(pin, 'system_resolve', return_value=['2606:4700:7::32d']), \
+                patch.object(pin, 'doh_resolve', return_value=[]), \
+                patch.object(pin, 'verified', return_value=True):
+            self.assertEqual(run(lambda: pin.resolve(HOST, 0, 1)), ['2606:4700:7::32d'])
+
+    def test_retries_until_the_resolver_recovers(self):
+        answers = [EAI_NONAME, ['172.66.3.58']]
+        with patch.object(pin, 'system_resolve', side_effect=answers), \
+                patch.object(pin, 'doh_resolve', side_effect=OSError('no route')), \
+                patch.object(pin, 'verified', return_value=True), \
+                patch.object(pin.time, 'sleep') as sleep:
+            self.assertEqual(run(lambda: pin.resolve(HOST, 60, 1)), ['172.66.3.58'])
+        sleep.assert_called_once()
+
+    def test_main_prints_hosts_lines_and_fails_closed(self):
+        out = io.StringIO()
+        with patch.object(pin, 'resolve', return_value=['172.66.3.58']), \
+                contextlib.redirect_stdout(out):
+            self.assertEqual(pin.main([HOST]), 0)
+        self.assertEqual(out.getvalue(), f'172.66.3.58 {HOST}\n')
+        out = io.StringIO()
+        with patch.object(pin, 'resolve', return_value=[]), \
+                contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
+            self.assertEqual(pin.main([HOST]), 1)
+        self.assertEqual(out.getvalue(), '')
+
+
+if __name__ == '__main__':
+    unittest.main()