From cb516abfc011ac9a06886957afd4a0642328e098 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Oct 2026 19:32:26 -0700 Subject: [PATCH] fix(ingress): check CAA before issuing when SET_CAA is off With SET_CAA=false plain dns-01 never looked at CAA before asking the CA. A CAA record that refuses this account -- one an earlier SET_CAA=true run pinned to a previous account, after the account changed, or the operator's own record naming another CA -- then refused every attempt, each spending one of Let's Encrypt's five failed validations per hostname per hour, and nothing in the container would ever fix it. Check the CAA in force first with dnsguide's existing check_caa (new --caa-only) and fail the pass with a clear message instead. DNS that cannot be read does not block. A fresh account has no URI yet, so an accounturi pin can only be caught from its second pass on. With SET_CAA=true the stale record is the container's own to replace, so that path is unchanged. Signed-off-by: Kevin Wang --- custom-domain/dstack-ingress/README.md | 4 +-- custom-domain/dstack-ingress/TESTING.md | 1 + custom-domain/dstack-ingress/scripts/dns01.sh | 28 +++++++++++++++ .../dstack-ingress/scripts/dnsguide.py | 30 ++++++++++++++++ .../scripts/tests/test_dnsguide.py | 36 +++++++++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index a91dab0d..21bc8063 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -175,7 +175,7 @@ otherwise arbitrary — the gateway answers every name under its domain. |----------|---------|-------------| | `PORT` | `443` | HAProxy listen port | | `ROUTING_MAP` | | Multi-domain routing: `domain=host:port` per line. A wildcard entry (`*.example.com=host:port`) matches one label, and exact entries take precedence over it. Unmatched names go to the first entry. It routes only: every name still needs to be in `DOMAIN`/`DOMAINS` to get a certificate | -| `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS). Replaces existing values with the same tag; for a wildcard whose base is a zone apex this restricts the whole zone (see [Wildcard Domain Support](#wildcard-domain-support)). Except on Cloudflare, a non-wildcard domain is then published as an A record to the gateway's current address instead of a CNAME, because a CNAME would hide the CAA beside it; the address is refreshed every `RENEW_INTERVAL`. The same happens whenever a CAA record already exists at the name, whoever wrote it, so turning `SET_CAA` off later keeps the A record until you delete the CAA. Linode always uses an A record. With `DELEGATION_ZONE` the CAA is always published, whatever this says | +| `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS). Replaces existing values with the same tag; for a wildcard whose base is a zone apex this restricts the whole zone (see [Wildcard Domain Support](#wildcard-domain-support)). Except on Cloudflare, a non-wildcard domain is then published as an A record to the gateway's current address instead of a CNAME, because a CNAME would hide the CAA beside it; the address is refreshed every `RENEW_INTERVAL`. The same happens whenever a CAA record already exists at the name, whoever wrote it, so turning `SET_CAA` off later keeps the A record until you delete the CAA. Linode always uses an A record. With `DELEGATION_ZONE` the CAA is always published, whatever this says. With it `false`, a domain is not sent to the CA while the CAA in force refuses this account (for example one left pinned to an earlier account) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | | `ACME_EMAIL` | | ACME contact address, in either mode. `CERTBOT_EMAIL` is the historical name and still works. See [below](#the-acme-contact-address-is-optional-and-public) — it is optional, and published | | `ACME_STAGING` | `false` | Use Let's Encrypt staging, in either mode. `CERTBOT_STAGING` is the historical name and still works | @@ -188,7 +188,7 @@ otherwise arbitrary — the gateway answers every name under its domain. | `DNS_WEBHOOK_URL` | | `DNS_SETUP_MODE=webhook`: endpoint to notify | | `DNS_WEBHOOK_TOKEN` | | Shared secret; the payload is HMAC-SHA256 signed with it | | `DNS_WEBHOOK_TIMEOUT` | `15` | Seconds per webhook POST attempt | -| `DOH_RESOLVERS` | Google + Cloudflare | Comma-separated DoH endpoints used to verify records. `DOH_RESOLVER` is the historical name and still works | +| `DOH_RESOLVERS` | Google + Cloudflare | Comma-separated DoH endpoints used to verify records, and to check CAA before issuing. `DOH_RESOLVER` is the historical name and still works | | `TLSALPN_ADDRESS` | `127.0.0.1` | Address lego's ACME responder binds to, and haproxy forwards `acme-tls/1` connections to | | `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to | | `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode | diff --git a/custom-domain/dstack-ingress/TESTING.md b/custom-domain/dstack-ingress/TESTING.md index c279cffd..425bcdb9 100644 --- a/custom-domain/dstack-ingress/TESTING.md +++ b/custom-domain/dstack-ingress/TESTING.md @@ -285,6 +285,7 @@ These fail fast and are cheap, so run them on every change: |---|---| | tls-alpn-01 + a wildcard domain | Refused before any ACME call, citing RFC 8737 | | Delegation, CAA in the delegated zone changed to forbid the CA | Blocked before any ACME attempt — proves the published CAA is reachable through the CNAME | +| Plain dns-01, `SET_CAA=false`, a CAA naming another CA or pinned to another account | Blocked before any ACME attempt with `CAA forbids issuing with this account` and the reason quoted back — except the first pass of a fresh account, which has no URI to compare yet | | Delegation with no CAA record at all | Proceeds, reporting `no CAA record set; issuance is unrestricted` — see below | | Delegation, later pass, gateway CNAME broken | Renewal proceeds — that record is for serving, and dns-01 does not use it | | A CAA record with `validationmethods=dns-01`, mode tls-alpn-01 | Blocked with the restriction quoted back | diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index c3e35b5b..e2a206fc 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -214,6 +214,20 @@ is_zone_apex() { >/dev/null } +# Exits 3 when the CAA in force forbids this account to issue for $1. A fresh +# account has no URI yet, so only issuer and method can be checked for it. +check_caa() { + local account_file account_uri="" + if account_file=$(acme_account_file 2>/dev/null); then + account_uri=$(jq -j '.uri' "$account_file") + fi + dnsguide.py --caa-only \ + --domain "$1" \ + --caa-tag "$(caa_tag_for "$1")" \ + --challenge dns-01 \ + --account-uri "$account_uri" +} + set_caa_record() { local domain="$1" @@ -300,6 +314,20 @@ process_domain() { else set_alias_record "$domain" set_txt_record "$domain" + # Without SET_CAA nothing here rewrites CAA, so a record that refuses + # this account -- one an earlier SET_CAA=true run pinned to a previous + # account, or the operator's own -- refuses every attempt, each one of + # Let's Encrypt's five failed validations per hostname per hour. Ask + # DNS before asking the CA. + # Only a definite refusal blocks; a failed check leaves it to the CA. + if [ "$SET_CAA" != "true" ]; then + local caa=0 + check_caa "$domain" || caa=$? + if [ "$caa" -eq 3 ]; then + echo "Error: CAA forbids issuing $domain with this account. Fix or delete that CAA record, or set SET_CAA=true to let this container manage it" >&2 + return 1 + fi + fi fi # The CAA record names our ACME account, so the account has to exist first: diff --git a/custom-domain/dstack-ingress/scripts/dnsguide.py b/custom-domain/dstack-ingress/scripts/dnsguide.py index d1540ff0..518f0526 100644 --- a/custom-domain/dstack-ingress/scripts/dnsguide.py +++ b/custom-domain/dstack-ingress/scripts/dnsguide.py @@ -373,6 +373,28 @@ def verify_once( return results +# Distinct from 1, which is also what an uncaught exception exits with. +CAA_FORBIDS = 3 + + +def caa_only(args: argparse.Namespace) -> int: + """Check once whether CAA lets this account issue for --domain. + + For a caller about to ask the CA: CAA_FORBIDS means the CA would refuse, + so do not spend a failed validation on it. Anything else, DNS that cannot + be read included, leaves the decision to the CA. + """ + domain = args.domain[2:] if args.domain.startswith("*.") else args.domain + try: + ok, why = check_caa(domain, args.caa_tag, args.challenge, args.account_uri, args.resolver) + except ResolveError as exc: + print(f"Warning: could not check CAA for {args.domain}: {exc}", file=sys.stderr, flush=True) + return 0 + if not ok: + print(f"Error: {why}", file=sys.stderr, flush=True) + return 0 if ok else CAA_FORBIDS + + # `delegated` is a shape, not a record kind: it replaces the per-kind records # rather than adding to them. Asking for both would emit two CNAMEs for the same # name -- one at the gateway, one at the delegation zone -- so refuse instead. @@ -513,7 +535,15 @@ def main() -> int: help="comma-separated subset of records to handle " "(cname,txt,caa,challenge-cname,delegated)", ) + parser.add_argument( + "--caa-only", + action="store_true", + help="only check, once, that CAA permits --challenge for --account-uri; " + f"exit {CAA_FORBIDS} if it forbids issuance", + ) args = parser.parse_args() + if args.caa_only: + return caa_only(args) missing = [n for n, v in (("--domain", args.domain), ("--alias-target", args.alias_target), diff --git a/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py index 15cf5582..d586cc46 100644 --- a/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py +++ b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py @@ -4,6 +4,9 @@ Run: python3 scripts/tests/test_dnsguide.py """ +import argparse +import contextlib +import io import os import sys import unittest @@ -163,6 +166,39 @@ def test_issuewild_overrides_issue_for_a_wildcard(self): self.assertFalse(ok) +class TestCaaOnly(unittest.TestCase): + """--caa-only: the pre-check dns-01 runs before asking the CA.""" + + ACCOUNT = "https://acme-v02.api.letsencrypt.org/acme/acct/1" + + def _run(self, query, domain="svc.example.com", tag="issue"): + saved = dnsguide.query_union + dnsguide.query_union = query + self.addCleanup(lambda: setattr(dnsguide, "query_union", saved)) + args = argparse.Namespace(domain=domain, caa_tag=tag, challenge="dns-01", + account_uri=self.ACCOUNT, resolver="") + with contextlib.redirect_stderr(io.StringIO()): + return dnsguide.caa_only(args) + + def test_pinned_to_another_account_fails(self): + other = '0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/2"' + self.assertEqual(self._run(lambda n, rr, r: [other] if n == "svc.example.com" else []), + dnsguide.CAA_FORBIDS) + + def test_inherited_other_ca_fails_a_wildcard(self): + query = lambda n, rr, r: ['0 issue "digicert.com"'] if n == "example.com" else [] + self.assertEqual(self._run(query, domain="*.svc.example.com", tag="issuewild"), + dnsguide.CAA_FORBIDS) + + def test_no_caa_passes(self): + self.assertEqual(self._run(lambda n, rr, r: []), 0) + + def test_unreadable_dns_does_not_block(self): + def broken(name, rr, resolvers): + raise dnsguide.ResolveError("all resolvers failed") + self.assertEqual(self._run(broken), 0) + + class TestDelegationRecord(unittest.TestCase): """The _acme-challenge CNAME that makes challenge delegation work."""