diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 56e8723..a9f27ca 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -20,7 +20,7 @@ dstack-ingress is a HAProxy-based L4 (TCP) proxy that provides: 2. **TLS Termination**: HAProxy terminates TLS and forwards the decrypted TCP stream to your backend. No HTTP inspection — the proxy operates entirely at L4. -3. **Certificate Renewal**: A background daemon checks for renewal every 12 hours. On renewal, HAProxy is gracefully reloaded with zero downtime. A failed pass is retried after 60s, backing off by 60s per attempt up to 30 minutes. A domain that fails does not stop the others from being served: in dns-01 mode the container exits at startup only if no domain has a certificate yet, and in tls-alpn-01 mode it serves a placeholder certificate until issuance succeeds. +3. **Certificate Renewal**: A background daemon checks for renewal every 12 hours. On renewal, HAProxy is gracefully reloaded with zero downtime. A failed pass is retried after 60s, backing off by 60s per attempt up to 30 minutes. A domain that fails does not stop the others from being served: in dns-01 mode a restart serves the certificates already on disk right away and checks them in the background, and the container exits at startup only if no domain has a certificate yet. In tls-alpn-01 mode it serves a placeholder certificate until issuance succeeds. 4. **Evidence Generation**: Generates cryptographically linked attestation evidence (ACME account, certificates, TDX quote / dstack attestation) for TEE verification. diff --git a/custom-domain/dstack-ingress/TESTING.md b/custom-domain/dstack-ingress/TESTING.md index a61e0eb..59fee0e 100644 --- a/custom-domain/dstack-ingress/TESTING.md +++ b/custom-domain/dstack-ingress/TESTING.md @@ -134,10 +134,10 @@ renewal window, and `RENEW_DAYS_BEFORE` sets it in either mode: - **not due**: leave `RENEW_DAYS_BEFORE` unset, set a short `RENEW_INTERVAL`, and restart the container. Expect lego `Skip renewal` / certbot `No - certificates need renewal`. The first pass after a start always republishes - the evidence (and reloads haproxy in tls-alpn-01 mode); the passes after it - must not. A later pass that reloads or regenerates evidence is a bug — it - means the renewal decision is not being read correctly. + certificates need renewal`. Every start republishes the evidence before + serving; no pass after that may reload haproxy or regenerate the evidence. + One that does is a bug — it means the renewal decision is not being read + correctly. - **due**: set `RENEW_DAYS_BEFORE=365` and restart. Expect a new certificate, a reload, and regenerated evidence. diff --git a/custom-domain/dstack-ingress/scripts/cert-lib.sh b/custom-domain/dstack-ingress/scripts/cert-lib.sh index c8a935d..8b80208 100644 --- a/custom-domain/dstack-ingress/scripts/cert-lib.sh +++ b/custom-domain/dstack-ingress/scripts/cert-lib.sh @@ -17,10 +17,10 @@ FIRST_PASS=true # show them yet. They survive across passes, so a failed step is retried on the # next one, and only that step: evidence that keeps failing must not reload # haproxy on every retry, as each reload leaves the old workers holding their -# connections for up to TIMEOUT_CLIENT. Both start set so every container start -# republishes the evidence: what a previous run left in /evidences may list -# removed domains or an old account, or be half published. -RELOAD_PENDING=true +# connections for up to TIMEOUT_CLIENT. The evidence starts pending, so every +# start republishes it (see prepare_serving); haproxy has nothing to reload +# before it starts. +RELOAD_PENDING=false EVIDENCE_PENDING=true build_combined_pems() { @@ -63,6 +63,22 @@ collect_evidence() { # Empty when the domain has no certificate yet. cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; } +# Run once, just before haproxy starts. haproxy reads the PEMs itself, so there +# is nothing to reload yet. What the last run left in /evidences attests a +# container that may no longer be running, so republish the evidence unless a +# startup pass already has -- it is local and quick -- or, if that is not +# possible yet, serve none. A first start has no ACME account to collect yet. +prepare_serving() { + build_combined_pems + RELOAD_PENDING=false + [ "$EVIDENCE_PENDING" = true ] || return 0 + if acme_account_file >/dev/null 2>&1 && collect_evidence; then + EVIDENCE_PENDING=false + else + evidence_withdraw + fi +} + # One pass over every domain: make sure the records are right, then issue or # renew. Idempotent, so the first pass and every later one are the same code. run_pass() { diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 8f78236..7eb3031 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -332,13 +332,6 @@ setup_py_env setup_certbot_env load_dstack_identity -# dns-01 needs no inbound traffic, so keep the original order: certificate -# first, serving second. A certificate failure here is fatal only when there is -# nothing to serve. Otherwise exiting would just restart into the same failure -# -- hitting the CA again on every restart -- while every domain goes unserved, -# including those whose certificates are fine (a newly added domain failing its -# first issuance would take down the others). Serve what there is and let the -# loop retry with backoff instead. have_any_cert() { local domain while IFS= read -r domain; do @@ -348,18 +341,29 @@ have_any_cert() { return 1 } -startup_status=0 -run_pass || startup_status=$? -case $startup_status in - 0) next_check=${RENEW_INTERVAL:-43200} ;; - 3) next_check=60 ;; - *) - have_any_cert || exit "$startup_status" - echo "Certificate pass failed; serving the certificates there are and retrying" >&2 - next_check=60 - ;; -esac -build_combined_pems +# With a certificate on disk, serve it first and run the pass behind haproxy. +# Running it first would leave every domain unserved until it finishes, and one +# new domain can hold it for long: a certbot run up to its timeout, or up to +# DNS_SETUP_TIMEOUT while its delegation CNAMEs are still being created. +# +# With none there is nothing to serve, and dns-01 needs no inbound traffic, so +# issue first. A pass that ends with still no certificate is fatal; one that +# got some serves them and leaves the loop to retry the rest with backoff. +next_check=0 +if ! have_any_cert; then + startup_status=0 + run_pass || startup_status=$? + case $startup_status in + 0) next_check=${RENEW_INTERVAL:-43200} ;; + 3) next_check=60 ;; + *) + have_any_cert || exit "$startup_status" + echo "Certificate pass failed; serving the certificates there are and retrying" >&2 + next_check=60 + ;; + esac +fi +prepare_serving haproxy_emit_global haproxy_emit_tls_frontend ":${PORT}$(haproxy_accept_proxy)" diff --git a/custom-domain/dstack-ingress/scripts/evidence-lib.sh b/custom-domain/dstack-ingress/scripts/evidence-lib.sh index 914974f..9d658e0 100644 --- a/custom-domain/dstack-ingress/scripts/evidence-lib.sh +++ b/custom-domain/dstack-ingress/scripts/evidence-lib.sh @@ -115,6 +115,12 @@ evidence_publish() { rm -rf "$stage" } +# Take down what an earlier run published, for when this one cannot replace it +# yet: no evidence is better than evidence for another container. +evidence_withdraw() { + rm -f "$EVIDENCE_DIR"/{acme-account.json,cert-*.pem,sha256sum.txt,quote.json,attestation.json} +} + evidence_start_server() { [ "${EVIDENCE_SERVER:-true}" = "true" ] || return 0 # Bind loopback explicitly. Left to itself mini_httpd binds [::]:port first, diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index 586c350..094260d 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -235,7 +235,7 @@ require_instance_id # first certificate can exist, so start on a placeholder and let the loop # converge behind it. ensure_placeholder_certs -build_combined_pems +prepare_serving haproxy_emit_global emit_peek_frontend