From 5d35c505c6fcd7f21ea0a5e6a9db1583a091e47d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 15 Oct 2026 17:56:11 -0700 Subject: [PATCH] refactor(ingress): share the certificate pass between modes run_pass, cert_loop, collect_evidence, build_combined_pems and the FIRST_PASS/APPLY_PENDING state were copied in dns01.sh and tlsalpn.sh, and cert_loop had already drifted. Move them into cert-lib.sh; each mode now only supplies process_domain, the certificate paths and acme_account_file. No behaviour change. Signed-off-by: Kevin Wang --- .../dstack-ingress/scripts/cert-lib.sh | 133 ++++++++++++++++++ custom-domain/dstack-ingress/scripts/dns01.sh | 127 +---------------- .../dstack-ingress/scripts/functions.sh | 6 - .../dstack-ingress/scripts/tlsalpn.sh | 125 +--------------- 4 files changed, 143 insertions(+), 248 deletions(-) create mode 100644 custom-domain/dstack-ingress/scripts/cert-lib.sh diff --git a/custom-domain/dstack-ingress/scripts/cert-lib.sh b/custom-domain/dstack-ingress/scripts/cert-lib.sh new file mode 100644 index 0000000..ae883d2 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/cert-lib.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# cert-lib.sh - the certificate pass and loop, identical in every mode. +# +# The modes differ in how a domain gets its certificate and where it lands on +# disk, not in what happens around that. The sourcing mode defines: +# process_domain DOMAIN issue or renew; 0 = changed, 2 = nothing to do +# cert_fullchain_path DOMAIN, cert_privkey_path DOMAIN +# acme_account_file print the path of the ACME account document + +# True until the first pass finishes. The first pass is setup and checks more +# than issuance strictly needs; later passes check only what would stop a +# renewal. +# shellcheck disable=SC2034 # read by the mode scripts +FIRST_PASS=true + +# Set when certificates have been renewed but not yet applied to haproxy. +# Survives across passes, so a failed apply is retried on the next one. Starts +# set so every container start republishes the evidence: what a previous run +# left in /evidences may list removed domains or an old account, or be half +# published. +APPLY_PENDING=true + +build_combined_pems() { + local domain fullchain privkey combined + mkdir -p /etc/haproxy/certs + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + fullchain=$(cert_fullchain_path "$domain") + privkey=$(cert_privkey_path "$domain") + combined="/etc/haproxy/certs/${domain}.pem" + if [ -f "$fullchain" ] && [ -f "$privkey" ]; then + cat "$fullchain" "$privkey" > "$combined" + chmod 600 "$combined" + echo "Combined PEM created: ${combined}" + elif [ -f "$combined" ]; then + # tls-alpn-01 seeds a placeholder here before its first certificate + # exists, so haproxy can bind and forward acme-tls/1. Leave it alone. + echo "No certificate for ${domain} yet; serving the existing PEM" + else + echo "Warning: no certificate for ${domain}, skipping" + fi + done <<<"$(get-all-domains.sh)" +} + +collect_evidence() { + local account domain + if ! account=$(acme_account_file); then + echo "Error: cannot generate evidence without the ACME account file" >&2 + return 1 + fi + evidence_reset || return 1 + evidence_collect_account "$account" + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" + done <<<"$(get-all-domains.sh)" + evidence_finalize +} + +# Empty when the domain has no certificate yet. +cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; } + +# One pass over every domain: make sure the records are right, then issue or +# renew. Idempotent, so the first pass and every later one are the same code. +run_pass() { + local domain status failed=0 changed=0 before + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + before=$(cert_checksum "$domain") + # `if` context, not a bare command: process_domain returns 2 for + # "nothing to do", and under `set -e` a bare non-zero would kill the + # whole script on every quiet renewal pass. + if ( process_domain "$domain" ); then status=0; else status=$?; fi + # A step after issuance can still fail (a CAA write, a timeout that + # kills the client after it saved the certificate). The new certificate + # is on disk by then and the next pass will find it current, so apply + # it now or never. + [ "$(cert_checksum "$domain")" = "$before" ] || changed=1 + case $status in + 0) changed=1 ;; + 2) ;; + *) echo "Certificate management failed for $domain" >&2; failed=1 ;; + esac + done <<<"$(get-all-domains.sh)" + + if [ "$changed" -eq 1 ]; then + APPLY_PENDING=true + fi + + # Applying is separate from renewing, and retried until it sticks. A + # renewal that reaches disk but not haproxy is invisible: the next pass + # sees a fresh certificate, reports "nothing to renew", and never reloads + # again -- so haproxy would serve the pre-renewal certificate until it + # expires, which is soon, because being near expiry is why it renewed. + if [ "$APPLY_PENDING" = true ]; then + local applied=true + collect_evidence || { echo "Evidence generation failed" >&2; applied=false; } + build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; } + haproxy_reload || applied=false + if [ "$applied" = true ]; then + APPLY_PENDING=false + else + echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2 + fi + fi + FIRST_PASS=false + [ "$failed" -eq 0 ] || return 1 + # Not a quiet pass either: wait the retry backoff, not RENEW_INTERVAL. + [ "$APPLY_PENDING" = false ] || return 3 +} + +# $1: seconds to wait before the first pass, for a caller that has just run one. +cert_loop() { + local attempt=0 delay="${1:-0}" + while true; do + if [ "$delay" -gt 0 ]; then + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" + fi + if run_pass; then + attempt=0 + delay=${RENEW_INTERVAL:-43200} + else + # Often the failure is a DNS record an operator has yet to create, + # so falling back to the 12-hour interval would be a terrible + # retry rate. + attempt=$((attempt + 1)) + delay=$((60 * attempt)) + [ "$delay" -gt 1800 ] && delay=1800 + echo "[$(date)] Pass failed (attempt ${attempt}); retrying" >&2 + fi + done +} diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 635f431..8f78236 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -10,6 +10,7 @@ set -e source /scripts/functions.sh source /scripts/haproxy-lib.sh source /scripts/evidence-lib.sh +source /scripts/cert-lib.sh # The zone this container writes into, and the switch that turns delegation on. DELEGATION_ZONE=${DELEGATION_ZONE:-} @@ -32,6 +33,9 @@ fi cert_fullchain_path() { echo "$(certman.py cert-dir --domain "$1")/fullchain.pem"; } cert_privkey_path() { echo "$(certman.py cert-dir --domain "$1")/privkey.pem"; } +# certman.py decides which ACME server is in use (ACME_STAGING, +# ACME_DIRECTORY_URL), and so where its account lives. +acme_account_file() { certman.py account-file; } setup_py_env() { if [ ! -d /opt/app-venv ]; then @@ -161,7 +165,7 @@ delegation_publish() { # record missing rather than the container pretending to have set it. delegation_publish_caa() { local domain="$1" account_file account_uri tag - account_file=$(get_letsencrypt_account_file) || return 1 + account_file=$(acme_account_file) || return 1 account_uri=$(jq -j '.uri' "$account_file") for tag in $(caa_tags_for "$domain"); do dnsman.py set_caa \ @@ -232,7 +236,7 @@ set_caa_record() { local ACCOUNT_URI local account_file - if ! account_file=$(get_letsencrypt_account_file); then + if ! account_file=$(acme_account_file); then echo "Warning: Cannot set CAA record - account file not found" echo "This is not critical - certificates can still be issued without CAA records" return @@ -324,125 +328,6 @@ issue_certificate() { certman.py auto --domain "$1" ${ACME_EMAIL:+--email "$ACME_EMAIL"} } -build_combined_pems() { - local domain fullchain privkey combined - mkdir -p /etc/haproxy/certs - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - fullchain=$(cert_fullchain_path "$domain") - privkey=$(cert_privkey_path "$domain") - combined="/etc/haproxy/certs/${domain}.pem" - if [ -f "$fullchain" ] && [ -f "$privkey" ]; then - cat "$fullchain" "$privkey" > "$combined" - chmod 600 "$combined" - echo "Combined PEM created: ${combined}" - else - echo "Warning: Cert files missing for ${domain}, skipping" - fi - done <<<"$(get-all-domains.sh)" -} - -collect_evidence() { - local account domain - if ! account=$(get_letsencrypt_account_file); then - echo "Error: cannot generate evidence without the ACME account file" >&2 - return 1 - fi - evidence_reset || return 1 - evidence_collect_account "$account" - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" - done <<<"$(get-all-domains.sh)" - evidence_finalize -} - -# True until the first pass finishes. The first pass is setup and checks more -# than issuance strictly needs; later passes check only what would stop a -# renewal. -FIRST_PASS=true - -# Set when certificates have been renewed but not yet applied to haproxy. -# Survives across passes, so a failed apply is retried on the next one. Starts -# set so every container start republishes the evidence: what a previous run -# left in /evidences may list removed domains or an old account, or be half -# published. -APPLY_PENDING=true - -# Empty when the domain has no certificate yet. -cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; } - -# One pass over every domain: make the DNS records right, then issue or renew. -# Idempotent, so the first pass and every later one are the same code. -run_pass() { - local domain status failed=0 changed=0 before - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - before=$(cert_checksum "$domain") - # `if` context, not a bare command: process_domain returns 2 for - # "nothing to do", and under `set -e` a bare non-zero would kill the - # whole script on every quiet renewal pass. - if ( process_domain "$domain" ); then status=0; else status=$?; fi - # A step after issuance can still fail (a CAA write, a timeout). The - # new certificate is on disk by then and the next pass will find it - # current, so apply it now or never. - [ "$(cert_checksum "$domain")" = "$before" ] || changed=1 - case $status in - 0) changed=1 ;; - 2) ;; - *) echo "Certificate management failed for $domain" >&2; failed=1 ;; - esac - done <<<"$(get-all-domains.sh)" - - if [ "$changed" -eq 1 ]; then - APPLY_PENDING=true - fi - - # Applying is separate from renewing, and retried until it sticks. A - # renewal that reaches disk but not haproxy is invisible: the next pass - # sees a fresh certificate, reports "nothing to renew", and never reloads - # again -- so haproxy would serve the pre-renewal certificate until it - # expires, which is soon, because being near expiry is why it renewed. - if [ "$APPLY_PENDING" = true ]; then - local applied=true - collect_evidence || { echo "Evidence generation failed" >&2; applied=false; } - build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; } - haproxy_reload || applied=false - if [ "$applied" = true ]; then - APPLY_PENDING=false - else - echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2 - fi - fi - FIRST_PASS=false - [ "$failed" -eq 0 ] || return 1 - # Not a quiet pass either: wait the retry backoff, not RENEW_INTERVAL. - [ "$APPLY_PENDING" = false ] || return 3 -} - -# $1: seconds to wait before the first pass. dns-01 issues before it serves, so -# the caller has already run a pass by the time we get here; without this the -# loop would immediately repeat it -- two full passes, and for a single domain -# two certbot invocations plus a duplicate round of DNS writes, on every start. -cert_loop() { - local attempt=0 delay="${1:-0}" - while true; do - if [ "$delay" -gt 0 ]; then - echo "[$(date)] Next certificate check in ${delay}s" - sleep "$delay" - fi - if run_pass; then - attempt=0 - delay=${RENEW_INTERVAL:-43200} - else - attempt=$((attempt + 1)) - delay=$((60 * attempt)) - [ "$delay" -gt 1800 ] && delay=1800 - echo "[$(date)] Pass failed (attempt ${attempt}); retrying" >&2 - fi - done -} - setup_py_env setup_certbot_env load_dstack_identity diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index 28e4f8e..935768f 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -204,12 +204,6 @@ sanitize_proxy_buffers() { fi } -# certman.py decides which ACME server is in use (ACME_STAGING, -# ACME_DIRECTORY_URL), and so where its account lives. -get_letsencrypt_account_file() { - certman.py account-file -} - txt_record_name() { local domain="$1" if [[ "$domain" == \*.* ]]; then diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index daa2bf3..586c350 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -11,6 +11,7 @@ set -e source /scripts/functions.sh source /scripts/haproxy-lib.sh source /scripts/evidence-lib.sh +source /scripts/cert-lib.sh # ACME_EMAIL / ACME_STAGING are normalised by entrypoint.sh and exported, so # they are already correct here whichever name the operator used. @@ -161,44 +162,6 @@ ensure_placeholder_certs() { done <<<"$all_domains" } -build_combined_pems() { - local domain fullchain privkey combined - mkdir -p /etc/haproxy/certs - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - fullchain=$(cert_fullchain_path "$domain") - privkey=$(cert_privkey_path "$domain") - combined="/etc/haproxy/certs/${domain}.pem" - if [ -f "$fullchain" ] && [ -f "$privkey" ]; then - cat "$fullchain" "$privkey" > "$combined" - chmod 600 "$combined" - echo "Combined PEM created: ${combined}" - elif [ -f "$combined" ]; then - # Expected on a first run: ensure_placeholder_certs has already put a - # self-signed certificate here so haproxy can bind and forward - # acme-tls/1, and the real one does not exist yet. Leave it alone. - echo "No certificate for ${domain} yet; serving the placeholder" - else - echo "Warning: no certificate or placeholder for ${domain}, skipping" - fi - done <<<"$(get-all-domains.sh)" -} - -collect_evidence() { - local account domain - if ! account=$(acme_account_file); then - echo "Error: cannot generate evidence without the ACME account file" >&2 - return 1 - fi - evidence_reset || return 1 - evidence_collect_account "$account" - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" - done <<<"$(get-all-domains.sh)" - evidence_finalize -} - process_domain() { local domain="$1" @@ -262,89 +225,6 @@ process_domain() { legoman.py auto --domain "$domain" ${ACME_EMAIL:+--email "$ACME_EMAIL"} } -# Set when certificates have been renewed but not yet applied to haproxy. -# Survives across passes, so a failed apply is retried on the next one. Starts -# set so every container start republishes the evidence: what a previous run -# left in /evidences may list removed domains or an old account, or be half -# published. -APPLY_PENDING=true - -# One pass over every domain: make sure the records exist, then issue or renew. -# Idempotent, so the first pass and every later one are the same code. There is -# deliberately only one of these -- an earlier split between a one-shot -# bootstrap and a renewal daemon raced for lego's challenge port, and the loser -# still spent one of Let's Encrypt's five failed validations per hostname/hour. -FIRST_PASS=true - -# Empty when the domain has no certificate yet. -cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; } - -run_pass() { - local domain status failed=0 changed=0 before - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - before=$(cert_checksum "$domain") - # `if` context, not a bare command: process_domain returns 2 for - # "nothing to do", and under `set -e` a bare non-zero would kill the - # whole script on every quiet renewal pass. - if ( process_domain "$domain" ); then status=0; else status=$?; fi - # lego can be killed by the timeout after saving a certificate. The - # new certificate is on disk by then and the next pass will find it - # current, so apply it now or never. - [ "$(cert_checksum "$domain")" = "$before" ] || changed=1 - case $status in - 0) changed=1 ;; - 2) ;; - *) echo "Certificate management failed for $domain" >&2; failed=1 ;; - esac - done <<<"$(get-all-domains.sh)" - - if [ "$changed" -eq 1 ]; then - APPLY_PENDING=true - fi - - # Applying is separate from renewing, and retried until it sticks. A - # renewal that reaches disk but not haproxy is invisible: the next pass - # sees a fresh certificate, reports "nothing to renew", and never reloads - # again -- so haproxy would serve the pre-renewal certificate until it - # expires, which is soon, because being near expiry is why it renewed. - if [ "$APPLY_PENDING" = true ]; then - local applied=true - collect_evidence || { echo "Evidence generation failed" >&2; applied=false; } - build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; } - haproxy_reload || applied=false - if [ "$applied" = true ]; then - APPLY_PENDING=false - else - echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2 - fi - fi - FIRST_PASS=false - [ "$failed" -eq 0 ] || return 1 - # Not a quiet pass either: wait the retry backoff, not RENEW_INTERVAL. - [ "$APPLY_PENDING" = false ] || return 3 -} - -cert_loop() { - local attempt=0 delay - while true; do - if run_pass; then - attempt=0 - delay=${RENEW_INTERVAL:-43200} - else - # The normal state here is "waiting for an operator to create a DNS - # record", so falling back to the 12-hour interval would be a - # terrible retry rate. - attempt=$((attempt + 1)) - delay=$((60 * attempt)) - [ "$delay" -gt 1800 ] && delay=1800 - echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" - fi - echo "[$(date)] Next certificate check in ${delay}s" - sleep "$delay" - done -} - reject_wildcards setup_py_env check_lego @@ -363,6 +243,9 @@ haproxy_emit_tls_frontend "127.0.0.1:${TLS_TERMINATE_PORT} accept-proxy" haproxy_emit_backends evidence_start_server +# The only certificate loop. An earlier split between a one-shot bootstrap and +# a renewal daemon raced for lego's challenge port, and the loser still spent +# one of Let's Encrypt's five failed validations per hostname/hour. cert_loop & exec "$@"