diff --git a/custom-domain/dstack-ingress/scripts/cert-lib.sh b/custom-domain/dstack-ingress/scripts/cert-lib.sh index ae883d2..c8a935d 100644 --- a/custom-domain/dstack-ingress/scripts/cert-lib.sh +++ b/custom-domain/dstack-ingress/scripts/cert-lib.sh @@ -13,12 +13,15 @@ # shellcheck disable=SC2034 # read by the mode scripts FIRST_PASS=true -# Set when certificates have been renewed but not yet applied to haproxy. -# Survives across passes, so a failed apply is retried on the next one. Starts -# set so every container start republishes the evidence: what a previous run -# left in /evidences may list removed domains or an old account, or be half -# published. -APPLY_PENDING=true +# Set when certificates have changed but haproxy, or the evidence, does not +# show them yet. They survive across passes, so a failed step is retried on the +# next one, and only that step: evidence that keeps failing must not reload +# haproxy on every retry, as each reload leaves the old workers holding their +# connections for up to TIMEOUT_CLIENT. Both start set so every container start +# republishes the evidence: what a previous run left in /evidences may list +# removed domains or an old account, or be half published. +RELOAD_PENDING=true +EVIDENCE_PENDING=true build_combined_pems() { local domain fullchain privkey combined @@ -84,7 +87,8 @@ run_pass() { done <<<"$(get-all-domains.sh)" if [ "$changed" -eq 1 ]; then - APPLY_PENDING=true + RELOAD_PENDING=true + EVIDENCE_PENDING=true fi # Applying is separate from renewing, and retried until it sticks. A @@ -92,21 +96,24 @@ run_pass() { # sees a fresh certificate, reports "nothing to renew", and never reloads # again -- so haproxy would serve the pre-renewal certificate until it # expires, which is soon, because being near expiry is why it renewed. - if [ "$APPLY_PENDING" = true ]; then - local applied=true - collect_evidence || { echo "Evidence generation failed" >&2; applied=false; } - build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; } - haproxy_reload || applied=false - if [ "$applied" = true ]; then - APPLY_PENDING=false + if [ "$EVIDENCE_PENDING" = true ]; then + if collect_evidence; then + EVIDENCE_PENDING=false else - echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2 + echo "[$(date)] Evidence generation failed; retrying on the next pass" >&2 + fi + fi + if [ "$RELOAD_PENDING" = true ]; then + if build_combined_pems && haproxy_reload; then + RELOAD_PENDING=false + else + echo "[$(date)] Certificate apply failed; retrying on the next pass" >&2 fi fi FIRST_PASS=false [ "$failed" -eq 0 ] || return 1 # Not a quiet pass either: wait the retry backoff, not RENEW_INTERVAL. - [ "$APPLY_PENDING" = false ] || return 3 + [ "$RELOAD_PENDING" = false ] && [ "$EVIDENCE_PENDING" = false ] || return 3 } # $1: seconds to wait before the first pass, for a caller that has just run one.