Skip to content

refactor(ingress): share the certificate pass between modes #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kvinwang merged 1 commit into from
Oct 16, 2026
Merged

refactor(ingress): share the certificate pass between modes #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions custom-domain/dstack-ingress/scripts/cert-lib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#!/bin/bash
# cert-lib.sh - the certificate pass and loop, identical in every mode.
#
# The modes differ in how a domain gets its certificate and where it lands on
# disk, not in what happens around that. The sourcing mode defines:
# process_domain DOMAIN issue or renew; 0 = changed, 2 = nothing to do
# cert_fullchain_path DOMAIN, cert_privkey_path DOMAIN
# acme_account_file print the path of the ACME account document

# True until the first pass finishes. The first pass is setup and checks more
# than issuance strictly needs; later passes check only what would stop a
# renewal.
# shellcheck disable=SC2034 # read by the mode scripts
FIRST_PASS=true

# Set when certificates have been renewed but not yet applied to haproxy.
# Survives across passes, so a failed apply is retried on the next one. Starts
# set so every container start republishes the evidence: what a previous run
# left in /evidences may list removed domains or an old account, or be half
# published.
APPLY_PENDING=true

build_combined_pems() {
local domain fullchain privkey combined
mkdir -p /etc/haproxy/certs
while IFS= read -r domain; do
[[ -n "$domain" ]] || continue
fullchain=$(cert_fullchain_path "$domain")
privkey=$(cert_privkey_path "$domain")
combined="/etc/haproxy/certs/${domain}.pem"
if [ -f "$fullchain" ] && [ -f "$privkey" ]; then
cat "$fullchain" "$privkey" > "$combined"
chmod 600 "$combined"
echo "Combined PEM created: ${combined}"
elif [ -f "$combined" ]; then
# tls-alpn-01 seeds a placeholder here before its first certificate
# exists, so haproxy can bind and forward acme-tls/1. Leave it alone.
echo "No certificate for ${domain} yet; serving the existing PEM"
else
echo "Warning: no certificate for ${domain}, skipping"
fi
done <<<"$(get-all-domains.sh)"
}

collect_evidence() {
local account domain
if ! account=$(acme_account_file); then
echo "Error: cannot generate evidence without the ACME account file" >&2
return 1
fi
evidence_reset || return 1
evidence_collect_account "$account"
while IFS= read -r domain; do
[[ -n "$domain" ]] || continue
evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")"
done <<<"$(get-all-domains.sh)"
evidence_finalize
}

# Empty when the domain has no certificate yet.
cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; }

# One pass over every domain: make sure the records are right, then issue or
# renew. Idempotent, so the first pass and every later one are the same code.
run_pass() {
local domain status failed=0 changed=0 before
while IFS= read -r domain; do
[[ -n "$domain" ]] || continue
before=$(cert_checksum "$domain")
# `if` context, not a bare command: process_domain returns 2 for
# "nothing to do", and under `set -e` a bare non-zero would kill the
# whole script on every quiet renewal pass.
if ( process_domain "$domain" ); then status=0; else status=$?; fi
# A step after issuance can still fail (a CAA write, a timeout that
# kills the client after it saved the certificate). The new certificate
# is on disk by then and the next pass will find it current, so apply
# it now or never.
[ "$(cert_checksum "$domain")" = "$before" ] || changed=1
case $status in
0) changed=1 ;;
2) ;;
*) echo "Certificate management failed for $domain" >&2; failed=1 ;;
esac
done <<<"$(get-all-domains.sh)"

if [ "$changed" -eq 1 ]; then
APPLY_PENDING=true
fi

# Applying is separate from renewing, and retried until it sticks. A
# renewal that reaches disk but not haproxy is invisible: the next pass
# sees a fresh certificate, reports "nothing to renew", and never reloads
# again -- so haproxy would serve the pre-renewal certificate until it
# expires, which is soon, because being near expiry is why it renewed.
if [ "$APPLY_PENDING" = true ]; then
local applied=true
collect_evidence || { echo "Evidence generation failed" >&2; applied=false; }
build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; }
haproxy_reload || applied=false
if [ "$applied" = true ]; then
APPLY_PENDING=false
else
echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2
fi
fi
FIRST_PASS=false
[ "$failed" -eq 0 ] || return 1
# Not a quiet pass either: wait the retry backoff, not RENEW_INTERVAL.
[ "$APPLY_PENDING" = false ] || return 3
}

# $1: seconds to wait before the first pass, for a caller that has just run one.
cert_loop() {
local attempt=0 delay="${1:-0}"
while true; do
if [ "$delay" -gt 0 ]; then
echo "[$(date)] Next certificate check in ${delay}s"
sleep "$delay"
fi
if run_pass; then
attempt=0
delay=${RENEW_INTERVAL:-43200}
else
# Often the failure is a DNS record an operator has yet to create,
# so falling back to the 12-hour interval would be a terrible
# retry rate.
attempt=$((attempt + 1))
delay=$((60 * attempt))
[ "$delay" -gt 1800 ] && delay=1800
echo "[$(date)] Pass failed (attempt ${attempt}); retrying" >&2
fi
done
}
127 changes: 6 additions & 121 deletions custom-domain/dstack-ingress/scripts/dns01.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ set -e
source /scripts/functions.sh
source /scripts/haproxy-lib.sh
source /scripts/evidence-lib.sh
source /scripts/cert-lib.sh

# The zone this container writes into, and the switch that turns delegation on.
DELEGATION_ZONE=${DELEGATION_ZONE:-}
Expand All @@ -32,6 +33,9 @@ fi

cert_fullchain_path() { echo "$(certman.py cert-dir --domain "$1")/fullchain.pem"; }
cert_privkey_path() { echo "$(certman.py cert-dir --domain "$1")/privkey.pem"; }
# certman.py decides which ACME server is in use (ACME_STAGING,
# ACME_DIRECTORY_URL), and so where its account lives.
acme_account_file() { certman.py account-file; }

setup_py_env() {
if [ ! -d /opt/app-venv ]; then
Expand Down Expand Up @@ -161,7 +165,7 @@ delegation_publish() {
# record missing rather than the container pretending to have set it.
delegation_publish_caa() {
local domain="$1" account_file account_uri tag
account_file=$(get_letsencrypt_account_file) || return 1
account_file=$(acme_account_file) || return 1
account_uri=$(jq -j '.uri' "$account_file")
for tag in $(caa_tags_for "$domain"); do
dnsman.py set_caa \
Expand Down Expand Up @@ -232,7 +236,7 @@ set_caa_record() {
local ACCOUNT_URI
local account_file

if ! account_file=$(get_letsencrypt_account_file); then
if ! account_file=$(acme_account_file); then
echo "Warning: Cannot set CAA record - account file not found"
echo "This is not critical - certificates can still be issued without CAA records"
return
Expand Down Expand Up @@ -324,125 +328,6 @@ issue_certificate() {
certman.py auto --domain "$1" ${ACME_EMAIL:+--email "$ACME_EMAIL"}
}

build_combined_pems() {
local domain fullchain privkey combined
mkdir -p /etc/haproxy/certs
while IFS= read -r domain; do
[[ -n "$domain" ]] || continue
fullchain=$(cert_fullchain_path "$domain")
privkey=$(cert_privkey_path "$domain")
combined="/etc/haproxy/certs/${domain}.pem"
if [ -f "$fullchain" ] && [ -f "$privkey" ]; then
cat "$fullchain" "$privkey" > "$combined"
chmod 600 "$combined"
echo "Combined PEM created: ${combined}"
else
echo "Warning: Cert files missing for ${domain}, skipping"
fi
done <<<"$(get-all-domains.sh)"
}

collect_evidence() {
local account domain
if ! account=$(get_letsencrypt_account_file); then
echo "Error: cannot generate evidence without the ACME account file" >&2
return 1
fi
evidence_reset || return 1
evidence_collect_account "$account"
while IFS= read -r domain; do
[[ -n "$domain" ]] || continue
evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")"
done <<<"$(get-all-domains.sh)"
evidence_finalize
}

# True until the first pass finishes. The first pass is setup and checks more
# than issuance strictly needs; later passes check only what would stop a
# renewal.
FIRST_PASS=true

# Set when certificates have been renewed but not yet applied to haproxy.
# Survives across passes, so a failed apply is retried on the next one. Starts
# set so every container start republishes the evidence: what a previous run
# left in /evidences may list removed domains or an old account, or be half
# published.
APPLY_PENDING=true

# Empty when the domain has no certificate yet.
cert_checksum() { sha256sum "$(cert_fullchain_path "$1")" 2>/dev/null || true; }

# One pass over every domain: make the DNS records right, then issue or renew.
# Idempotent, so the first pass and every later one are the same code.
run_pass() {
local domain status failed=0 changed=0 before
while IFS= read -r domain; do
[[ -n "$domain" ]] || continue
before=$(cert_checksum "$domain")
# `if` context, not a bare command: process_domain returns 2 for
# "nothing to do", and under `set -e` a bare non-zero would kill the
# whole script on every quiet renewal pass.
if ( process_domain "$domain" ); then status=0; else status=$?; fi
# A step after issuance can still fail (a CAA write, a timeout). The
# new certificate is on disk by then and the next pass will find it
# current, so apply it now or never.
[ "$(cert_checksum "$domain")" = "$before" ] || changed=1
case $status in
0) changed=1 ;;
2) ;;
*) echo "Certificate management failed for $domain" >&2; failed=1 ;;
esac
done <<<"$(get-all-domains.sh)"

if [ "$changed" -eq 1 ]; then
APPLY_PENDING=true
fi

# Applying is separate from renewing, and retried until it sticks. A
# renewal that reaches disk but not haproxy is invisible: the next pass
# sees a fresh certificate, reports "nothing to renew", and never reloads
# again -- so haproxy would serve the pre-renewal certificate until it
# expires, which is soon, because being near expiry is why it renewed.
if [ "$APPLY_PENDING" = true ]; then
local applied=true
collect_evidence || { echo "Evidence generation failed" >&2; applied=false; }
build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; }
haproxy_reload || applied=false
if [ "$applied" = true ]; then
APPLY_PENDING=false
else
echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2
fi
fi
FIRST_PASS=false
[ "$failed" -eq 0 ] || return 1
# Not a quiet pass either: wait the retry backoff, not RENEW_INTERVAL.
[ "$APPLY_PENDING" = false ] || return 3
}

# $1: seconds to wait before the first pass. dns-01 issues before it serves, so
# the caller has already run a pass by the time we get here; without this the
# loop would immediately repeat it -- two full passes, and for a single domain
# two certbot invocations plus a duplicate round of DNS writes, on every start.
cert_loop() {
local attempt=0 delay="${1:-0}"
while true; do
if [ "$delay" -gt 0 ]; then
echo "[$(date)] Next certificate check in ${delay}s"
sleep "$delay"
fi
if run_pass; then
attempt=0
delay=${RENEW_INTERVAL:-43200}
else
attempt=$((attempt + 1))
delay=$((60 * attempt))
[ "$delay" -gt 1800 ] && delay=1800
echo "[$(date)] Pass failed (attempt ${attempt}); retrying" >&2
fi
done
}

setup_py_env
setup_certbot_env
load_dstack_identity
Expand Down
6 changes: 0 additions & 6 deletions custom-domain/dstack-ingress/scripts/functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,12 +204,6 @@ sanitize_proxy_buffers() {
fi
}

# certman.py decides which ACME server is in use (ACME_STAGING,
# ACME_DIRECTORY_URL), and so where its account lives.
get_letsencrypt_account_file() {
certman.py account-file
}

txt_record_name() {
local domain="$1"
if [[ "$domain" == \*.* ]]; then
Expand Down
Loading
Loading