From 59fd874a35021608f27f06dd0f1a0a5e091cca4c Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 19 Oct 2026 09:32:58 -0300 Subject: [PATCH 1/2] truenas-appliance-runner: wait for node B to answer ssh from a new boot after giving it its own ids --- .../actions/truenas-appliance-runner/ha.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/ha.sh b/.github/actions/truenas-appliance-runner/ha.sh index f168229..220a4ac 100755 --- a/.github/actions/truenas-appliance-runner/ha.sh +++ b/.github/actions/truenas-appliance-runner/ha.sh @@ -232,14 +232,21 @@ unique() { head -c 16 /dev/urandom | od -An -tx1 | tr -d " \n" | sudo tee /etc/machine-id >/dev/null sudo rm -f /var/lib/dbus/machine-id sudo zgenhostid -f' + local boot now i + boot="$(ssh_ b cat /proc/sys/kernel/random/boot_id)" ssh_ b 'sudo systemctl reboot' || true - # Down before it is waited for: the API answers for a while after the - # reboot is asked for. - local i - for i in $(seq 1 60); do - ssh_ b true 2>/dev/null || break - sleep 2 + # Back once B answers ssh from a new boot. Neither "ssh stopped answering" + # nor "the API answers" proves that: the API answers for a while after the + # reboot is asked for, and on the new boot it can answer before sshd does + # (run 39251437564 read B's ids over ssh and was refused). + for i in $(seq 1 120); do + now="$(ssh_ b cat /proc/sys/kernel/random/boot_id 2>/dev/null)" || now= + [ -n "$now" ] && [ "$now" != "$boot" ] && break + sleep 5 done + if [ -z "$now" ] || [ "$now" = "$boot" ]; then + die "node B did not come back over ssh from its reboot within 10 minutes" + fi "${TNCTL[@]}" wait "ws://${IP[b]}" "$APPLIANCE_USER" "$password" --timeout 600 >/dev/null b="$(ids b)" [ "$a" != "$b" ] || die "node B still has A's ids after its reboot: $b" From 10b12afe173a81fe2f3ca6e595a5b7e94269af36 Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 19 Oct 2026 09:40:19 -0300 Subject: [PATCH 2/2] truenas-appliance-runner: give node B ten minutes by the clock, retrying refused connections --- .github/actions/truenas-appliance-runner/ha.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/ha.sh b/.github/actions/truenas-appliance-runner/ha.sh index 220a4ac..2cbfacc 100755 --- a/.github/actions/truenas-appliance-runner/ha.sh +++ b/.github/actions/truenas-appliance-runner/ha.sh @@ -232,14 +232,17 @@ unique() { head -c 16 /dev/urandom | od -An -tx1 | tr -d " \n" | sudo tee /etc/machine-id >/dev/null sudo rm -f /var/lib/dbus/machine-id sudo zgenhostid -f' - local boot now i + local boot now deadline boot="$(ssh_ b cat /proc/sys/kernel/random/boot_id)" ssh_ b 'sudo systemctl reboot' || true # Back once B answers ssh from a new boot. Neither "ssh stopped answering" # nor "the API answers" proves that: the API answers for a while after the # reboot is asked for, and on the new boot it can answer before sshd does # (run 39251437564 read B's ids over ssh and was refused). - for i in $(seq 1 120); do + # A refused or timed-out connection (sshd not started yet, the network + # not up) is just tried again. + deadline=$((SECONDS + 600)) + while [ "$SECONDS" -lt "$deadline" ]; do now="$(ssh_ b cat /proc/sys/kernel/random/boot_id 2>/dev/null)" || now= [ -n "$now" ] && [ "$now" != "$boot" ] && break sleep 5