From c58f7519936dc547ee2aec5bfbb1e477356656d9 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sun, 20 Sep 2026 00:34:23 +0000 Subject: Verify deploys, and add a self-recovering health check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploy used to end with `systemctl status`, which a human skims and a script ignores: it exits zero whether or not anything works, and says nothing about the sub-services behind the gateway. A deploy could therefore report success while /scout/ served 502s — which is exactly what happened, undetected, for three and a half weeks. scripts/deploy now: * syncs scripts/ to ${SITE_DIR}/app-scripts/ so the timer runs the deployed version from a stable path * restarts sub-services that are not active. They are separate units and a doot deploy does not touch them, so one sitting in `failed` after exhausting its restart limit stays down through any number of green deploys. * calls scripts/verify-deploy, which asserts and exits non-zero scripts/verify-deploy asserts all five binaries are executable, all four units are active, all four gateway routes answer without a 5xx, and the running build stamp matches the commit just built. scripts/health-check runs the same assertions on a 5-minute timer and recovers what it can: chmod +x a binary that lost its execute bit, reset-failed and start a unit that gave up. There is no mail/ntfy/apprise on this host, so alerting is a greppable HEALTH ALERT line in the journal, a status JSON at data/health-status.json for the dashboard to surface later, and a non-zero exit so the unit shows in `systemctl --failed`. OnBootSec=2min because a reboot is when this failure mode first appeared. Shared assertions live in scripts/lib/site-checks.sh so verify-deploy and health-check cannot drift apart. Two bugs were found and fixed in these checks by testing their failure paths, both of which would have made them report a broken site as healthy: * `$(curl ... || echo 000)` concatenates curl's own "000" transport-error output with the echoed one, giving "000000", which matched no case arm and was reported ok. * check_services sets FAILED_SERVICES, but health-check captured it with `$(...)`, so the array was populated in a subshell and discarded — silently disabling service recovery entirely. Both the calling convention and the subshell semantics are now pinned by tests. scripts/test-health-checks.sh: 9 cases, mostly negative, against a temp tree. Recovery additionally verified live on this host by stopping hawaii.service and by clearing scout's execute bit; both were detected, recovered and re-verified. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- scripts/health-check | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100755 scripts/health-check (limited to 'scripts/health-check') diff --git a/scripts/health-check b/scripts/health-check new file mode 100755 index 0000000..b1c34d0 --- /dev/null +++ b/scripts/health-check @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Periodic health check for doot.terst.org and its sub-services, with recovery. +# +# The failure this exists for: doot itself stays green while a sub-service +# behind the gateway is dead, so /scout/ returns 502 and nothing anywhere says +# so. Scout and hawaii were down for three and a half weeks, and claudomator +# crash-looped 426,000 times, before anyone opened the page and noticed. +# +# There is no mail/ntfy/apprise on this host, so "alerting" here means: +# * a loud, greppable line in the journal +# * a machine-readable status file the dashboard can surface later +# * a non-zero exit, which makes the systemd unit go failed and show up in +# `systemctl --failed` +# +# It also attempts recovery, because the common case (a unit that hit its +# restart limit and gave up) is fixed by exactly two commands and waiting weeks +# for a human to run them is the actual problem. +# +# Usage: +# scripts/health-check [--no-recover] + +set -uo pipefail + +cd "$(dirname "$0")/.." +# shellcheck source=scripts/lib/site-checks.sh +. "$(dirname "$0")/lib/site-checks.sh" + +STATUS_FILE="${STATUS_FILE:-${SITE_DIR}/data/health-status.json}" +recover=1 +[ "${1:-}" = "--no-recover" ] && recover=0 + +log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } + +rc=0 +recovered=() +still_broken=() + +# ── binaries ────────────────────────────────────────────────────────────────── +check_binaries; bin_rc=$? +if [ $bin_rc -ne 0 ]; then + rc=1 + if [ $recover -eq 1 ]; then + # A lost execute bit is recoverable in place and is the known failure + # mode here, so fix it rather than just reporting it. + for f in "${SITE_BINARIES[@]}"; do + if [ -f "$f" ] && [ ! -x "$f" ]; then + log "HEALTH recovering: chmod +x $f" + chmod +x "$f" && recovered+=("chmod +x $f") + fi + done + fi +fi + +# ── services ────────────────────────────────────────────────────────────────── +# Called directly, NOT in a command substitution: check_services sets +# FAILED_SERVICES, and a subshell would populate the array then throw it away, +# leaving the recovery loop below with nothing to do. +check_services; svc_rc=$? +if [ $svc_rc -ne 0 ]; then + rc=1 + if [ $recover -eq 1 ]; then + for s in "${FAILED_SERVICES[@]}"; do + log "HEALTH recovering: restarting $s" + systemctl reset-failed "$s" 2>/dev/null || true + if systemctl start "$s" 2>/dev/null; then + sleep 3 + if [ "$(systemctl is-active "$s" 2>/dev/null || true)" = "active" ]; then + log "HEALTH recovered: $s is active" + recovered+=("restarted $s") + else + log "HEALTH ALERT: $s still not active after restart" + still_broken+=("$s") + fi + else + log "HEALTH ALERT: failed to start $s" + still_broken+=("$s") + fi + done + fi +fi + +# ── routes, re-checked after any recovery ───────────────────────────────────── +check_routes; route_rc=$? +[ $route_rc -ne 0 ] && rc=1 + +# A recovered site is healthy now even though it was not a moment ago. Re-run +# the checks so the final verdict describes the current state, not the state we +# walked in on. +if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then + echo "==> Re-checking after recovery..." + rc=0 + check_binaries >/dev/null || rc=1 + check_services >/dev/null || rc=1 + check_routes >/dev/null || rc=1 +fi + +# ── status file ─────────────────────────────────────────────────────────────── +mkdir -p "$(dirname "$STATUS_FILE")" 2>/dev/null || true +{ + printf '{\n' + printf ' "checked_at": "%s",\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf ' "healthy": %s,\n' "$([ $rc -eq 0 ] && echo true || echo false)" + printf ' "recovered": [' + for i in "${!recovered[@]}"; do + [ "$i" -gt 0 ] && printf ', ' + printf '"%s"' "${recovered[$i]}" + done + printf '],\n' + printf ' "still_broken": [' + for i in "${!still_broken[@]}"; do + [ "$i" -gt 0 ] && printf ', ' + printf '"%s"' "${still_broken[$i]}" + done + printf ']\n}\n' +} > "$STATUS_FILE" 2>/dev/null || log "HEALTH warn: could not write $STATUS_FILE" + +if [ $rc -eq 0 ]; then + if [ ${#recovered[@]} -gt 0 ]; then + log "HEALTH ALERT recovered: ${recovered[*]}" + # Deliberately non-zero is NOT used here: the site is healthy now. The + # ALERT line in the journal is the durable record that something had + # broken and was silently fixed. + fi + exit 0 +fi + +log "HEALTH ALERT: doot.terst.org is unhealthy; still broken: ${still_broken[*]:-see checks above}" +exit 1 -- cgit v1.2.3