diff options
| -rwxr-xr-x | scripts/health-check | 30 | ||||
| -rwxr-xr-x | scripts/health-watchdog.sh | 33 | ||||
| -rw-r--r-- | scripts/lib/site-checks.sh | 24 | ||||
| -rwxr-xr-x | scripts/test-health-checks.sh | 40 | ||||
| -rwxr-xr-x | scripts/verify-deploy | 3 |
5 files changed, 94 insertions, 36 deletions
diff --git a/scripts/health-check b/scripts/health-check index b1c34d0..29481ce 100755 --- a/scripts/health-check +++ b/scripts/health-check @@ -84,15 +84,39 @@ fi check_routes; route_rc=$? [ $route_rc -ne 0 ] && rc=1 +# ── app health (database ping) ──────────────────────────────────────────────── +# Separate from the route checks: the app can be listening and answering +# redirects while every database-touching path is wedged. That is what happened +# on 2026-08-04, undetected for three days. Recovery is a restart of the main +# service, which is what the (never-scheduled) health-watchdog.sh this replaces +# was written to do. +check_app_health; health_rc=$? +if [ $health_rc -ne 0 ]; then + rc=1 + if [ $recover -eq 1 ]; then + log "HEALTH recovering: restarting task-dashboard@doot.terst.org.service (db ping failing)" + systemctl restart task-dashboard@doot.terst.org.service 2>/dev/null || true + sleep 5 + if check_app_health >/dev/null; then + log "HEALTH recovered: app health restored after restart" + recovered+=("restarted task-dashboard (db ping)") + else + log "HEALTH ALERT: app health still failing after restart" + still_broken+=("task-dashboard@doot.terst.org.service (db ping)") + fi + fi +fi + # A recovered site is healthy now even though it was not a moment ago. Re-run # the checks so the final verdict describes the current state, not the state we # walked in on. if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then echo "==> Re-checking after recovery..." rc=0 - check_binaries >/dev/null || rc=1 - check_services >/dev/null || rc=1 - check_routes >/dev/null || rc=1 + check_binaries >/dev/null || rc=1 + check_services >/dev/null || rc=1 + check_routes >/dev/null || rc=1 + check_app_health >/dev/null || rc=1 fi # ── status file ─────────────────────────────────────────────────────────────── diff --git a/scripts/health-watchdog.sh b/scripts/health-watchdog.sh deleted file mode 100755 index 9fde6a5..0000000 --- a/scripts/health-watchdog.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Health watchdog for the doot server. -# -# /health pings the database (see cmd/dashboard/main.go) rather than -# unconditionally returning 200 -- it used to be a liveness stub that -# stayed green for three days during the 2026-08-04 incident while every -# DB-touching request path was wedged. This polls it and restarts the -# service if it's stuck. Meant to run from cron every few minutes. -# -# Usage: ./scripts/health-watchdog.sh - -APP_URL="http://127.0.0.1:38080/health" -SERVICE="task-dashboard@doot.terst.org.service" -LOG="/var/log/doot-health-watchdog.log" -TIMEOUT=5 - -check() { - curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" "$APP_URL" 2>/dev/null -} - -# Two attempts, 5s apart -- don't restart on a single transient blip. -code=$(check) -if [ "$code" != "200" ]; then - sleep 5 - code=$(check) -fi -if [ "$code" = "200" ]; then - exit 0 -fi - -echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') UNHEALTHY (http_code=${code:-timeout}) -- restarting ${SERVICE}" >> "$LOG" -systemctl restart "$SERVICE" -echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') restart issued" >> "$LOG" diff --git a/scripts/lib/site-checks.sh b/scripts/lib/site-checks.sh index f6ac55c..4f46acd 100644 --- a/scripts/lib/site-checks.sh +++ b/scripts/lib/site-checks.sh @@ -108,6 +108,30 @@ check_routes() { return $rc } +# Is the app actually serving, not merely listening? +# +# /health pings the database rather than returning an unconditional 200. During +# the 2026-08-04 incident a liveness stub stayed green for three days while +# every DB-touching request path was wedged, which is why it was changed. The +# gateway route checks above would not catch that: `/` answers 303 to the login +# page without going near the database. +APP_HEALTH_URL="${APP_HEALTH_URL:-http://127.0.0.1:38080/health}" +check_app_health() { + local code + # Two attempts a few seconds apart — a single blip should not be reported as + # an outage, and the caller may act on this by restarting the service. + for attempt in 1 2; do + if ! code=$(curl -sS -o /dev/null --max-time 10 -w '%{http_code}' "$APP_HEALTH_URL" 2>/dev/null); then + code="000" + fi + code=${code:-000} + [ "$code" = "200" ] && { _chk_pass "app health (db ping) -> 200"; return 0; } + [ "$attempt" = "1" ] && sleep 5 + done + _chk_fail "app health (db ping) -> ${code}" + return 1 +} + # Does the running binary report the commit we expect? Only meaningful right # after a deploy, so it is optional. check_build_stamp() { diff --git a/scripts/test-health-checks.sh b/scripts/test-health-checks.sh index fbe9e47..fa59d8b 100755 --- a/scripts/test-health-checks.sh +++ b/scripts/test-health-checks.sh @@ -121,6 +121,46 @@ else bad "subshell semantics" "array unexpectedly survived a subshell" fi +printf '=== check_app_health ===\n' + +# Unreachable app: must fail, not be mistaken for healthy. +APP_HEALTH_URL="http://127.0.0.1:9/health" +out=$(check_app_health); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "app health"; then + ok "catches an unreachable app health endpoint" +else + bad "unreachable health endpoint" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" +fi + +# A 503 is what /health returns when the database is unreachable — the app is +# listening and answering, so only this check can see it. +python3 - "$tmp/hport" <<'PY2' & +import http.server, socketserver, sys +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(503); self.end_headers() + self.wfile.write(b"db unreachable") + def log_message(self, *a): pass +with socketserver.TCPServer(("127.0.0.1", 0), H) as s: + open(sys.argv[1], "w").write(str(s.server_address[1])) + s.serve_forever() +PY2 +hpid=$! +for _ in $(seq 1 50); do [ -s "$tmp/hport" ] && break; sleep 0.1; done +hport=$(cat "$tmp/hport" 2>/dev/null || echo "") +if [ -n "$hport" ]; then + APP_HEALTH_URL="http://127.0.0.1:${hport}/health" + out=$(check_app_health); rc=$? + if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "503"; then + ok "catches a listening app whose database ping fails (503)" + else + bad "503 db-ping detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" + fi +else + bad "503 db-ping detection" "could not start stub server" +fi +kill "$hpid" 2>/dev/null; wait "$hpid" 2>/dev/null + printf '=== check_build_stamp ===\n' out=$(check_build_stamp "definitely-not-the-running-commit"); rc=$? diff --git a/scripts/verify-deploy b/scripts/verify-deploy index 712c26d..db3b1d8 100755 --- a/scripts/verify-deploy +++ b/scripts/verify-deploy @@ -32,6 +32,9 @@ check_services || rc=1 echo "==> Verifying gateway routes..." check_routes || rc=1 +echo "==> Verifying app health (database reachable)..." +check_app_health || rc=1 + if [ -n "$expected_build" ]; then echo "==> Verifying deployed build..." check_build_stamp "$expected_build" || rc=1 |