From 36a82d934366cc2ddbd92ee3885b1c547eeb8244 Mon Sep 17 00:00:00 2001 From: Peter Stone Date: Sun, 20 Sep 2026 00:36:17 +0000 Subject: Fold the DB-ping health check in; delete the watchdog that never ran MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/health-watchdog.sh was written after the 2026-08-04 incident, where a liveness stub stayed green for three days while every database-touching request path was wedged. Its header says "Meant to run from cron every few minutes". It was never scheduled — not in any unit, not in any crontab — so it has been dead code since August, offering coverage it did not provide. Its one real idea is worth keeping: /health pings the database and returns 503 when that fails, which the gateway route checks cannot see, because `/` answers 303 to the login page without going near the database. check_app_health now does that poll (two attempts, five seconds apart, so a single blip is not an outage), verify-deploy asserts it, and health-check recovers from it by restarting the main service — the action the old watchdog would have taken. The dead script is removed rather than left to imply cover. Tests now 11, including a stub server returning 503 to prove a listening-but- wedged app is caught. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B --- scripts/lib/site-checks.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'scripts/lib/site-checks.sh') diff --git a/scripts/lib/site-checks.sh b/scripts/lib/site-checks.sh index f6ac55c..4f46acd 100644 --- a/scripts/lib/site-checks.sh +++ b/scripts/lib/site-checks.sh @@ -108,6 +108,30 @@ check_routes() { return $rc } +# Is the app actually serving, not merely listening? +# +# /health pings the database rather than returning an unconditional 200. During +# the 2026-08-04 incident a liveness stub stayed green for three days while +# every DB-touching request path was wedged, which is why it was changed. The +# gateway route checks above would not catch that: `/` answers 303 to the login +# page without going near the database. +APP_HEALTH_URL="${APP_HEALTH_URL:-http://127.0.0.1:38080/health}" +check_app_health() { + local code + # Two attempts a few seconds apart — a single blip should not be reported as + # an outage, and the caller may act on this by restarting the service. + for attempt in 1 2; do + if ! code=$(curl -sS -o /dev/null --max-time 10 -w '%{http_code}' "$APP_HEALTH_URL" 2>/dev/null); then + code="000" + fi + code=${code:-000} + [ "$code" = "200" ] && { _chk_pass "app health (db ping) -> 200"; return 0; } + [ "$attempt" = "1" ] && sleep 5 + done + _chk_fail "app health (db ping) -> ${code}" + return 1 +} + # Does the running binary report the commit we expect? Only meaningful right # after a deploy, so it is optional. check_build_stamp() { -- cgit v1.2.3