Fold the DB-ping health check in; delete the watchdog that never ran

scripts/health-watchdog.sh was written after the 2026-08-04 incident, where a
liveness stub stayed green for three days while every database-touching request
path was wedged. Its header says "Meant to run from cron every few minutes". It
was never scheduled — not in any unit, not in any crontab — so it has been dead
code since August, offering coverage it did not provide.

Its one real idea is worth keeping: /health pings the database and returns 503
when that fails, which the gateway route checks cannot see, because `/` answers
303 to the login page without going near the database.

check_app_health now does that poll (two attempts, five seconds apart, so a
single blip is not an outage), verify-deploy asserts it, and health-check
recovers from it by restarting the main service — the action the old watchdog
would have taken. The dead script is removed rather than left to imply cover.

Tests now 11, including a stub server returning 503 to prove a listening-but-
wedged app is caught.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018AwnALZL6bD4VBtATDVv3B

5 files changed, 94 insertions, 36 deletions

diff --git a/scripts/health-check b/scripts/health-check
index b1c34d0..29481ce 100755
--- a/scripts/health-check
+++ b/scripts/health-check
@@ -84,15 +84,39 @@ fi
 check_routes; route_rc=$?
 [ $route_rc -ne 0 ] && rc=1
 
+# ── app health (database ping) ────────────────────────────────────────────────
+# Separate from the route checks: the app can be listening and answering
+# redirects while every database-touching path is wedged. That is what happened
+# on 2026-08-04, undetected for three days. Recovery is a restart of the main
+# service, which is what the (never-scheduled) health-watchdog.sh this replaces
+# was written to do.
+check_app_health; health_rc=$?
+if [ $health_rc -ne 0 ]; then
+    rc=1
+    if [ $recover -eq 1 ]; then
+        log "HEALTH recovering: restarting task-dashboard@doot.terst.org.service (db ping failing)"
+        systemctl restart task-dashboard@doot.terst.org.service 2>/dev/null || true
+        sleep 5
+        if check_app_health >/dev/null; then
+            log "HEALTH recovered: app health restored after restart"
+            recovered+=("restarted task-dashboard (db ping)")
+        else
+            log "HEALTH ALERT: app health still failing after restart"
+            still_broken+=("task-dashboard@doot.terst.org.service (db ping)")
+        fi
+    fi
+fi
+
 # A recovered site is healthy now even though it was not a moment ago. Re-run
 # the checks so the final verdict describes the current state, not the state we
 # walked in on.
 if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then
     echo "==> Re-checking after recovery..."
     rc=0
-    check_binaries >/dev/null || rc=1
-    check_services >/dev/null || rc=1
-    check_routes   >/dev/null || rc=1
+    check_binaries   >/dev/null || rc=1
+    check_services   >/dev/null || rc=1
+    check_routes     >/dev/null || rc=1
+    check_app_health >/dev/null || rc=1
 fi
 
 # ── status file ───────────────────────────────────────────────────────────────
diff --git a/scripts/health-watchdog.sh b/scripts/health-watchdog.sh
deleted file mode 100755
index 9fde6a5..0000000
--- a/scripts/health-watchdog.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# Health watchdog for the doot server.
-#
-# /health pings the database (see cmd/dashboard/main.go) rather than
-# unconditionally returning 200 -- it used to be a liveness stub that
-# stayed green for three days during the 2026-08-04 incident while every
-# DB-touching request path was wedged. This polls it and restarts the
-# service if it's stuck. Meant to run from cron every few minutes.
-#
-# Usage: ./scripts/health-watchdog.sh
-
-APP_URL="http://127.0.0.1:38080/health"
-SERVICE="task-dashboard@doot.terst.org.service"
-LOG="/var/log/doot-health-watchdog.log"
-TIMEOUT=5
-
-check() {
-    curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" "$APP_URL" 2>/dev/null
-}
-
-# Two attempts, 5s apart -- don't restart on a single transient blip.
-code=$(check)
-if [ "$code" != "200" ]; then
-    sleep 5
-    code=$(check)
-fi
-if [ "$code" = "200" ]; then
-    exit 0
-fi
-
-echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') UNHEALTHY (http_code=${code:-timeout}) -- restarting ${SERVICE}" >> "$LOG"
-systemctl restart "$SERVICE"
-echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') restart issued" >> "$LOG"
diff --git a/scripts/lib/site-checks.sh b/scripts/lib/site-checks.sh
index f6ac55c..4f46acd 100644
--- a/scripts/lib/site-checks.sh
+++ b/scripts/lib/site-checks.sh
@@ -108,6 +108,30 @@ check_routes() {
     return $rc
 }
 
+# Is the app actually serving, not merely listening?
+#
+# /health pings the database rather than returning an unconditional 200. During
+# the 2026-08-04 incident a liveness stub stayed green for three days while
+# every DB-touching request path was wedged, which is why it was changed. The
+# gateway route checks above would not catch that: `/` answers 303 to the login
+# page without going near the database.
+APP_HEALTH_URL="${APP_HEALTH_URL:-http://127.0.0.1:38080/health}"
+check_app_health() {
+    local code
+    # Two attempts a few seconds apart — a single blip should not be reported as
+    # an outage, and the caller may act on this by restarting the service.
+    for attempt in 1 2; do
+        if ! code=$(curl -sS -o /dev/null --max-time 10 -w '%{http_code}' "$APP_HEALTH_URL" 2>/dev/null); then
+            code="000"
+        fi
+        code=${code:-000}
+        [ "$code" = "200" ] && { _chk_pass "app health (db ping) -> 200"; return 0; }
+        [ "$attempt" = "1" ] && sleep 5
+    done
+    _chk_fail "app health (db ping) -> ${code}"
+    return 1
+}
+
 # Does the running binary report the commit we expect? Only meaningful right
 # after a deploy, so it is optional.
 check_build_stamp() {
diff --git a/scripts/test-health-checks.sh b/scripts/test-health-checks.sh
index fbe9e47..fa59d8b 100755
--- a/scripts/test-health-checks.sh
+++ b/scripts/test-health-checks.sh
@@ -121,6 +121,46 @@ else
     bad "subshell semantics" "array unexpectedly survived a subshell"
 fi
 
+printf '=== check_app_health ===\n'
+
+# Unreachable app: must fail, not be mistaken for healthy.
+APP_HEALTH_URL="http://127.0.0.1:9/health"
+out=$(check_app_health); rc=$?
+if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "app health"; then
+    ok "catches an unreachable app health endpoint"
+else
+    bad "unreachable health endpoint" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
+fi
+
+# A 503 is what /health returns when the database is unreachable — the app is
+# listening and answering, so only this check can see it.
+python3 - "$tmp/hport" <<'PY2' &
+import http.server, socketserver, sys
+class H(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        self.send_response(503); self.end_headers()
+        self.wfile.write(b"db unreachable")
+    def log_message(self, *a): pass
+with socketserver.TCPServer(("127.0.0.1", 0), H) as s:
+    open(sys.argv[1], "w").write(str(s.server_address[1]))
+    s.serve_forever()
+PY2
+hpid=$!
+for _ in $(seq 1 50); do [ -s "$tmp/hport" ] && break; sleep 0.1; done
+hport=$(cat "$tmp/hport" 2>/dev/null || echo "")
+if [ -n "$hport" ]; then
+    APP_HEALTH_URL="http://127.0.0.1:${hport}/health"
+    out=$(check_app_health); rc=$?
+    if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "503"; then
+        ok "catches a listening app whose database ping fails (503)"
+    else
+        bad "503 db-ping detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
+    fi
+else
+    bad "503 db-ping detection" "could not start stub server"
+fi
+kill "$hpid" 2>/dev/null; wait "$hpid" 2>/dev/null
+
 printf '=== check_build_stamp ===\n'
 
 out=$(check_build_stamp "definitely-not-the-running-commit"); rc=$?
diff --git a/scripts/verify-deploy b/scripts/verify-deploy
index 712c26d..db3b1d8 100755
--- a/scripts/verify-deploy
+++ b/scripts/verify-deploy
@@ -32,6 +32,9 @@ check_services || rc=1
 echo "==> Verifying gateway routes..."
 check_routes || rc=1
 
+echo "==> Verifying app health (database reachable)..."
+check_app_health || rc=1
+
 if [ -n "$expected_build" ]; then
     echo "==> Verifying deployed build..."
     check_build_stamp "$expected_build" || rc=1