diff options
Diffstat (limited to 'scripts/health-check')
| -rwxr-xr-x | scripts/health-check | 30 |
1 files changed, 27 insertions, 3 deletions
diff --git a/scripts/health-check b/scripts/health-check index b1c34d0..29481ce 100755 --- a/scripts/health-check +++ b/scripts/health-check @@ -84,15 +84,39 @@ fi check_routes; route_rc=$? [ $route_rc -ne 0 ] && rc=1 +# ── app health (database ping) ──────────────────────────────────────────────── +# Separate from the route checks: the app can be listening and answering +# redirects while every database-touching path is wedged. That is what happened +# on 2026-08-04, undetected for three days. Recovery is a restart of the main +# service, which is what the (never-scheduled) health-watchdog.sh this replaces +# was written to do. +check_app_health; health_rc=$? +if [ $health_rc -ne 0 ]; then + rc=1 + if [ $recover -eq 1 ]; then + log "HEALTH recovering: restarting task-dashboard@doot.terst.org.service (db ping failing)" + systemctl restart task-dashboard@doot.terst.org.service 2>/dev/null || true + sleep 5 + if check_app_health >/dev/null; then + log "HEALTH recovered: app health restored after restart" + recovered+=("restarted task-dashboard (db ping)") + else + log "HEALTH ALERT: app health still failing after restart" + still_broken+=("task-dashboard@doot.terst.org.service (db ping)") + fi + fi +fi + # A recovered site is healthy now even though it was not a moment ago. Re-run # the checks so the final verdict describes the current state, not the state we # walked in on. if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then echo "==> Re-checking after recovery..." rc=0 - check_binaries >/dev/null || rc=1 - check_services >/dev/null || rc=1 - check_routes >/dev/null || rc=1 + check_binaries >/dev/null || rc=1 + check_services >/dev/null || rc=1 + check_routes >/dev/null || rc=1 + check_app_health >/dev/null || rc=1 fi # ── status file ─────────────────────────────────────────────────────────────── |