1 files changed, 33 insertions, 0 deletions
diff --git a/scripts/health-watchdog.sh b/scripts/health-watchdog.sh
new file mode 100755
index 0000000..9fde6a5
--- /dev/null
+++ b/scripts/health-watchdog.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Health watchdog for the doot server.
+#
+# /health pings the database (see cmd/dashboard/main.go) rather than
+# unconditionally returning 200 -- it used to be a liveness stub that
+# stayed green for three days during the 2026-08-04 incident while every
+# DB-touching request path was wedged. This polls it and restarts the
+# service if it's stuck. Meant to run from cron every few minutes.
+#
+# Usage: ./scripts/health-watchdog.sh
+
+APP_URL="http://127.0.0.1:38080/health"
+SERVICE="task-dashboard@doot.terst.org.service"
+LOG="/var/log/doot-health-watchdog.log"
+TIMEOUT=5
+
+check() {
+    curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" "$APP_URL" 2>/dev/null
+}
+
+# Two attempts, 5s apart -- don't restart on a single transient blip.
+code=$(check)
+if [ "$code" != "200" ]; then
+    sleep 5
+    code=$(check)
+fi
+if [ "$code" = "200" ]; then
+    exit 0
+fi
+
+echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') UNHEALTHY (http_code=${code:-timeout}) -- restarting ${SERVICE}" >> "$LOG"
+systemctl restart "$SERVICE"
+echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') restart issued" >> "$LOG"