blob: 9fde6a56e09e2510c13e8c837aa58060728b9a25 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
#!/bin/bash
# Health watchdog for the doot server.
#
# /health pings the database (see cmd/dashboard/main.go) rather than
# unconditionally returning 200 -- it used to be a liveness stub that
# stayed green for three days during the 2026-08-04 incident while every
# DB-touching request path was wedged. This polls it and restarts the
# service if it's stuck. Meant to run from cron every few minutes.
#
# Usage: ./scripts/health-watchdog.sh
APP_URL="http://127.0.0.1:38080/health"
SERVICE="task-dashboard@doot.terst.org.service"
LOG="/var/log/doot-health-watchdog.log"
TIMEOUT=5
check() {
curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" "$APP_URL" 2>/dev/null
}
# Two attempts, 5s apart -- don't restart on a single transient blip.
code=$(check)
if [ "$code" != "200" ]; then
sleep 5
code=$(check)
fi
if [ "$code" = "200" ]; then
exit 0
fi
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') UNHEALTHY (http_code=${code:-timeout}) -- restarting ${SERVICE}" >> "$LOG"
systemctl restart "$SERVICE"
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') restart issued" >> "$LOG"
|
