#!/bin/bash
# Health watchdog for the doot server.
#
# /health pings the database (see cmd/dashboard/main.go) rather than
# unconditionally returning 200 -- it used to be a liveness stub that
# stayed green for three days during the 2026-08-04 incident while every
# DB-touching request path was wedged. This polls it and restarts the
# service if it's stuck. Meant to run from cron every few minutes.
#
# Usage: ./scripts/health-watchdog.sh

APP_URL="http://127.0.0.1:38080/health"
SERVICE="task-dashboard@doot.terst.org.service"
LOG="/var/log/doot-health-watchdog.log"
TIMEOUT=5

check() {
    curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" "$APP_URL" 2>/dev/null
}

# Two attempts, 5s apart -- don't restart on a single transient blip.
code=$(check)
if [ "$code" != "200" ]; then
    sleep 5
    code=$(check)
fi
if [ "$code" = "200" ]; then
    exit 0
fi

echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') UNHEALTHY (http_code=${code:-timeout}) -- restarting ${SERVICE}" >> "$LOG"
systemctl restart "$SERVICE"
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') restart issued" >> "$LOG"