#!/bin/bash
#
# Periodic health check for doot.terst.org and its sub-services, with recovery.
#
# The failure this exists for: doot itself stays green while a sub-service
# behind the gateway is dead, so /scout/ returns 502 and nothing anywhere says
# so. Scout and hawaii were down for three and a half weeks, and claudomator
# crash-looped 426,000 times, before anyone opened the page and noticed.
#
# There is no mail/ntfy/apprise on this host, so "alerting" here means:
#   * a loud, greppable line in the journal
#   * a machine-readable status file the dashboard can surface later
#   * a non-zero exit, which makes the systemd unit go failed and show up in
#     `systemctl --failed`
#
# It also attempts recovery, because the common case (a unit that hit its
# restart limit and gave up) is fixed by exactly two commands and waiting weeks
# for a human to run them is the actual problem.
#
# Usage:
#   scripts/health-check [--no-recover]

set -uo pipefail

cd "$(dirname "$0")/.."
# shellcheck source=scripts/lib/site-checks.sh
. "$(dirname "$0")/lib/site-checks.sh"

STATUS_FILE="${STATUS_FILE:-${SITE_DIR}/data/health-status.json}"
recover=1
[ "${1:-}" = "--no-recover" ] && recover=0

log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }

rc=0
recovered=()
still_broken=()

# ── binaries ──────────────────────────────────────────────────────────────────
check_binaries; bin_rc=$?
if [ $bin_rc -ne 0 ]; then
    rc=1
    if [ $recover -eq 1 ]; then
        # A lost execute bit is recoverable in place and is the known failure
        # mode here, so fix it rather than just reporting it.
        for f in "${SITE_BINARIES[@]}"; do
            if [ -f "$f" ] && [ ! -x "$f" ]; then
                log "HEALTH recovering: chmod +x $f"
                chmod +x "$f" && recovered+=("chmod +x $f")
            fi
        done
    fi
fi

# ── services ──────────────────────────────────────────────────────────────────
# Called directly, NOT in a command substitution: check_services sets
# FAILED_SERVICES, and a subshell would populate the array then throw it away,
# leaving the recovery loop below with nothing to do.
check_services; svc_rc=$?
if [ $svc_rc -ne 0 ]; then
    rc=1
    if [ $recover -eq 1 ]; then
        for s in "${FAILED_SERVICES[@]}"; do
            log "HEALTH recovering: restarting $s"
            systemctl reset-failed "$s" 2>/dev/null || true
            if systemctl start "$s" 2>/dev/null; then
                sleep 3
                if [ "$(systemctl is-active "$s" 2>/dev/null || true)" = "active" ]; then
                    log "HEALTH recovered: $s is active"
                    recovered+=("restarted $s")
                else
                    log "HEALTH ALERT: $s still not active after restart"
                    still_broken+=("$s")
                fi
            else
                log "HEALTH ALERT: failed to start $s"
                still_broken+=("$s")
            fi
        done
    fi
fi

# ── routes, re-checked after any recovery ─────────────────────────────────────
check_routes; route_rc=$?
[ $route_rc -ne 0 ] && rc=1

# ── app health (database ping) ────────────────────────────────────────────────
# Separate from the route checks: the app can be listening and answering
# redirects while every database-touching path is wedged. That is what happened
# on 2026-08-04, undetected for three days. Recovery is a restart of the main
# service, which is what the (never-scheduled) health-watchdog.sh this replaces
# was written to do.
check_app_health; health_rc=$?
if [ $health_rc -ne 0 ]; then
    rc=1
    if [ $recover -eq 1 ]; then
        log "HEALTH recovering: restarting task-dashboard@doot.terst.org.service (db ping failing)"
        systemctl restart task-dashboard@doot.terst.org.service 2>/dev/null || true
        sleep 5
        if check_app_health >/dev/null; then
            log "HEALTH recovered: app health restored after restart"
            recovered+=("restarted task-dashboard (db ping)")
        else
            log "HEALTH ALERT: app health still failing after restart"
            still_broken+=("task-dashboard@doot.terst.org.service (db ping)")
        fi
    fi
fi

# A recovered site is healthy now even though it was not a moment ago. Re-run
# the checks so the final verdict describes the current state, not the state we
# walked in on.
if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then
    echo "==> Re-checking after recovery..."
    rc=0
    check_binaries   >/dev/null || rc=1
    check_services   >/dev/null || rc=1
    check_routes     >/dev/null || rc=1
    check_app_health >/dev/null || rc=1
fi

# ── status file ───────────────────────────────────────────────────────────────
mkdir -p "$(dirname "$STATUS_FILE")" 2>/dev/null || true
{
    printf '{\n'
    printf '  "checked_at": "%s",\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
    printf '  "healthy": %s,\n' "$([ $rc -eq 0 ] && echo true || echo false)"
    printf '  "recovered": ['
    for i in "${!recovered[@]}"; do
        [ "$i" -gt 0 ] && printf ', '
        printf '"%s"' "${recovered[$i]}"
    done
    printf '],\n'
    printf '  "still_broken": ['
    for i in "${!still_broken[@]}"; do
        [ "$i" -gt 0 ] && printf ', '
        printf '"%s"' "${still_broken[$i]}"
    done
    printf ']\n}\n'
} > "$STATUS_FILE" 2>/dev/null || log "HEALTH warn: could not write $STATUS_FILE"

if [ $rc -eq 0 ]; then
    if [ ${#recovered[@]} -gt 0 ]; then
        log "HEALTH ALERT recovered: ${recovered[*]}"
        # Deliberately non-zero is NOT used here: the site is healthy now. The
        # ALERT line in the journal is the durable record that something had
        # broken and was silently fixed.
    fi
    exit 0
fi

log "HEALTH ALERT: doot.terst.org is unhealthy; still broken: ${still_broken[*]:-see checks above}"
exit 1