1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#!/bin/bash
#
# Periodic health check for doot.terst.org and its sub-services, with recovery.
#
# The failure this exists for: doot itself stays green while a sub-service
# behind the gateway is dead, so /scout/ returns 502 and nothing anywhere says
# so. Scout and hawaii were down for three and a half weeks, and claudomator
# crash-looped 426,000 times, before anyone opened the page and noticed.
#
# There is no mail/ntfy/apprise on this host, so "alerting" here means:
# * a loud, greppable line in the journal
# * a machine-readable status file the dashboard can surface later
# * a non-zero exit, which makes the systemd unit go failed and show up in
# `systemctl --failed`
#
# It also attempts recovery, because the common case (a unit that hit its
# restart limit and gave up) is fixed by exactly two commands and waiting weeks
# for a human to run them is the actual problem.
#
# Usage:
# scripts/health-check [--no-recover]
set -uo pipefail
cd "$(dirname "$0")/.."
# shellcheck source=scripts/lib/site-checks.sh
. "$(dirname "$0")/lib/site-checks.sh"
STATUS_FILE="${STATUS_FILE:-${SITE_DIR}/data/health-status.json}"
recover=1
[ "${1:-}" = "--no-recover" ] && recover=0
log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
rc=0
recovered=()
still_broken=()
# ── binaries ──────────────────────────────────────────────────────────────────
check_binaries; bin_rc=$?
if [ $bin_rc -ne 0 ]; then
rc=1
if [ $recover -eq 1 ]; then
# A lost execute bit is recoverable in place and is the known failure
# mode here, so fix it rather than just reporting it.
for f in "${SITE_BINARIES[@]}"; do
if [ -f "$f" ] && [ ! -x "$f" ]; then
log "HEALTH recovering: chmod +x $f"
chmod +x "$f" && recovered+=("chmod +x $f")
fi
done
fi
fi
# ── services ──────────────────────────────────────────────────────────────────
# Called directly, NOT in a command substitution: check_services sets
# FAILED_SERVICES, and a subshell would populate the array then throw it away,
# leaving the recovery loop below with nothing to do.
check_services; svc_rc=$?
if [ $svc_rc -ne 0 ]; then
rc=1
if [ $recover -eq 1 ]; then
for s in "${FAILED_SERVICES[@]}"; do
log "HEALTH recovering: restarting $s"
systemctl reset-failed "$s" 2>/dev/null || true
if systemctl start "$s" 2>/dev/null; then
sleep 3
if [ "$(systemctl is-active "$s" 2>/dev/null || true)" = "active" ]; then
log "HEALTH recovered: $s is active"
recovered+=("restarted $s")
else
log "HEALTH ALERT: $s still not active after restart"
still_broken+=("$s")
fi
else
log "HEALTH ALERT: failed to start $s"
still_broken+=("$s")
fi
done
fi
fi
# ── routes, re-checked after any recovery ─────────────────────────────────────
check_routes; route_rc=$?
[ $route_rc -ne 0 ] && rc=1
# ── app health (database ping) ────────────────────────────────────────────────
# Separate from the route checks: the app can be listening and answering
# redirects while every database-touching path is wedged. That is what happened
# on 2026-08-04, undetected for three days. Recovery is a restart of the main
# service, which is what the (never-scheduled) health-watchdog.sh this replaces
# was written to do.
check_app_health; health_rc=$?
if [ $health_rc -ne 0 ]; then
rc=1
if [ $recover -eq 1 ]; then
log "HEALTH recovering: restarting task-dashboard@doot.terst.org.service (db ping failing)"
systemctl restart task-dashboard@doot.terst.org.service 2>/dev/null || true
sleep 5
if check_app_health >/dev/null; then
log "HEALTH recovered: app health restored after restart"
recovered+=("restarted task-dashboard (db ping)")
else
log "HEALTH ALERT: app health still failing after restart"
still_broken+=("task-dashboard@doot.terst.org.service (db ping)")
fi
fi
fi
# A recovered site is healthy now even though it was not a moment ago. Re-run
# the checks so the final verdict describes the current state, not the state we
# walked in on.
if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then
echo "==> Re-checking after recovery..."
rc=0
check_binaries >/dev/null || rc=1
check_services >/dev/null || rc=1
check_routes >/dev/null || rc=1
check_app_health >/dev/null || rc=1
fi
# ── status file ───────────────────────────────────────────────────────────────
mkdir -p "$(dirname "$STATUS_FILE")" 2>/dev/null || true
{
printf '{\n'
printf ' "checked_at": "%s",\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
printf ' "healthy": %s,\n' "$([ $rc -eq 0 ] && echo true || echo false)"
printf ' "recovered": ['
for i in "${!recovered[@]}"; do
[ "$i" -gt 0 ] && printf ', '
printf '"%s"' "${recovered[$i]}"
done
printf '],\n'
printf ' "still_broken": ['
for i in "${!still_broken[@]}"; do
[ "$i" -gt 0 ] && printf ', '
printf '"%s"' "${still_broken[$i]}"
done
printf ']\n}\n'
} > "$STATUS_FILE" 2>/dev/null || log "HEALTH warn: could not write $STATUS_FILE"
if [ $rc -eq 0 ]; then
if [ ${#recovered[@]} -gt 0 ]; then
log "HEALTH ALERT recovered: ${recovered[*]}"
# Deliberately non-zero is NOT used here: the site is healthy now. The
# ALERT line in the journal is the durable record that something had
# broken and was silently fixed.
fi
exit 0
fi
log "HEALTH ALERT: doot.terst.org is unhealthy; still broken: ${still_broken[*]:-see checks above}"
exit 1
|
