1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
#!/bin/bash
#
# Periodic health check for doot.terst.org and its sub-services, with recovery.
#
# The failure this exists for: doot itself stays green while a sub-service
# behind the gateway is dead, so /scout/ returns 502 and nothing anywhere says
# so. Scout and hawaii were down for three and a half weeks, and claudomator
# crash-looped 426,000 times, before anyone opened the page and noticed.
#
# There is no mail/ntfy/apprise on this host, so "alerting" here means:
# * a loud, greppable line in the journal
# * a machine-readable status file the dashboard can surface later
# * a non-zero exit, which makes the systemd unit go failed and show up in
# `systemctl --failed`
#
# It also attempts recovery, because the common case (a unit that hit its
# restart limit and gave up) is fixed by exactly two commands and waiting weeks
# for a human to run them is the actual problem.
#
# Usage:
# scripts/health-check [--no-recover]
set -uo pipefail
cd "$(dirname "$0")/.."
# shellcheck source=scripts/lib/site-checks.sh
. "$(dirname "$0")/lib/site-checks.sh"
STATUS_FILE="${STATUS_FILE:-${SITE_DIR}/data/health-status.json}"
recover=1
[ "${1:-}" = "--no-recover" ] && recover=0
log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
rc=0
recovered=()
still_broken=()
# ── binaries ──────────────────────────────────────────────────────────────────
check_binaries; bin_rc=$?
if [ $bin_rc -ne 0 ]; then
rc=1
if [ $recover -eq 1 ]; then
# A lost execute bit is recoverable in place and is the known failure
# mode here, so fix it rather than just reporting it.
for f in "${SITE_BINARIES[@]}"; do
if [ -f "$f" ] && [ ! -x "$f" ]; then
log "HEALTH recovering: chmod +x $f"
chmod +x "$f" && recovered+=("chmod +x $f")
fi
done
fi
fi
# ── services ──────────────────────────────────────────────────────────────────
# Called directly, NOT in a command substitution: check_services sets
# FAILED_SERVICES, and a subshell would populate the array then throw it away,
# leaving the recovery loop below with nothing to do.
check_services; svc_rc=$?
if [ $svc_rc -ne 0 ]; then
rc=1
if [ $recover -eq 1 ]; then
for s in "${FAILED_SERVICES[@]}"; do
log "HEALTH recovering: restarting $s"
systemctl reset-failed "$s" 2>/dev/null || true
if systemctl start "$s" 2>/dev/null; then
sleep 3
if [ "$(systemctl is-active "$s" 2>/dev/null || true)" = "active" ]; then
log "HEALTH recovered: $s is active"
recovered+=("restarted $s")
else
log "HEALTH ALERT: $s still not active after restart"
still_broken+=("$s")
fi
else
log "HEALTH ALERT: failed to start $s"
still_broken+=("$s")
fi
done
fi
fi
# ── routes, re-checked after any recovery ─────────────────────────────────────
check_routes; route_rc=$?
[ $route_rc -ne 0 ] && rc=1
# A recovered site is healthy now even though it was not a moment ago. Re-run
# the checks so the final verdict describes the current state, not the state we
# walked in on.
if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then
echo "==> Re-checking after recovery..."
rc=0
check_binaries >/dev/null || rc=1
check_services >/dev/null || rc=1
check_routes >/dev/null || rc=1
fi
# ── status file ───────────────────────────────────────────────────────────────
mkdir -p "$(dirname "$STATUS_FILE")" 2>/dev/null || true
{
printf '{\n'
printf ' "checked_at": "%s",\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
printf ' "healthy": %s,\n' "$([ $rc -eq 0 ] && echo true || echo false)"
printf ' "recovered": ['
for i in "${!recovered[@]}"; do
[ "$i" -gt 0 ] && printf ', '
printf '"%s"' "${recovered[$i]}"
done
printf '],\n'
printf ' "still_broken": ['
for i in "${!still_broken[@]}"; do
[ "$i" -gt 0 ] && printf ', '
printf '"%s"' "${still_broken[$i]}"
done
printf ']\n}\n'
} > "$STATUS_FILE" 2>/dev/null || log "HEALTH warn: could not write $STATUS_FILE"
if [ $rc -eq 0 ]; then
if [ ${#recovered[@]} -gt 0 ]; then
log "HEALTH ALERT recovered: ${recovered[*]}"
# Deliberately non-zero is NOT used here: the site is healthy now. The
# ALERT line in the journal is the durable record that something had
# broken and was silently fixed.
fi
exit 0
fi
log "HEALTH ALERT: doot.terst.org is unhealthy; still broken: ${still_broken[*]:-see checks above}"
exit 1
|
