1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
#!/bin/bash
#
# Shared assertions about a healthy doot.terst.org, used by both
# scripts/verify-deploy and scripts/health-check so the two cannot drift apart.
#
# Source this; do not execute it.
#
# Every function returns non-zero on failure and prints one line per check, so
# callers can aggregate. Nothing here restarts or changes anything — recovery is
# the caller's decision.
SITE_DIR="${SITE_DIR:-/site/doot.terst.org}"
BASE_URL="${BASE_URL:-https://doot.terst.org}"
# Executables the site needs. A missing +x here is invisible until the next
# reboot, because a running process keeps executing from its open file — this is
# exactly how scout, hawaii, claudomator and claude were all disarmed at once and
# stayed down for three and a half weeks.
SITE_BINARIES=(
"${SITE_DIR}/app"
"${SITE_DIR}/bin/claudomator"
"${SITE_DIR}/bin/claude"
"${SITE_DIR}/services/scout/app"
"${SITE_DIR}/services/hawaii/app"
)
# systemd units that must be running.
SITE_SERVICES=(
"task-dashboard@doot.terst.org.service"
"scout.service"
"hawaii.service"
"claudomator@doot.terst.org.service"
)
# Gateway paths that must not return a server error. Checked unauthenticated, so
# a login redirect or the login page itself is a pass — what we are detecting is
# 502/503/504 from a dead upstream, which is what a down sub-service looks like
# from outside.
SITE_ROUTES=(
"/"
"/scout/"
"/hawaii/"
"/claudomator/"
)
_chk_pass() { printf ' ok %s\n' "$*"; }
_chk_fail() { printf ' FAIL %s\n' "$*"; }
# Are all expected binaries executable?
check_binaries() {
local rc=0 f
for f in "${SITE_BINARIES[@]}"; do
if [ ! -f "$f" ]; then
_chk_fail "missing binary: $f"; rc=1
elif [ ! -x "$f" ]; then
_chk_fail "not executable: $f ($(stat -c %A "$f"))"; rc=1
else
_chk_pass "executable: $f"
fi
done
return $rc
}
# Are all expected services active?
#
# Sets FAILED_SERVICES as a side effect. IMPORTANT: that means callers must NOT
# invoke this inside a command substitution — `out=$(check_services)` runs it in
# a subshell, the array is populated there and discarded, and the caller sees an
# empty list. health-check drives recovery off this array, so that mistake
# silently disables recovery entirely. Call it directly and redirect if you need
# the output: `check_services > file`.
FAILED_SERVICES=()
check_services() {
local rc=0 s state
FAILED_SERVICES=()
for s in "${SITE_SERVICES[@]}"; do
state=$(systemctl is-active "$s" 2>/dev/null || true)
if [ "$state" = "active" ]; then
_chk_pass "active: $s"
else
_chk_fail "not active: $s (state=$state)"
FAILED_SERVICES+=("$s")
rc=1
fi
done
return $rc
}
# Do all gateway routes answer without a server error?
check_routes() {
local rc=0 p code
for p in "${SITE_ROUTES[@]}"; do
# --max-time so a hung upstream fails the check instead of hanging it.
#
# Note the shape of this: `$(curl ... || echo 000)` would CONCATENATE
# curl's own "000" transport-error output with the echoed one, producing
# "000000", which then matches no case arm and gets reported as healthy.
# Assigning inside `if !` keeps exactly one value.
if ! code=$(curl -sS -o /dev/null --max-time 15 -w '%{http_code}' "${BASE_URL}${p}" 2>/dev/null); then
code="000"
fi
code=${code:-000}
case "$code" in
000|5*) _chk_fail "route ${p} -> ${code}"; rc=1 ;;
*) _chk_pass "route ${p} -> ${code}" ;;
esac
done
return $rc
}
# Is the app actually serving, not merely listening?
#
# /health pings the database rather than returning an unconditional 200. During
# the 2026-08-04 incident a liveness stub stayed green for three days while
# every DB-touching request path was wedged, which is why it was changed. The
# gateway route checks above would not catch that: `/` answers 303 to the login
# page without going near the database.
APP_HEALTH_URL="${APP_HEALTH_URL:-http://127.0.0.1:38080/health}"
check_app_health() {
local code
# Two attempts a few seconds apart — a single blip should not be reported as
# an outage, and the caller may act on this by restarting the service.
for attempt in 1 2; do
if ! code=$(curl -sS -o /dev/null --max-time 10 -w '%{http_code}' "$APP_HEALTH_URL" 2>/dev/null); then
code="000"
fi
code=${code:-000}
[ "$code" = "200" ] && { _chk_pass "app health (db ping) -> 200"; return 0; }
[ "$attempt" = "1" ] && sleep 5
done
_chk_fail "app health (db ping) -> ${code}"
return 1
}
# Does the running binary report the commit we expect? Only meaningful right
# after a deploy, so it is optional.
check_build_stamp() {
local expected=$1 seen
seen=$(journalctl -u "task-dashboard@doot.terst.org.service" --no-pager -n 50 2>/dev/null \
| grep -oE 'build=[0-9a-f]+' | tail -1 | cut -d= -f2 || true)
if [ -z "$seen" ]; then
_chk_fail "no build stamp found in recent journal"
return 1
fi
if [ "$seen" = "$expected" ]; then
_chk_pass "running build=$seen"
return 0
fi
_chk_fail "running build=$seen but expected $expected"
return 1
}
|
