1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#!/bin/bash
#
# Shared assertions about a healthy doot.terst.org, used by both
# scripts/verify-deploy and scripts/health-check so the two cannot drift apart.
#
# Source this; do not execute it.
#
# Every function returns non-zero on failure and prints one line per check, so
# callers can aggregate. Nothing here restarts or changes anything — recovery is
# the caller's decision.
SITE_DIR="${SITE_DIR:-/site/doot.terst.org}"
BASE_URL="${BASE_URL:-https://doot.terst.org}"
# Executables the site needs. A missing +x here is invisible until the next
# reboot, because a running process keeps executing from its open file — this is
# exactly how scout, hawaii, claudomator and claude were all disarmed at once and
# stayed down for three and a half weeks.
SITE_BINARIES=(
"${SITE_DIR}/app"
"${SITE_DIR}/bin/claudomator"
"${SITE_DIR}/bin/claude"
"${SITE_DIR}/services/scout/app"
"${SITE_DIR}/services/hawaii/app"
)
# systemd units that must be running.
SITE_SERVICES=(
"task-dashboard@doot.terst.org.service"
"scout.service"
"hawaii.service"
"claudomator@doot.terst.org.service"
)
# Gateway paths that must not return a server error. Checked unauthenticated, so
# a login redirect or the login page itself is a pass — what we are detecting is
# 502/503/504 from a dead upstream, which is what a down sub-service looks like
# from outside.
SITE_ROUTES=(
"/"
"/scout/"
"/hawaii/"
"/claudomator/"
)
_chk_pass() { printf ' ok %s\n' "$*"; }
_chk_fail() { printf ' FAIL %s\n' "$*"; }
# Are all expected binaries executable?
check_binaries() {
local rc=0 f
for f in "${SITE_BINARIES[@]}"; do
if [ ! -f "$f" ]; then
_chk_fail "missing binary: $f"; rc=1
elif [ ! -x "$f" ]; then
_chk_fail "not executable: $f ($(stat -c %A "$f"))"; rc=1
else
_chk_pass "executable: $f"
fi
done
return $rc
}
# Are all expected services active?
#
# Sets FAILED_SERVICES as a side effect. IMPORTANT: that means callers must NOT
# invoke this inside a command substitution — `out=$(check_services)` runs it in
# a subshell, the array is populated there and discarded, and the caller sees an
# empty list. health-check drives recovery off this array, so that mistake
# silently disables recovery entirely. Call it directly and redirect if you need
# the output: `check_services > file`.
FAILED_SERVICES=()
check_services() {
local rc=0 s state
FAILED_SERVICES=()
for s in "${SITE_SERVICES[@]}"; do
state=$(systemctl is-active "$s" 2>/dev/null || true)
if [ "$state" = "active" ]; then
_chk_pass "active: $s"
else
_chk_fail "not active: $s (state=$state)"
FAILED_SERVICES+=("$s")
rc=1
fi
done
return $rc
}
# Do all gateway routes answer without a server error?
check_routes() {
local rc=0 p code
for p in "${SITE_ROUTES[@]}"; do
# --max-time so a hung upstream fails the check instead of hanging it.
#
# Note the shape of this: `$(curl ... || echo 000)` would CONCATENATE
# curl's own "000" transport-error output with the echoed one, producing
# "000000", which then matches no case arm and gets reported as healthy.
# Assigning inside `if !` keeps exactly one value.
if ! code=$(curl -sS -o /dev/null --max-time 15 -w '%{http_code}' "${BASE_URL}${p}" 2>/dev/null); then
code="000"
fi
code=${code:-000}
case "$code" in
000|5*) _chk_fail "route ${p} -> ${code}"; rc=1 ;;
*) _chk_pass "route ${p} -> ${code}" ;;
esac
done
return $rc
}
# Does the running binary report the commit we expect? Only meaningful right
# after a deploy, so it is optional.
check_build_stamp() {
local expected=$1 seen
seen=$(journalctl -u "task-dashboard@doot.terst.org.service" --no-pager -n 50 2>/dev/null \
| grep -oE 'build=[0-9a-f]+' | tail -1 | cut -d= -f2 || true)
if [ -z "$seen" ]; then
_chk_fail "no build stamp found in recent journal"
return 1
fi
if [ "$seen" = "$expected" ]; then
_chk_pass "running build=$seen"
return 0
fi
_chk_fail "running build=$seen but expected $expected"
return 1
}
|
