1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
#!/bin/bash
#
# Tests for scripts/lib/site-checks.sh.
#
# These drive each check into a failing state and assert it is caught. A checker
# that only ever reports "ok" would have passed happily through the three-and-a-
# half-week outage these scripts exist to prevent, so the negative cases are the
# whole point.
#
# Nothing here touches the real site: SITE_DIR and BASE_URL are overridden to a
# temp tree and a dead port.
set -uo pipefail
cd "$(dirname "$0")/.."
pass=0; fail=0
ok() { printf ' PASS %s\n' "$1"; pass=$((pass+1)); }
bad() { printf ' FAIL %s\n -> %s\n' "$1" "$2"; fail=$((fail+1)); }
tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
# Build a fake site tree with all five binaries present and executable.
mkdir -p "$tmp/bin" "$tmp/services/scout" "$tmp/services/hawaii" "$tmp/data"
for f in "$tmp/app" "$tmp/bin/claudomator" "$tmp/bin/claude" \
"$tmp/services/scout/app" "$tmp/services/hawaii/app"; do
touch "$f"; chmod +x "$f"
done
# Load the library against the fake tree. Routes point at a closed port so the
# route check exercises its transport-error path rather than hitting the network.
SITE_DIR="$tmp"
BASE_URL="http://127.0.0.1:9"
# shellcheck source=scripts/lib/site-checks.sh
. scripts/lib/site-checks.sh
printf '=== check_binaries ===\n'
out=$(check_binaries); rc=$?
if [ $rc -eq 0 ]; then ok "passes when every binary is executable"
else bad "all-executable passes" "rc=$rc"; fi
chmod -x "$tmp/services/scout/app"
out=$(check_binaries); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "not executable.*scout/app"; then
ok "catches a binary that lost its execute bit"
else
bad "lost +x detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
chmod +x "$tmp/services/scout/app"
mv "$tmp/bin/claude" "$tmp/bin/claude.moved"
out=$(check_binaries); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "missing binary.*claude"; then
ok "catches a missing binary"
else
bad "missing binary detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
mv "$tmp/bin/claude.moved" "$tmp/bin/claude"
out=$(check_binaries); rc=$?
[ $rc -eq 0 ] && ok "recovers to passing once restored" \
|| bad "restore" "rc=$rc"
printf '=== check_routes ===\n'
out=$(check_routes); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route / -> 000"; then
ok "catches an unreachable gateway (transport error, not a status code)"
else
bad "unreachable gateway" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
# A 502 from a live server is the shape of a dead sub-service. Serve one.
python3 - "$tmp/port" <<'PY' &
import http.server, socketserver, sys, threading
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(502); self.end_headers()
def log_message(self, *a): pass
with socketserver.TCPServer(("127.0.0.1", 0), H) as s:
open(sys.argv[1], "w").write(str(s.server_address[1]))
s.serve_forever()
PY
server_pid=$!
for _ in $(seq 1 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done
port=$(cat "$tmp/port" 2>/dev/null || echo "")
if [ -n "$port" ]; then
BASE_URL="http://127.0.0.1:${port}"
out=$(check_routes); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route /scout/ -> 502"; then
ok "catches a 502 from a dead sub-service upstream"
else
bad "502 detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
else
bad "502 detection" "could not start stub server"
fi
kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null
printf '=== check_services ===\n'
# A unit name that cannot exist must be reported, not silently skipped.
SITE_SERVICES=("definitely-not-a-real-unit-$$.service")
# Redirected, not captured in $( ): check_services sets FAILED_SERVICES, and a
# subshell would discard it. health-check drives recovery off that array, so
# this test also pins the calling convention.
check_services > "$tmp/svc.out"; rc=$?
if [ $rc -ne 0 ] && [ "${#FAILED_SERVICES[@]}" -eq 1 ]; then
ok "catches a non-running unit and records it for recovery"
else
bad "dead unit detected" "rc=$rc failed=${#FAILED_SERVICES[@]}"
fi
# Regression guard for the subshell trap itself.
FAILED_SERVICES=()
_discard=$(check_services)
if [ "${#FAILED_SERVICES[@]}" -eq 0 ]; then
ok "documents that command substitution loses FAILED_SERVICES (call directly)"
else
bad "subshell semantics" "array unexpectedly survived a subshell"
fi
printf '=== check_app_health ===\n'
# Unreachable app: must fail, not be mistaken for healthy.
APP_HEALTH_URL="http://127.0.0.1:9/health"
out=$(check_app_health); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "app health"; then
ok "catches an unreachable app health endpoint"
else
bad "unreachable health endpoint" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
# A 503 is what /health returns when the database is unreachable — the app is
# listening and answering, so only this check can see it.
python3 - "$tmp/hport" <<'PY2' &
import http.server, socketserver, sys
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(503); self.end_headers()
self.wfile.write(b"db unreachable")
def log_message(self, *a): pass
with socketserver.TCPServer(("127.0.0.1", 0), H) as s:
open(sys.argv[1], "w").write(str(s.server_address[1]))
s.serve_forever()
PY2
hpid=$!
for _ in $(seq 1 50); do [ -s "$tmp/hport" ] && break; sleep 0.1; done
hport=$(cat "$tmp/hport" 2>/dev/null || echo "")
if [ -n "$hport" ]; then
APP_HEALTH_URL="http://127.0.0.1:${hport}/health"
out=$(check_app_health); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "503"; then
ok "catches a listening app whose database ping fails (503)"
else
bad "503 db-ping detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
else
bad "503 db-ping detection" "could not start stub server"
fi
kill "$hpid" 2>/dev/null; wait "$hpid" 2>/dev/null
printf '=== check_build_stamp ===\n'
out=$(check_build_stamp "definitely-not-the-running-commit"); rc=$?
if [ $rc -ne 0 ]; then
ok "catches a build stamp that does not match what was deployed"
else
bad "build stamp mismatch" "rc=$rc"
fi
printf '\n=== summary: %d passed, %d failed ===\n' "$pass" "$fail"
[ "$fail" -eq 0 ]
|
