1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#!/bin/bash
#
# Tests for scripts/lib/site-checks.sh.
#
# These drive each check into a failing state and assert it is caught. A checker
# that only ever reports "ok" would have passed happily through the three-and-a-
# half-week outage these scripts exist to prevent, so the negative cases are the
# whole point.
#
# Nothing here touches the real site: SITE_DIR and BASE_URL are overridden to a
# temp tree and a dead port.
set -uo pipefail
cd "$(dirname "$0")/.."
pass=0; fail=0
ok() { printf ' PASS %s\n' "$1"; pass=$((pass+1)); }
bad() { printf ' FAIL %s\n -> %s\n' "$1" "$2"; fail=$((fail+1)); }
tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
# Build a fake site tree with all five binaries present and executable.
mkdir -p "$tmp/bin" "$tmp/services/scout" "$tmp/services/hawaii" "$tmp/data"
for f in "$tmp/app" "$tmp/bin/claudomator" "$tmp/bin/claude" \
"$tmp/services/scout/app" "$tmp/services/hawaii/app"; do
touch "$f"; chmod +x "$f"
done
# Load the library against the fake tree. Routes point at a closed port so the
# route check exercises its transport-error path rather than hitting the network.
SITE_DIR="$tmp"
BASE_URL="http://127.0.0.1:9"
# shellcheck source=scripts/lib/site-checks.sh
. scripts/lib/site-checks.sh
printf '=== check_binaries ===\n'
out=$(check_binaries); rc=$?
if [ $rc -eq 0 ]; then ok "passes when every binary is executable"
else bad "all-executable passes" "rc=$rc"; fi
chmod -x "$tmp/services/scout/app"
out=$(check_binaries); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "not executable.*scout/app"; then
ok "catches a binary that lost its execute bit"
else
bad "lost +x detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
chmod +x "$tmp/services/scout/app"
mv "$tmp/bin/claude" "$tmp/bin/claude.moved"
out=$(check_binaries); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "missing binary.*claude"; then
ok "catches a missing binary"
else
bad "missing binary detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
mv "$tmp/bin/claude.moved" "$tmp/bin/claude"
out=$(check_binaries); rc=$?
[ $rc -eq 0 ] && ok "recovers to passing once restored" \
|| bad "restore" "rc=$rc"
printf '=== check_routes ===\n'
out=$(check_routes); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route / -> 000"; then
ok "catches an unreachable gateway (transport error, not a status code)"
else
bad "unreachable gateway" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
# A 502 from a live server is the shape of a dead sub-service. Serve one.
python3 - "$tmp/port" <<'PY' &
import http.server, socketserver, sys, threading
class H(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(502); self.end_headers()
def log_message(self, *a): pass
with socketserver.TCPServer(("127.0.0.1", 0), H) as s:
open(sys.argv[1], "w").write(str(s.server_address[1]))
s.serve_forever()
PY
server_pid=$!
for _ in $(seq 1 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done
port=$(cat "$tmp/port" 2>/dev/null || echo "")
if [ -n "$port" ]; then
BASE_URL="http://127.0.0.1:${port}"
out=$(check_routes); rc=$?
if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route /scout/ -> 502"; then
ok "catches a 502 from a dead sub-service upstream"
else
bad "502 detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')"
fi
else
bad "502 detection" "could not start stub server"
fi
kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null
printf '=== check_services ===\n'
# A unit name that cannot exist must be reported, not silently skipped.
SITE_SERVICES=("definitely-not-a-real-unit-$$.service")
# Redirected, not captured in $( ): check_services sets FAILED_SERVICES, and a
# subshell would discard it. health-check drives recovery off that array, so
# this test also pins the calling convention.
check_services > "$tmp/svc.out"; rc=$?
if [ $rc -ne 0 ] && [ "${#FAILED_SERVICES[@]}" -eq 1 ]; then
ok "catches a non-running unit and records it for recovery"
else
bad "dead unit detected" "rc=$rc failed=${#FAILED_SERVICES[@]}"
fi
# Regression guard for the subshell trap itself.
FAILED_SERVICES=()
_discard=$(check_services)
if [ "${#FAILED_SERVICES[@]}" -eq 0 ]; then
ok "documents that command substitution loses FAILED_SERVICES (call directly)"
else
bad "subshell semantics" "array unexpectedly survived a subshell"
fi
printf '=== check_build_stamp ===\n'
out=$(check_build_stamp "definitely-not-the-running-commit"); rc=$?
if [ $rc -ne 0 ]; then
ok "catches a build stamp that does not match what was deployed"
else
bad "build stamp mismatch" "rc=$rc"
fi
printf '\n=== summary: %d passed, %d failed ===\n' "$pass" "$fail"
[ "$fail" -eq 0 ]
|
