#!/bin/bash # # Tests for scripts/lib/site-checks.sh. # # These drive each check into a failing state and assert it is caught. A checker # that only ever reports "ok" would have passed happily through the three-and-a- # half-week outage these scripts exist to prevent, so the negative cases are the # whole point. # # Nothing here touches the real site: SITE_DIR and BASE_URL are overridden to a # temp tree and a dead port. set -uo pipefail cd "$(dirname "$0")/.." pass=0; fail=0 ok() { printf ' PASS %s\n' "$1"; pass=$((pass+1)); } bad() { printf ' FAIL %s\n -> %s\n' "$1" "$2"; fail=$((fail+1)); } tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT # Build a fake site tree with all five binaries present and executable. mkdir -p "$tmp/bin" "$tmp/services/scout" "$tmp/services/hawaii" "$tmp/data" for f in "$tmp/app" "$tmp/bin/claudomator" "$tmp/bin/claude" \ "$tmp/services/scout/app" "$tmp/services/hawaii/app"; do touch "$f"; chmod +x "$f" done # Load the library against the fake tree. Routes point at a closed port so the # route check exercises its transport-error path rather than hitting the network. SITE_DIR="$tmp" BASE_URL="http://127.0.0.1:9" # shellcheck source=scripts/lib/site-checks.sh . scripts/lib/site-checks.sh printf '=== check_binaries ===\n' out=$(check_binaries); rc=$? if [ $rc -eq 0 ]; then ok "passes when every binary is executable" else bad "all-executable passes" "rc=$rc"; fi chmod -x "$tmp/services/scout/app" out=$(check_binaries); rc=$? if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "not executable.*scout/app"; then ok "catches a binary that lost its execute bit" else bad "lost +x detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" fi chmod +x "$tmp/services/scout/app" mv "$tmp/bin/claude" "$tmp/bin/claude.moved" out=$(check_binaries); rc=$? if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "missing binary.*claude"; then ok "catches a missing binary" else bad "missing binary detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" fi mv "$tmp/bin/claude.moved" "$tmp/bin/claude" out=$(check_binaries); rc=$? [ $rc -eq 0 ] && ok "recovers to passing once restored" \ || bad "restore" "rc=$rc" printf '=== check_routes ===\n' out=$(check_routes); rc=$? if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route / -> 000"; then ok "catches an unreachable gateway (transport error, not a status code)" else bad "unreachable gateway" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" fi # A 502 from a live server is the shape of a dead sub-service. Serve one. python3 - "$tmp/port" <<'PY' & import http.server, socketserver, sys, threading class H(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(502); self.end_headers() def log_message(self, *a): pass with socketserver.TCPServer(("127.0.0.1", 0), H) as s: open(sys.argv[1], "w").write(str(s.server_address[1])) s.serve_forever() PY server_pid=$! for _ in $(seq 1 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done port=$(cat "$tmp/port" 2>/dev/null || echo "") if [ -n "$port" ]; then BASE_URL="http://127.0.0.1:${port}" out=$(check_routes); rc=$? if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route /scout/ -> 502"; then ok "catches a 502 from a dead sub-service upstream" else bad "502 detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" fi else bad "502 detection" "could not start stub server" fi kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null printf '=== check_services ===\n' # A unit name that cannot exist must be reported, not silently skipped. SITE_SERVICES=("definitely-not-a-real-unit-$$.service") # Redirected, not captured in $( ): check_services sets FAILED_SERVICES, and a # subshell would discard it. health-check drives recovery off that array, so # this test also pins the calling convention. check_services > "$tmp/svc.out"; rc=$? if [ $rc -ne 0 ] && [ "${#FAILED_SERVICES[@]}" -eq 1 ]; then ok "catches a non-running unit and records it for recovery" else bad "dead unit detected" "rc=$rc failed=${#FAILED_SERVICES[@]}" fi # Regression guard for the subshell trap itself. FAILED_SERVICES=() _discard=$(check_services) if [ "${#FAILED_SERVICES[@]}" -eq 0 ]; then ok "documents that command substitution loses FAILED_SERVICES (call directly)" else bad "subshell semantics" "array unexpectedly survived a subshell" fi printf '=== check_app_health ===\n' # Unreachable app: must fail, not be mistaken for healthy. APP_HEALTH_URL="http://127.0.0.1:9/health" out=$(check_app_health); rc=$? if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "app health"; then ok "catches an unreachable app health endpoint" else bad "unreachable health endpoint" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" fi # A 503 is what /health returns when the database is unreachable — the app is # listening and answering, so only this check can see it. python3 - "$tmp/hport" <<'PY2' & import http.server, socketserver, sys class H(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(503); self.end_headers() self.wfile.write(b"db unreachable") def log_message(self, *a): pass with socketserver.TCPServer(("127.0.0.1", 0), H) as s: open(sys.argv[1], "w").write(str(s.server_address[1])) s.serve_forever() PY2 hpid=$! for _ in $(seq 1 50); do [ -s "$tmp/hport" ] && break; sleep 0.1; done hport=$(cat "$tmp/hport" 2>/dev/null || echo "") if [ -n "$hport" ]; then APP_HEALTH_URL="http://127.0.0.1:${hport}/health" out=$(check_app_health); rc=$? if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "503"; then ok "catches a listening app whose database ping fails (503)" else bad "503 db-ping detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" fi else bad "503 db-ping detection" "could not start stub server" fi kill "$hpid" 2>/dev/null; wait "$hpid" 2>/dev/null printf '=== check_build_stamp ===\n' out=$(check_build_stamp "definitely-not-the-running-commit"); rc=$? if [ $rc -ne 0 ]; then ok "catches a build stamp that does not match what was deployed" else bad "build stamp mismatch" "rc=$rc" fi printf '\n=== summary: %d passed, %d failed ===\n' "$pass" "$fail" [ "$fail" -eq 0 ]