diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/deploy | 35 | ||||
| -rwxr-xr-x | scripts/health-check | 129 | ||||
| -rw-r--r-- | scripts/lib/site-checks.sh | 127 | ||||
| -rwxr-xr-x | scripts/test-health-checks.sh | 134 | ||||
| -rwxr-xr-x | scripts/verify-deploy | 49 |
5 files changed, 472 insertions, 2 deletions
diff --git a/scripts/deploy b/scripts/deploy index 6579839..5ec6175 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -32,6 +32,9 @@ rsync -av --delete web/static/ ${SITE_DIR}/public/ rsync -av --delete web/templates/ ${SITE_DIR}/templates/ # Sync migrations rsync -av --delete migrations/ ${SITE_DIR}/migrations/ +# Sync operational scripts so the health-check timer runs the current version +# from a stable path rather than whatever happens to be in a source checkout. +rsync -av --delete scripts/ ${SITE_DIR}/app-scripts/ echo "==> Swapping binary..." mv app ${SITE_DIR}/app @@ -71,9 +74,37 @@ for svc_bin in ${SITE_DIR}/services/*/app ${SITE_DIR}/bin/*; do chmod +x "$svc_bin" done +# The synced operational scripts need to stay executable too — the health-check +# timer ExecStarts one of them, and a non-executable ExecStart is the same +# 203/EXEC failure this whole change exists to prevent. +chmod +x ${SITE_DIR}/app-scripts/deploy \ + ${SITE_DIR}/app-scripts/verify-deploy \ + ${SITE_DIR}/app-scripts/health-check 2>/dev/null || true + echo "==> Starting service..." systemctl start ${SERVICE} -sleep 1 -systemctl status ${SERVICE} --no-pager -l +sleep 2 + +# Sub-services are separate units and are NOT restarted by deploying doot. If +# one is sitting in `failed` after exhausting its restart limit, it stays down +# through any number of successful deploys — bring them back here so a deploy +# leaves the whole site healthy, not just the main app. +echo "==> Ensuring sub-services are running..." +for unit in scout.service hawaii.service claudomator@${FQDN}.service; do + systemctl list-unit-files "${unit}" >/dev/null 2>&1 || continue + if [ "$(systemctl is-active "${unit}" 2>/dev/null || true)" != "active" ]; then + echo " ${unit} is not active — restarting" + systemctl reset-failed "${unit}" 2>/dev/null || true + systemctl start "${unit}" || echo " WARNING: could not start ${unit}" + fi +done +sleep 3 + +# Assert the outcome instead of printing a status block for a human to skim. +# `systemctl status` exits zero whether or not anything works and says nothing +# about the gateway's sub-services, so a deploy could report success while +# /scout/ served 502s. +echo "==> Verifying deploy..." +"$(dirname "$0")/verify-deploy" "${BUILD_COMMIT}" echo "==> Deploy complete!" diff --git a/scripts/health-check b/scripts/health-check new file mode 100755 index 0000000..b1c34d0 --- /dev/null +++ b/scripts/health-check @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Periodic health check for doot.terst.org and its sub-services, with recovery. +# +# The failure this exists for: doot itself stays green while a sub-service +# behind the gateway is dead, so /scout/ returns 502 and nothing anywhere says +# so. Scout and hawaii were down for three and a half weeks, and claudomator +# crash-looped 426,000 times, before anyone opened the page and noticed. +# +# There is no mail/ntfy/apprise on this host, so "alerting" here means: +# * a loud, greppable line in the journal +# * a machine-readable status file the dashboard can surface later +# * a non-zero exit, which makes the systemd unit go failed and show up in +# `systemctl --failed` +# +# It also attempts recovery, because the common case (a unit that hit its +# restart limit and gave up) is fixed by exactly two commands and waiting weeks +# for a human to run them is the actual problem. +# +# Usage: +# scripts/health-check [--no-recover] + +set -uo pipefail + +cd "$(dirname "$0")/.." +# shellcheck source=scripts/lib/site-checks.sh +. "$(dirname "$0")/lib/site-checks.sh" + +STATUS_FILE="${STATUS_FILE:-${SITE_DIR}/data/health-status.json}" +recover=1 +[ "${1:-}" = "--no-recover" ] && recover=0 + +log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } + +rc=0 +recovered=() +still_broken=() + +# ── binaries ────────────────────────────────────────────────────────────────── +check_binaries; bin_rc=$? +if [ $bin_rc -ne 0 ]; then + rc=1 + if [ $recover -eq 1 ]; then + # A lost execute bit is recoverable in place and is the known failure + # mode here, so fix it rather than just reporting it. + for f in "${SITE_BINARIES[@]}"; do + if [ -f "$f" ] && [ ! -x "$f" ]; then + log "HEALTH recovering: chmod +x $f" + chmod +x "$f" && recovered+=("chmod +x $f") + fi + done + fi +fi + +# ── services ────────────────────────────────────────────────────────────────── +# Called directly, NOT in a command substitution: check_services sets +# FAILED_SERVICES, and a subshell would populate the array then throw it away, +# leaving the recovery loop below with nothing to do. +check_services; svc_rc=$? +if [ $svc_rc -ne 0 ]; then + rc=1 + if [ $recover -eq 1 ]; then + for s in "${FAILED_SERVICES[@]}"; do + log "HEALTH recovering: restarting $s" + systemctl reset-failed "$s" 2>/dev/null || true + if systemctl start "$s" 2>/dev/null; then + sleep 3 + if [ "$(systemctl is-active "$s" 2>/dev/null || true)" = "active" ]; then + log "HEALTH recovered: $s is active" + recovered+=("restarted $s") + else + log "HEALTH ALERT: $s still not active after restart" + still_broken+=("$s") + fi + else + log "HEALTH ALERT: failed to start $s" + still_broken+=("$s") + fi + done + fi +fi + +# ── routes, re-checked after any recovery ───────────────────────────────────── +check_routes; route_rc=$? +[ $route_rc -ne 0 ] && rc=1 + +# A recovered site is healthy now even though it was not a moment ago. Re-run +# the checks so the final verdict describes the current state, not the state we +# walked in on. +if [ $rc -ne 0 ] && [ ${#recovered[@]} -gt 0 ]; then + echo "==> Re-checking after recovery..." + rc=0 + check_binaries >/dev/null || rc=1 + check_services >/dev/null || rc=1 + check_routes >/dev/null || rc=1 +fi + +# ── status file ─────────────────────────────────────────────────────────────── +mkdir -p "$(dirname "$STATUS_FILE")" 2>/dev/null || true +{ + printf '{\n' + printf ' "checked_at": "%s",\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf ' "healthy": %s,\n' "$([ $rc -eq 0 ] && echo true || echo false)" + printf ' "recovered": [' + for i in "${!recovered[@]}"; do + [ "$i" -gt 0 ] && printf ', ' + printf '"%s"' "${recovered[$i]}" + done + printf '],\n' + printf ' "still_broken": [' + for i in "${!still_broken[@]}"; do + [ "$i" -gt 0 ] && printf ', ' + printf '"%s"' "${still_broken[$i]}" + done + printf ']\n}\n' +} > "$STATUS_FILE" 2>/dev/null || log "HEALTH warn: could not write $STATUS_FILE" + +if [ $rc -eq 0 ]; then + if [ ${#recovered[@]} -gt 0 ]; then + log "HEALTH ALERT recovered: ${recovered[*]}" + # Deliberately non-zero is NOT used here: the site is healthy now. The + # ALERT line in the journal is the durable record that something had + # broken and was silently fixed. + fi + exit 0 +fi + +log "HEALTH ALERT: doot.terst.org is unhealthy; still broken: ${still_broken[*]:-see checks above}" +exit 1 diff --git a/scripts/lib/site-checks.sh b/scripts/lib/site-checks.sh new file mode 100644 index 0000000..f6ac55c --- /dev/null +++ b/scripts/lib/site-checks.sh @@ -0,0 +1,127 @@ +#!/bin/bash +# +# Shared assertions about a healthy doot.terst.org, used by both +# scripts/verify-deploy and scripts/health-check so the two cannot drift apart. +# +# Source this; do not execute it. +# +# Every function returns non-zero on failure and prints one line per check, so +# callers can aggregate. Nothing here restarts or changes anything — recovery is +# the caller's decision. + +SITE_DIR="${SITE_DIR:-/site/doot.terst.org}" +BASE_URL="${BASE_URL:-https://doot.terst.org}" + +# Executables the site needs. A missing +x here is invisible until the next +# reboot, because a running process keeps executing from its open file — this is +# exactly how scout, hawaii, claudomator and claude were all disarmed at once and +# stayed down for three and a half weeks. +SITE_BINARIES=( + "${SITE_DIR}/app" + "${SITE_DIR}/bin/claudomator" + "${SITE_DIR}/bin/claude" + "${SITE_DIR}/services/scout/app" + "${SITE_DIR}/services/hawaii/app" +) + +# systemd units that must be running. +SITE_SERVICES=( + "task-dashboard@doot.terst.org.service" + "scout.service" + "hawaii.service" + "claudomator@doot.terst.org.service" +) + +# Gateway paths that must not return a server error. Checked unauthenticated, so +# a login redirect or the login page itself is a pass — what we are detecting is +# 502/503/504 from a dead upstream, which is what a down sub-service looks like +# from outside. +SITE_ROUTES=( + "/" + "/scout/" + "/hawaii/" + "/claudomator/" +) + +_chk_pass() { printf ' ok %s\n' "$*"; } +_chk_fail() { printf ' FAIL %s\n' "$*"; } + +# Are all expected binaries executable? +check_binaries() { + local rc=0 f + for f in "${SITE_BINARIES[@]}"; do + if [ ! -f "$f" ]; then + _chk_fail "missing binary: $f"; rc=1 + elif [ ! -x "$f" ]; then + _chk_fail "not executable: $f ($(stat -c %A "$f"))"; rc=1 + else + _chk_pass "executable: $f" + fi + done + return $rc +} + +# Are all expected services active? +# +# Sets FAILED_SERVICES as a side effect. IMPORTANT: that means callers must NOT +# invoke this inside a command substitution — `out=$(check_services)` runs it in +# a subshell, the array is populated there and discarded, and the caller sees an +# empty list. health-check drives recovery off this array, so that mistake +# silently disables recovery entirely. Call it directly and redirect if you need +# the output: `check_services > file`. +FAILED_SERVICES=() +check_services() { + local rc=0 s state + FAILED_SERVICES=() + for s in "${SITE_SERVICES[@]}"; do + state=$(systemctl is-active "$s" 2>/dev/null || true) + if [ "$state" = "active" ]; then + _chk_pass "active: $s" + else + _chk_fail "not active: $s (state=$state)" + FAILED_SERVICES+=("$s") + rc=1 + fi + done + return $rc +} + +# Do all gateway routes answer without a server error? +check_routes() { + local rc=0 p code + for p in "${SITE_ROUTES[@]}"; do + # --max-time so a hung upstream fails the check instead of hanging it. + # + # Note the shape of this: `$(curl ... || echo 000)` would CONCATENATE + # curl's own "000" transport-error output with the echoed one, producing + # "000000", which then matches no case arm and gets reported as healthy. + # Assigning inside `if !` keeps exactly one value. + if ! code=$(curl -sS -o /dev/null --max-time 15 -w '%{http_code}' "${BASE_URL}${p}" 2>/dev/null); then + code="000" + fi + code=${code:-000} + case "$code" in + 000|5*) _chk_fail "route ${p} -> ${code}"; rc=1 ;; + *) _chk_pass "route ${p} -> ${code}" ;; + esac + done + return $rc +} + +# Does the running binary report the commit we expect? Only meaningful right +# after a deploy, so it is optional. +check_build_stamp() { + local expected=$1 seen + seen=$(journalctl -u "task-dashboard@doot.terst.org.service" --no-pager -n 50 2>/dev/null \ + | grep -oE 'build=[0-9a-f]+' | tail -1 | cut -d= -f2 || true) + if [ -z "$seen" ]; then + _chk_fail "no build stamp found in recent journal" + return 1 + fi + if [ "$seen" = "$expected" ]; then + _chk_pass "running build=$seen" + return 0 + fi + _chk_fail "running build=$seen but expected $expected" + return 1 +} diff --git a/scripts/test-health-checks.sh b/scripts/test-health-checks.sh new file mode 100755 index 0000000..fbe9e47 --- /dev/null +++ b/scripts/test-health-checks.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# +# Tests for scripts/lib/site-checks.sh. +# +# These drive each check into a failing state and assert it is caught. A checker +# that only ever reports "ok" would have passed happily through the three-and-a- +# half-week outage these scripts exist to prevent, so the negative cases are the +# whole point. +# +# Nothing here touches the real site: SITE_DIR and BASE_URL are overridden to a +# temp tree and a dead port. + +set -uo pipefail +cd "$(dirname "$0")/.." + +pass=0; fail=0 +ok() { printf ' PASS %s\n' "$1"; pass=$((pass+1)); } +bad() { printf ' FAIL %s\n -> %s\n' "$1" "$2"; fail=$((fail+1)); } + +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT + +# Build a fake site tree with all five binaries present and executable. +mkdir -p "$tmp/bin" "$tmp/services/scout" "$tmp/services/hawaii" "$tmp/data" +for f in "$tmp/app" "$tmp/bin/claudomator" "$tmp/bin/claude" \ + "$tmp/services/scout/app" "$tmp/services/hawaii/app"; do + touch "$f"; chmod +x "$f" +done + +# Load the library against the fake tree. Routes point at a closed port so the +# route check exercises its transport-error path rather than hitting the network. +SITE_DIR="$tmp" +BASE_URL="http://127.0.0.1:9" +# shellcheck source=scripts/lib/site-checks.sh +. scripts/lib/site-checks.sh + +printf '=== check_binaries ===\n' + +out=$(check_binaries); rc=$? +if [ $rc -eq 0 ]; then ok "passes when every binary is executable" +else bad "all-executable passes" "rc=$rc"; fi + +chmod -x "$tmp/services/scout/app" +out=$(check_binaries); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "not executable.*scout/app"; then + ok "catches a binary that lost its execute bit" +else + bad "lost +x detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" +fi +chmod +x "$tmp/services/scout/app" + +mv "$tmp/bin/claude" "$tmp/bin/claude.moved" +out=$(check_binaries); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "missing binary.*claude"; then + ok "catches a missing binary" +else + bad "missing binary detected" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" +fi +mv "$tmp/bin/claude.moved" "$tmp/bin/claude" + +out=$(check_binaries); rc=$? +[ $rc -eq 0 ] && ok "recovers to passing once restored" \ + || bad "restore" "rc=$rc" + +printf '=== check_routes ===\n' + +out=$(check_routes); rc=$? +if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route / -> 000"; then + ok "catches an unreachable gateway (transport error, not a status code)" +else + bad "unreachable gateway" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" +fi + +# A 502 from a live server is the shape of a dead sub-service. Serve one. +python3 - "$tmp/port" <<'PY' & +import http.server, socketserver, sys, threading +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(502); self.end_headers() + def log_message(self, *a): pass +with socketserver.TCPServer(("127.0.0.1", 0), H) as s: + open(sys.argv[1], "w").write(str(s.server_address[1])) + s.serve_forever() +PY +server_pid=$! +for _ in $(seq 1 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done +port=$(cat "$tmp/port" 2>/dev/null || echo "") + +if [ -n "$port" ]; then + BASE_URL="http://127.0.0.1:${port}" + out=$(check_routes); rc=$? + if [ $rc -ne 0 ] && printf '%s' "$out" | grep -q "route /scout/ -> 502"; then + ok "catches a 502 from a dead sub-service upstream" + else + bad "502 detection" "rc=$rc out=$(printf '%s' "$out" | tr '\n' ' ')" + fi +else + bad "502 detection" "could not start stub server" +fi +kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null + +printf '=== check_services ===\n' + +# A unit name that cannot exist must be reported, not silently skipped. +SITE_SERVICES=("definitely-not-a-real-unit-$$.service") +# Redirected, not captured in $( ): check_services sets FAILED_SERVICES, and a +# subshell would discard it. health-check drives recovery off that array, so +# this test also pins the calling convention. +check_services > "$tmp/svc.out"; rc=$? +if [ $rc -ne 0 ] && [ "${#FAILED_SERVICES[@]}" -eq 1 ]; then + ok "catches a non-running unit and records it for recovery" +else + bad "dead unit detected" "rc=$rc failed=${#FAILED_SERVICES[@]}" +fi + +# Regression guard for the subshell trap itself. +FAILED_SERVICES=() +_discard=$(check_services) +if [ "${#FAILED_SERVICES[@]}" -eq 0 ]; then + ok "documents that command substitution loses FAILED_SERVICES (call directly)" +else + bad "subshell semantics" "array unexpectedly survived a subshell" +fi + +printf '=== check_build_stamp ===\n' + +out=$(check_build_stamp "definitely-not-the-running-commit"); rc=$? +if [ $rc -ne 0 ]; then + ok "catches a build stamp that does not match what was deployed" +else + bad "build stamp mismatch" "rc=$rc" +fi + +printf '\n=== summary: %d passed, %d failed ===\n' "$pass" "$fail" +[ "$fail" -eq 0 ] diff --git a/scripts/verify-deploy b/scripts/verify-deploy new file mode 100755 index 0000000..712c26d --- /dev/null +++ b/scripts/verify-deploy @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Assert that the site is actually healthy. Run automatically at the end of +# scripts/deploy, and usable standalone at any time. +# +# scripts/deploy used to finish with `systemctl status`, which a human reads and +# a script ignores — it exits zero whether or not anything works, and it says +# nothing at all about the sub-services behind the gateway. A deploy could +# therefore "succeed" while /scout/ returned 502, which is precisely what +# happened for three and a half weeks. +# +# Usage: +# scripts/verify-deploy [expected-build-commit] +# +# Exit: 0 everything healthy, 1 something is not. + +set -uo pipefail + +cd "$(dirname "$0")/.." +# shellcheck source=scripts/lib/site-checks.sh +. "$(dirname "$0")/lib/site-checks.sh" + +expected_build=${1:-} +rc=0 + +echo "==> Verifying binaries are executable..." +check_binaries || rc=1 + +echo "==> Verifying services are active..." +check_services || rc=1 + +echo "==> Verifying gateway routes..." +check_routes || rc=1 + +if [ -n "$expected_build" ]; then + echo "==> Verifying deployed build..." + check_build_stamp "$expected_build" || rc=1 +fi + +echo +if [ $rc -eq 0 ]; then + echo "==> Verification PASSED" +else + echo "==> Verification FAILED — the site is not fully healthy" >&2 + echo " Sub-services are started separately from the main app; a failed" >&2 + echo " unit that has exhausted its restart limit needs:" >&2 + echo " systemctl reset-failed <unit> && systemctl start <unit>" >&2 +fi +exit $rc |