#!/bin/bash
#
# Shared assertions about a healthy doot.terst.org, used by both
# scripts/verify-deploy and scripts/health-check so the two cannot drift apart.
#
# Source this; do not execute it.
#
# Every function returns non-zero on failure and prints one line per check, so
# callers can aggregate. Nothing here restarts or changes anything — recovery is
# the caller's decision.

SITE_DIR="${SITE_DIR:-/site/doot.terst.org}"
BASE_URL="${BASE_URL:-https://doot.terst.org}"

# Executables the site needs. A missing +x here is invisible until the next
# reboot, because a running process keeps executing from its open file — this is
# exactly how scout, hawaii, claudomator and claude were all disarmed at once and
# stayed down for three and a half weeks.
SITE_BINARIES=(
    "${SITE_DIR}/app"
    "${SITE_DIR}/bin/claudomator"
    "${SITE_DIR}/bin/claude"
    "${SITE_DIR}/services/scout/app"
    "${SITE_DIR}/services/hawaii/app"
)

# systemd units that must be running.
SITE_SERVICES=(
    "task-dashboard@doot.terst.org.service"
    "scout.service"
    "hawaii.service"
    "claudomator@doot.terst.org.service"
)

# Gateway paths that must not return a server error. Checked unauthenticated, so
# a login redirect or the login page itself is a pass — what we are detecting is
# 502/503/504 from a dead upstream, which is what a down sub-service looks like
# from outside.
SITE_ROUTES=(
    "/"
    "/scout/"
    "/hawaii/"
    "/claudomator/"
)

_chk_pass() { printf '  ok    %s\n' "$*"; }
_chk_fail() { printf '  FAIL  %s\n' "$*"; }

# Are all expected binaries executable?
check_binaries() {
    local rc=0 f
    for f in "${SITE_BINARIES[@]}"; do
        if [ ! -f "$f" ]; then
            _chk_fail "missing binary: $f"; rc=1
        elif [ ! -x "$f" ]; then
            _chk_fail "not executable: $f ($(stat -c %A "$f"))"; rc=1
        else
            _chk_pass "executable: $f"
        fi
    done
    return $rc
}

# Are all expected services active?
#
# Sets FAILED_SERVICES as a side effect. IMPORTANT: that means callers must NOT
# invoke this inside a command substitution — `out=$(check_services)` runs it in
# a subshell, the array is populated there and discarded, and the caller sees an
# empty list. health-check drives recovery off this array, so that mistake
# silently disables recovery entirely. Call it directly and redirect if you need
# the output: `check_services > file`.
FAILED_SERVICES=()
check_services() {
    local rc=0 s state
    FAILED_SERVICES=()
    for s in "${SITE_SERVICES[@]}"; do
        state=$(systemctl is-active "$s" 2>/dev/null || true)
        if [ "$state" = "active" ]; then
            _chk_pass "active: $s"
        else
            _chk_fail "not active: $s (state=$state)"
            FAILED_SERVICES+=("$s")
            rc=1
        fi
    done
    return $rc
}

# Do all gateway routes answer without a server error?
check_routes() {
    local rc=0 p code
    for p in "${SITE_ROUTES[@]}"; do
        # --max-time so a hung upstream fails the check instead of hanging it.
        #
        # Note the shape of this: `$(curl ... || echo 000)` would CONCATENATE
        # curl's own "000" transport-error output with the echoed one, producing
        # "000000", which then matches no case arm and gets reported as healthy.
        # Assigning inside `if !` keeps exactly one value.
        if ! code=$(curl -sS -o /dev/null --max-time 15 -w '%{http_code}' "${BASE_URL}${p}" 2>/dev/null); then
            code="000"
        fi
        code=${code:-000}
        case "$code" in
            000|5*) _chk_fail "route ${p} -> ${code}"; rc=1 ;;
            *)      _chk_pass "route ${p} -> ${code}" ;;
        esac
    done
    return $rc
}

# Is the app actually serving, not merely listening?
#
# /health pings the database rather than returning an unconditional 200. During
# the 2026-08-04 incident a liveness stub stayed green for three days while
# every DB-touching request path was wedged, which is why it was changed. The
# gateway route checks above would not catch that: `/` answers 303 to the login
# page without going near the database.
APP_HEALTH_URL="${APP_HEALTH_URL:-http://127.0.0.1:38080/health}"
check_app_health() {
    local code
    # Two attempts a few seconds apart — a single blip should not be reported as
    # an outage, and the caller may act on this by restarting the service.
    for attempt in 1 2; do
        if ! code=$(curl -sS -o /dev/null --max-time 10 -w '%{http_code}' "$APP_HEALTH_URL" 2>/dev/null); then
            code="000"
        fi
        code=${code:-000}
        [ "$code" = "200" ] && { _chk_pass "app health (db ping) -> 200"; return 0; }
        [ "$attempt" = "1" ] && sleep 5
    done
    _chk_fail "app health (db ping) -> ${code}"
    return 1
}

# Does the running binary report the commit we expect? Only meaningful right
# after a deploy, so it is optional.
check_build_stamp() {
    local expected=$1 seen
    seen=$(journalctl -u "task-dashboard@doot.terst.org.service" --no-pager -n 50 2>/dev/null \
           | grep -oE 'build=[0-9a-f]+' | tail -1 | cut -d= -f2 || true)
    if [ -z "$seen" ]; then
        _chk_fail "no build stamp found in recent journal"
        return 1
    fi
    if [ "$seen" = "$expected" ]; then
        _chk_pass "running build=$seen"
        return 0
    fi
    _chk_fail "running build=$seen but expected $expected"
    return 1
}