blob: db3b1d8df2baa5627d41883141e130377a762bbf (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
|
#!/bin/bash
#
# Assert that the site is actually healthy. Run automatically at the end of
# scripts/deploy, and usable standalone at any time.
#
# scripts/deploy used to finish with `systemctl status`, which a human reads and
# a script ignores — it exits zero whether or not anything works, and it says
# nothing at all about the sub-services behind the gateway. A deploy could
# therefore "succeed" while /scout/ returned 502, which is precisely what
# happened for three and a half weeks.
#
# Usage:
# scripts/verify-deploy [expected-build-commit]
#
# Exit: 0 everything healthy, 1 something is not.
set -uo pipefail
cd "$(dirname "$0")/.."
# shellcheck source=scripts/lib/site-checks.sh
. "$(dirname "$0")/lib/site-checks.sh"
expected_build=${1:-}
rc=0
echo "==> Verifying binaries are executable..."
check_binaries || rc=1
echo "==> Verifying services are active..."
check_services || rc=1
echo "==> Verifying gateway routes..."
check_routes || rc=1
echo "==> Verifying app health (database reachable)..."
check_app_health || rc=1
if [ -n "$expected_build" ]; then
echo "==> Verifying deployed build..."
check_build_stamp "$expected_build" || rc=1
fi
echo
if [ $rc -eq 0 ]; then
echo "==> Verification PASSED"
else
echo "==> Verification FAILED — the site is not fully healthy" >&2
echo " Sub-services are started separately from the main app; a failed" >&2
echo " unit that has exhausted its restart limit needs:" >&2
echo " systemctl reset-failed <unit> && systemctl start <unit>" >&2
fi
exit $rc
|
