1 files changed, 49 insertions, 0 deletions
diff --git a/scripts/verify-deploy b/scripts/verify-deploy
new file mode 100755
index 0000000..712c26d
--- /dev/null
+++ b/scripts/verify-deploy
@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Assert that the site is actually healthy. Run automatically at the end of
+# scripts/deploy, and usable standalone at any time.
+#
+# scripts/deploy used to finish with `systemctl status`, which a human reads and
+# a script ignores — it exits zero whether or not anything works, and it says
+# nothing at all about the sub-services behind the gateway. A deploy could
+# therefore "succeed" while /scout/ returned 502, which is precisely what
+# happened for three and a half weeks.
+#
+# Usage:
+#   scripts/verify-deploy [expected-build-commit]
+#
+# Exit: 0 everything healthy, 1 something is not.
+
+set -uo pipefail
+
+cd "$(dirname "$0")/.."
+# shellcheck source=scripts/lib/site-checks.sh
+. "$(dirname "$0")/lib/site-checks.sh"
+
+expected_build=${1:-}
+rc=0
+
+echo "==> Verifying binaries are executable..."
+check_binaries || rc=1
+
+echo "==> Verifying services are active..."
+check_services || rc=1
+
+echo "==> Verifying gateway routes..."
+check_routes || rc=1
+
+if [ -n "$expected_build" ]; then
+    echo "==> Verifying deployed build..."
+    check_build_stamp "$expected_build" || rc=1
+fi
+
+echo
+if [ $rc -eq 0 ]; then
+    echo "==> Verification PASSED"
+else
+    echo "==> Verification FAILED — the site is not fully healthy" >&2
+    echo "    Sub-services are started separately from the main app; a failed" >&2
+    echo "    unit that has exhausted its restart limit needs:" >&2
+    echo "      systemctl reset-failed <unit> && systemctl start <unit>" >&2
+fi
+exit $rc