#!/bin/bash set -e cd "$(dirname "$0")/.." # Configuration FQDN="doot.terst.org" SITE_DIR="/site/${FQDN}" SERVICE="task-dashboard@${FQDN}" DEPLOY_HOST="tungsten" if [ "$(hostname)" != "${DEPLOY_HOST}" ]; then echo "Refusing to deploy: this is $(hostname), not ${DEPLOY_HOST}." >&2 exit 1 fi echo "==> Building CSS..." npm run css:build echo "==> Building binary..." BUILD_COMMIT=$(git rev-parse --short HEAD) BUILD_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ') LDFLAGS="-X main.buildCommit=${BUILD_COMMIT} -X main.buildTime=${BUILD_TIME}" go build -ldflags "$LDFLAGS" -o app cmd/dashboard/main.go echo "==> Stopping service..." systemctl stop ${SERVICE} || true echo "==> Syncing assets and migrations to ${SITE_DIR}..." # Sync static files rsync -av --delete web/static/ ${SITE_DIR}/public/ # Sync templates rsync -av --delete web/templates/ ${SITE_DIR}/templates/ # Sync migrations rsync -av --delete migrations/ ${SITE_DIR}/migrations/ # Sync operational scripts so the health-check timer runs the current version # from a stable path rather than whatever happens to be in a source checkout. rsync -av --delete scripts/ ${SITE_DIR}/app-scripts/ echo "==> Swapping binary..." mv app ${SITE_DIR}/app echo "==> Setting permissions..." # Scope permission fixing to the directories this script actually manages. # # This previously ran `find ${SITE_DIR} -type f -exec chmod 644 {} +` across the # whole site directory and then restored +x on ${SITE_DIR}/app alone. That also # stripped the execute bit from every sub-service binary under # ${SITE_DIR}/services/, e.g. services/scout/app. Those services keep running # from their already-open file, so nothing appears to break — until the next # reboot, when systemd cannot exec them and they fail with 203/EXEC. Scout was # down for three and a half weeks that way, serving 502s behind /scout/, because # the failure is invisible until someone opens the page. for managed in public templates migrations; do [ -d "${SITE_DIR}/${managed}" ] || continue chown -R www-data:www-data "${SITE_DIR}/${managed}" find "${SITE_DIR}/${managed}" -type d -exec chmod 755 {} + find "${SITE_DIR}/${managed}" -type f -exec chmod 644 {} + done chown www-data:www-data ${SITE_DIR}/app chmod +x ${SITE_DIR}/app chmod 600 ${SITE_DIR}/.env 2>/dev/null || true # Belt and braces: re-assert +x on every executable the site depends on. Even # though the blanket chmod above is gone, a missing execute bit here is what an # outage looks like, and asserting it costs nothing. # # The old blanket chmod took out four binaries in one go — services/scout/app, # services/hawaii/app, bin/claudomator and bin/claude (which claudomator execs # via --claude-bin). Only ${SITE_DIR}/app survived, because it was the single # path the script put the bit back on. Both locations are covered here. for svc_bin in ${SITE_DIR}/services/*/app ${SITE_DIR}/bin/*; do [ -f "$svc_bin" ] || continue chmod +x "$svc_bin" done # The synced operational scripts need to stay executable too — the health-check # timer ExecStarts one of them, and a non-executable ExecStart is the same # 203/EXEC failure this whole change exists to prevent. chmod +x ${SITE_DIR}/app-scripts/deploy \ ${SITE_DIR}/app-scripts/verify-deploy \ ${SITE_DIR}/app-scripts/health-check 2>/dev/null || true echo "==> Starting service..." systemctl start ${SERVICE} sleep 2 # Sub-services are separate units and are NOT restarted by deploying doot. If # one is sitting in `failed` after exhausting its restart limit, it stays down # through any number of successful deploys — bring them back here so a deploy # leaves the whole site healthy, not just the main app. echo "==> Ensuring sub-services are running..." for unit in scout.service hawaii.service claudomator@${FQDN}.service; do systemctl list-unit-files "${unit}" >/dev/null 2>&1 || continue if [ "$(systemctl is-active "${unit}" 2>/dev/null || true)" != "active" ]; then echo " ${unit} is not active — restarting" systemctl reset-failed "${unit}" 2>/dev/null || true systemctl start "${unit}" || echo " WARNING: could not start ${unit}" fi done sleep 3 # Assert the outcome instead of printing a status block for a human to skim. # `systemctl status` exits zero whether or not anything works and says nothing # about the gateway's sub-services, so a deploy could report success while # /scout/ served 502s. echo "==> Verifying deploy..." "$(dirname "$0")/verify-deploy" "${BUILD_COMMIT}" echo "==> Deploy complete!"