ce454e0be6
backend and frontend now reference their own images (BACKEND_IMAGE_PATH/TAG, FRONTEND_IMAGE_PATH/TAG) instead of the shared emi/thermograph/app. deploy.sh takes SERVICE=backend|frontend|all, rolls just that service with --no-deps, persists each service's live tag in deploy/.image-tags.env so a single-service deploy never disturbs the sibling, and fixes the old IMAGE_TAG:? guard that made every workflow deploy hard-fail. Health-checks per service (8137/8080). Claude-Session: https://claude.ai/code/session_01RdARHDJaYC1wSQRTYM6t3Z
231 lines
10 KiB
Bash
Executable file
231 lines
10 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Pull this INFRA repo's checkout up to date, then roll ONE (or all) of the
|
|
# docker-compose app services onto its separately-published image. Run on the
|
|
# VPS — each app repo's Forgejo Actions workflow invokes this over SSH (see
|
|
# .forgejo/workflows/deploy.yml in thermograph-backend / thermograph-frontend),
|
|
# and you can run it by hand too.
|
|
#
|
|
# # roll just the backend onto a specific image:
|
|
# ssh deploy@vps 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh'
|
|
# # roll just the frontend:
|
|
# ssh deploy@vps 'SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh'
|
|
# # bring the whole stack up (both tags required):
|
|
# ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha-<a> FRONTEND_IMAGE_TAG=sha-<b> /opt/thermograph/deploy/deploy.sh'
|
|
#
|
|
# FE/BE CI-CD split: backend and frontend are published from separate repos as
|
|
# separate images (emi/thermograph-backend/app, emi/thermograph-frontend/app),
|
|
# so a deploy targets ONE service and leaves the other's running container +
|
|
# tag untouched. Each service's live tag is persisted host-side in
|
|
# deploy/.image-tags.env (untracked -- survives the git reset below) so a
|
|
# single-service roll re-renders compose with BOTH services' real tags and
|
|
# never accidentally recreates or downgrades the sibling.
|
|
#
|
|
# This checkout is thermograph-infra, not an app repo: BRANCH is this repo's
|
|
# branch (compose files, db init, the secrets vault, this script itself);
|
|
# the *_IMAGE_TAG values are the separately-published app images to run -- the
|
|
# two axes are independent and rarely change together.
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thermograph}"
|
|
BRANCH="${BRANCH:-main}"
|
|
# Which service this deploy rolls: backend | frontend | all. Defaults to `all`
|
|
# (a full-stack bring-up) so a by-hand run with both tags still works; the
|
|
# per-repo deploy.yml workflows always pass an explicit single service.
|
|
SERVICE="${SERVICE:-all}"
|
|
HEALTH_PORT="${HEALTH_PORT:-8137}"
|
|
cd "$APP_DIR"
|
|
|
|
case "$SERVICE" in
|
|
backend|frontend|all) ;;
|
|
*) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
|
|
esac
|
|
|
|
# Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose
|
|
# interpolation and are also loaded into the backend container via env_file.
|
|
#
|
|
# When this host is configured for SOPS (an age key + /etc/thermograph/secrets-env),
|
|
# first render /etc/thermograph.env from the committed encrypted source of truth
|
|
# (deploy/secrets/*.yaml) so a key rotation is just an edit+commit+deploy. The guard
|
|
# on the helper's existence keeps the very deploy that INTRODUCES this file safe: on
|
|
# the first pass the checkout may predate it (it arrives with the git reset below,
|
|
# after which deploy.sh re-execs), so a missing helper simply falls back to the
|
|
# existing /etc/thermograph.env. Then source it so a by-hand run interpolates the
|
|
# same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/.
|
|
if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then
|
|
# shellcheck source=deploy/render-secrets.sh
|
|
. "$APP_DIR/deploy/render-secrets.sh"
|
|
render_thermograph_secrets "$APP_DIR"
|
|
fi
|
|
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
|
|
|
|
# Pre-warm the ~750 city-page archives so /climate pages serve from cache and a
|
|
# search-engine crawl never bursts the archive API quota. Detached inside the
|
|
# backend container (compose exec -d), idempotent (skips already-cached cells),
|
|
# so it never blocks the deploy or health check and is cheap on every deploy
|
|
# after the first full warm.
|
|
warm_city_archives() {
|
|
echo "==> Warming city-page archives in the background (backend:/app/logs/warm-cities.log)"
|
|
docker compose exec -d backend sh -c \
|
|
'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
|
|
}
|
|
|
|
# Notify IndexNow (Bing / DuckDuckGo / Yandex) of the site's URLs, but only when
|
|
# the set of pages actually changed (a new/removed city) — code-only deploys skip.
|
|
# Best-effort: never fails the deploy.
|
|
ping_indexnow() {
|
|
echo "==> Pinging IndexNow (only if the URL set changed)"
|
|
local base="${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
|
|
docker compose exec -T backend python indexnow.py --if-changed "$base" \
|
|
|| echo "!! IndexNow ping failed (non-fatal)" >&2
|
|
}
|
|
|
|
echo "==> Fetching $BRANCH"
|
|
git fetch --prune origin "$BRANCH"
|
|
git reset --hard "origin/$BRANCH"
|
|
|
|
# Re-exec: git reset --hard just rewrote this very file's bytes on disk while
|
|
# it's still running. bash reads a script via buffered, byte-offset I/O, so
|
|
# anything AFTER this point in the OLD execution can read from the wrong
|
|
# offset once the file's size/content changed underneath it -- a classic
|
|
# self-modifying-script footgun. Confirmed live: after this PR added ~15
|
|
# lines above, one deploy ran with the OLD "Building images" log lines even
|
|
# though `git status` showed the checkout correctly at the NEW commit --
|
|
# the file changed under a running interpreter, not the checkout. Restart
|
|
# fresh from the now-updated file so everything after this line is
|
|
# guaranteed self-consistent. Guarded so the second invocation doesn't
|
|
# fetch+reset+re-exec forever.
|
|
if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then
|
|
export DEPLOY_SH_REEXECED=1
|
|
exec "$0" "$@"
|
|
fi
|
|
|
|
# Registry-pull cutover: pull the image each app repo's build-push.yml already
|
|
# built and pushed, instead of building in place. This checkout is
|
|
# thermograph-infra, not an app repo, so there's no "current commit" to derive
|
|
# a tag from -- the caller (the deploying repo's deploy.yml) exports its own
|
|
# service's tag (BACKEND_IMAGE_TAG or FRONTEND_IMAGE_TAG = sha-<12 hex> of the
|
|
# app commit, or a semver tag).
|
|
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
|
|
export REGISTRY_HOST BACKEND_IMAGE_PATH FRONTEND_IMAGE_PATH
|
|
|
|
# Load the last-deployed tag for BOTH services first, so a single-service roll
|
|
# still renders compose with the sibling's real, currently-running tag (never a
|
|
# bare `local` that would recreate/downgrade it). The incoming env for the
|
|
# service being deployed then overrides its line below. This file is untracked
|
|
# (see .gitignore), so `git reset --hard` above leaves it in place.
|
|
TAGS_FILE="$APP_DIR/deploy/.image-tags.env"
|
|
if [ -f "$TAGS_FILE" ]; then
|
|
set -a; . "$TAGS_FILE"; set +a
|
|
fi
|
|
|
|
# Guard: the service(s) being rolled MUST have a concrete tag supplied now (the
|
|
# sibling's may come from the persisted file). `all` needs both.
|
|
case "$SERVICE" in
|
|
backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> for a backend deploy}" ;;
|
|
frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> for a frontend deploy}" ;;
|
|
all)
|
|
: "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
|
|
: "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
|
|
;;
|
|
esac
|
|
# Compose interpolates both vars for the whole file even when we act on one
|
|
# service; default the not-yet-known sibling (first-ever deploy) to `local` so
|
|
# interpolation doesn't warn -- harmless since --no-deps never touches it.
|
|
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
|
|
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
|
|
|
|
# Which compose services this run pulls/rolls.
|
|
case "$SERVICE" in
|
|
backend) TARGETS=(backend) ;;
|
|
frontend) TARGETS=(frontend) ;;
|
|
all) TARGETS=(backend frontend) ;;
|
|
esac
|
|
|
|
echo "==> Logging in to the registry ($REGISTRY_HOST)"
|
|
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
|
|
|
echo "==> Pulling images (backend=$BACKEND_IMAGE_TAG frontend=$FRONTEND_IMAGE_TAG; rolling: ${TARGETS[*]})"
|
|
# Retry: build-push.yml (triggered by the same push) has no ordering guarantee
|
|
# against this deploy -- Forgejo Actions `needs:` only works between jobs in ONE
|
|
# workflow file, not across the separate build-push.yml triggered by the same
|
|
# event. Confirmed live: a deploy raced ahead of the push and failed with "not
|
|
# found". A bounded retry (~5 min) covers a normal build; a genuine problem
|
|
# (bad tag, registry down) still fails loudly after that.
|
|
pull_ok=0
|
|
for i in $(seq 1 30); do
|
|
if docker compose pull "${TARGETS[@]}"; then
|
|
pull_ok=1
|
|
break
|
|
fi
|
|
echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
|
|
sleep 10
|
|
done
|
|
if [ "$pull_ok" != 1 ]; then
|
|
echo "!! docker compose pull failed after 30 attempts" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Roll only the target service(s). Backend schema migrations run inside its own
|
|
# entrypoint (alembic upgrade head) before uvicorn, so there's no separate
|
|
# migrate step; frontend is stateless.
|
|
#
|
|
# Single-service rolls use --no-deps so recreating backend doesn't also bounce
|
|
# db, and recreating frontend doesn't touch backend -- that independence is the
|
|
# whole point of the split. A full `all` deploy instead uses --remove-orphans,
|
|
# which matters when the service topology itself changes (a renamed-away
|
|
# service's old container would otherwise keep running and squat its port --
|
|
# confirmed live, this is what blocked beta's first dual-service deploy with
|
|
# "port is already allocated").
|
|
echo "==> Rolling ${TARGETS[*]}"
|
|
if [ "$SERVICE" = all ]; then
|
|
docker compose up -d --remove-orphans
|
|
else
|
|
docker compose up -d --no-deps "${TARGETS[@]}"
|
|
fi
|
|
|
|
# Persist the now-live tags so the next single-service deploy knows the
|
|
# sibling's real tag. Written after `up` so a failed pull never records a tag
|
|
# that isn't actually running.
|
|
mkdir -p "$(dirname "$TAGS_FILE")"
|
|
cat > "$TAGS_FILE" <<EOF
|
|
# Written by deploy.sh -- the image tag each service is currently running.
|
|
# Untracked (see .gitignore); lets a single-service deploy leave the other alone.
|
|
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
|
|
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
|
|
EOF
|
|
|
|
# Health check the service(s) we rolled: backend on 8137, frontend on 8080.
|
|
# (For `all`, backend's `/` serving is the readiness signal the old script used
|
|
# and the frontend depends_on backend anyway.)
|
|
declare -A HEALTH_PORTS=( [backend]=8137 [frontend]=8080 )
|
|
health_ok=1
|
|
for svc in "${TARGETS[@]}"; do
|
|
port="${HEALTH_PORTS[$svc]}"
|
|
url="http://127.0.0.1:${port}/healthz"
|
|
echo "==> Health check: $svc ($url)"
|
|
ok=0
|
|
for i in $(seq 1 30); do
|
|
if curl -fsS -o /dev/null "$url"; then ok=1; break; fi
|
|
sleep 1
|
|
done
|
|
if [ "$ok" = 1 ]; then
|
|
echo "==> OK: $svc is serving"
|
|
else
|
|
echo "!! Health check failed for $svc ($url)" >&2
|
|
health_ok=0
|
|
fi
|
|
done
|
|
|
|
if [ "$health_ok" != 1 ]; then
|
|
docker compose ps || true
|
|
for svc in "${TARGETS[@]}"; do docker compose logs --tail=50 "$svc" || true; done
|
|
exit 1
|
|
fi
|
|
|
|
# Post-deploy warm/IndexNow only make sense once the backend is (re)deployed --
|
|
# they exec inside the backend container. Skip them on a frontend-only roll.
|
|
if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then
|
|
warm_city_archives
|
|
ping_indexnow
|
|
fi
|
|
exit 0
|