thermograph/deploy/deploy.sh

#!/usr/bin/env bash
# Pull this INFRA repo's checkout up to date, then roll ONE (or all) of the
# docker-compose app services onto its separately-published image. Run on the
# VPS — each app repo's Forgejo Actions workflow invokes this over SSH (see
# .forgejo/workflows/deploy.yml in thermograph-backend / thermograph-frontend),
# and you can run it by hand too.
#
#   # roll just the backend onto a specific image:
#   ssh deploy@vps 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh'
#   # roll just the frontend:
#   ssh deploy@vps 'SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh'
#   # bring the whole stack up (both tags required):
#   ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha-<a> FRONTEND_IMAGE_TAG=sha-<b> /opt/thermograph/deploy/deploy.sh'
#
# FE/BE CI-CD split: backend and frontend are published from separate repos as
# separate images (emi/thermograph-backend/app, emi/thermograph-frontend/app),
# so a deploy targets ONE service and leaves the other's running container +
# tag untouched. Each service's live tag is persisted host-side in
# deploy/.image-tags.env (untracked -- survives the git reset below) so a
# single-service roll re-renders compose with BOTH services' real tags and
# never accidentally recreates or downgrades the sibling.
#
# This checkout is thermograph-infra, not an app repo: BRANCH is this repo's
# branch (compose files, db init, the secrets vault, this script itself);
# the *_IMAGE_TAG values are the separately-published app images to run -- the
# two axes are independent and rarely change together.
set -euo pipefail

APP_DIR="${APP_DIR:-/opt/thermograph}"
BRANCH="${BRANCH:-main}"
# Which service this deploy rolls: backend | frontend | all. Defaults to `all`
# (a full-stack bring-up) so a by-hand run with both tags still works; the
# per-repo deploy.yml workflows always pass an explicit single service.
SERVICE="${SERVICE:-all}"
HEALTH_PORT="${HEALTH_PORT:-8137}"
cd "$APP_DIR"

case "$SERVICE" in
  backend|frontend|all) ;;
  *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
esac

# Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose
# interpolation and are also loaded into the backend container via env_file.
#
# When this host is configured for SOPS (an age key + /etc/thermograph/secrets-env),
# first render /etc/thermograph.env from the committed encrypted source of truth
# (deploy/secrets/*.yaml) so a key rotation is just an edit+commit+deploy. The guard
# on the helper's existence keeps the very deploy that INTRODUCES this file safe: on
# the first pass the checkout may predate it (it arrives with the git reset below,
# after which deploy.sh re-execs), so a missing helper simply falls back to the
# existing /etc/thermograph.env. Then source it so a by-hand run interpolates the
# same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/.
if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then
  # shellcheck source=deploy/render-secrets.sh
  . "$APP_DIR/deploy/render-secrets.sh"
  render_thermograph_secrets "$APP_DIR"
fi
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a

# Pre-warm the ~750 city-page archives so /climate pages serve from cache and a
# search-engine crawl never bursts the archive API quota. Detached inside the
# backend container (compose exec -d), idempotent (skips already-cached cells),
# so it never blocks the deploy or health check and is cheap on every deploy
# after the first full warm.
warm_city_archives() {
  echo "==> Warming city-page archives in the background (backend:/app/logs/warm-cities.log)"
  docker compose exec -d backend sh -c \
    'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
}

# Notify IndexNow (Bing / DuckDuckGo / Yandex) of the site's URLs, but only when
# the set of pages actually changed (a new/removed city) — code-only deploys skip.
# Best-effort: never fails the deploy.
ping_indexnow() {
  echo "==> Pinging IndexNow (only if the URL set changed)"
  local base="${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
  docker compose exec -T backend python indexnow.py --if-changed "$base" \
    || echo "!! IndexNow ping failed (non-fatal)" >&2
}

echo "==> Fetching $BRANCH"
git fetch --prune origin "$BRANCH"
git reset --hard "origin/$BRANCH"

# Re-exec: git reset --hard just rewrote this very file's bytes on disk while
# it's still running. bash reads a script via buffered, byte-offset I/O, so
# anything AFTER this point in the OLD execution can read from the wrong
# offset once the file's size/content changed underneath it -- a classic
# self-modifying-script footgun. Confirmed live: after this PR added ~15
# lines above, one deploy ran with the OLD "Building images" log lines even
# though `git status` showed the checkout correctly at the NEW commit --
# the file changed under a running interpreter, not the checkout. Restart
# fresh from the now-updated file so everything after this line is
# guaranteed self-consistent. Guarded so the second invocation doesn't
# fetch+reset+re-exec forever.
if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then
  export DEPLOY_SH_REEXECED=1
  exec "$0" "$@"
fi

# Registry-pull cutover: pull the image each app repo's build-push.yml already
# built and pushed, instead of building in place. This checkout is
# thermograph-infra, not an app repo, so there's no "current commit" to derive
# a tag from -- the caller (the deploying repo's deploy.yml) exports its own
# service's tag (BACKEND_IMAGE_TAG or FRONTEND_IMAGE_TAG = sha-<12 hex> of the
# app commit, or a semver tag).
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
export REGISTRY_HOST BACKEND_IMAGE_PATH FRONTEND_IMAGE_PATH

# Load the last-deployed tag for BOTH services first, so a single-service roll
# still renders compose with the sibling's real, currently-running tag (never a
# bare `local` that would recreate/downgrade it). The incoming env for the
# service being deployed then overrides its line below. This file is untracked
# (see .gitignore), so `git reset --hard` above leaves it in place.
TAGS_FILE="$APP_DIR/deploy/.image-tags.env"
if [ -f "$TAGS_FILE" ]; then
  set -a; . "$TAGS_FILE"; set +a
fi

# Guard: the service(s) being rolled MUST have a concrete tag supplied now (the
# sibling's may come from the persisted file). `all` needs both.
case "$SERVICE" in
  backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> for a backend deploy}" ;;
  frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> for a frontend deploy}" ;;
  all)
    : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
    : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
    ;;
esac
# Compose interpolates both vars for the whole file even when we act on one
# service; default the not-yet-known sibling (first-ever deploy) to `local` so
# interpolation doesn't warn -- harmless since --no-deps never touches it.
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"

# Which compose services this run pulls/rolls.
case "$SERVICE" in
  backend) TARGETS=(backend) ;;
  frontend) TARGETS=(frontend) ;;
  all) TARGETS=(backend frontend) ;;
esac

# Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml,
# deploy-prod.yml, deploy-dev on the LAN runner) don't pass REGISTRY_TOKEN --
# the host is already `docker login`ed to the registry (persistent cred in
# ~/.docker/config.json), so an unconditional login with an empty token would
# abort the deploy under `set -e`. Use the token if present, else trust the
# host's existing cred; a genuine auth problem then fails loudly at `pull`.
if [ -n "${REGISTRY_TOKEN:-}" ]; then
  echo "==> Logging in to the registry ($REGISTRY_HOST)"
  echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
else
  echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST"
fi

echo "==> Pulling images (backend=$BACKEND_IMAGE_TAG frontend=$FRONTEND_IMAGE_TAG; rolling: ${TARGETS[*]})"
# Retry: build-push.yml (triggered by the same push) has no ordering guarantee
# against this deploy -- Forgejo Actions `needs:` only works between jobs in ONE
# workflow file, not across the separate build-push.yml triggered by the same
# event. Confirmed live: a deploy raced ahead of the push and failed with "not
# found". A bounded retry (~5 min) covers a normal build; a genuine problem
# (bad tag, registry down) still fails loudly after that.
pull_ok=0
for i in $(seq 1 30); do
  if docker compose pull "${TARGETS[@]}"; then
    pull_ok=1
    break
  fi
  echo "   pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
  sleep 10
done
if [ "$pull_ok" != 1 ]; then
  echo "!! docker compose pull failed after 30 attempts" >&2
  exit 1
fi

# Roll only the target service(s). Backend schema migrations run inside its own
# entrypoint (alembic upgrade head) before uvicorn, so there's no separate
# migrate step; frontend is stateless.
#
# Single-service rolls use --no-deps so recreating backend doesn't also bounce
# db, and recreating frontend doesn't touch backend -- that independence is the
# whole point of the split. A full `all` deploy instead uses --remove-orphans,
# which matters when the service topology itself changes (a renamed-away
# service's old container would otherwise keep running and squat its port --
# confirmed live, this is what blocked beta's first dual-service deploy with
# "port is already allocated").
echo "==> Rolling ${TARGETS[*]}"
if [ "$SERVICE" = all ]; then
  docker compose up -d --remove-orphans
else
  docker compose up -d --no-deps "${TARGETS[@]}"
fi

# Persist the now-live tags so the next single-service deploy knows the
# sibling's real tag. Written after `up` so a failed pull never records a tag
# that isn't actually running.
mkdir -p "$(dirname "$TAGS_FILE")"
cat > "$TAGS_FILE" <<EOF
# Written by deploy.sh -- the image tag each service is currently running.
# Untracked (see .gitignore); lets a single-service deploy leave the other alone.
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
EOF

# Health check the service(s) we rolled: backend on 8137, frontend on 8080.
# (For `all`, backend's `/` serving is the readiness signal the old script used
# and the frontend depends_on backend anyway.)
declare -A HEALTH_PORTS=( [backend]=8137 [frontend]=8080 )
health_ok=1
for svc in "${TARGETS[@]}"; do
  port="${HEALTH_PORTS[$svc]}"
  url="http://127.0.0.1:${port}/healthz"
  echo "==> Health check: $svc ($url)"
  ok=0
  for i in $(seq 1 30); do
    if curl -fsS -o /dev/null "$url"; then ok=1; break; fi
    sleep 1
  done
  if [ "$ok" = 1 ]; then
    echo "==> OK: $svc is serving"
  else
    echo "!! Health check failed for $svc ($url)" >&2
    health_ok=0
  fi
done

if [ "$health_ok" != 1 ]; then
  docker compose ps || true
  for svc in "${TARGETS[@]}"; do docker compose logs --tail=50 "$svc" || true; done
  exit 1
fi

# Post-deploy warm/IndexNow only make sense once the backend is (re)deployed --
# they exec inside the backend container. Skip them on a frontend-only roll.
if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then
  warm_city_archives
  ping_indexnow
fi
exit 0