#!/usr/bin/env bash # Pull this INFRA repo's checkout up to date, then roll ONE (or all) of the # docker-compose app services onto its separately-published image. Run on the # VPS — each app repo's Forgejo Actions workflow invokes this over SSH (see # .forgejo/workflows/deploy.yml in thermograph-backend / thermograph-frontend), # and you can run it by hand too. # # # roll just the backend onto a specific image: # ssh deploy@vps 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh' # # roll just the frontend: # ssh deploy@vps 'SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh' # # bring the whole stack up (both tags required): # ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- /opt/thermograph/deploy/deploy.sh' # # FE/BE CI-CD split: backend and frontend are published from separate repos as # separate images (emi/thermograph-backend/app, emi/thermograph-frontend/app), # so a deploy targets ONE service and leaves the other's running container + # tag untouched. Each service's live tag is persisted host-side in # deploy/.image-tags.env (untracked -- survives the git reset below) so a # single-service roll re-renders compose with BOTH services' real tags and # never accidentally recreates or downgrades the sibling. # # This checkout is thermograph-infra, not an app repo: BRANCH is this repo's # branch (compose files, db init, the secrets vault, this script itself); # the *_IMAGE_TAG values are the separately-published app images to run -- the # two axes are independent and rarely change together. set -euo pipefail APP_DIR="${APP_DIR:-/opt/thermograph}" BRANCH="${BRANCH:-main}" # Which service this deploy rolls: backend | frontend | all. Defaults to `all` # (a full-stack bring-up) so a by-hand run with both tags still works; the # per-repo deploy.yml workflows always pass an explicit single service. SERVICE="${SERVICE:-all}" HEALTH_PORT="${HEALTH_PORT:-8137}" cd "$APP_DIR" case "$SERVICE" in backend|frontend|all) ;; *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac # Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose # interpolation and are also loaded into the backend container via env_file. # # When this host is configured for SOPS (an age key + /etc/thermograph/secrets-env), # first render /etc/thermograph.env from the committed encrypted source of truth # (deploy/secrets/*.yaml) so a key rotation is just an edit+commit+deploy. The guard # on the helper's existence keeps the very deploy that INTRODUCES this file safe: on # the first pass the checkout may predate it (it arrives with the git reset below, # after which deploy.sh re-execs), so a missing helper simply falls back to the # existing /etc/thermograph.env. Then source it so a by-hand run interpolates the # same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/. if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then # shellcheck source=deploy/render-secrets.sh . "$APP_DIR/deploy/render-secrets.sh" render_thermograph_secrets "$APP_DIR" fi set -a; . /etc/thermograph.env 2>/dev/null || true; set +a # Pre-warm the ~750 city-page archives so /climate pages serve from cache and a # search-engine crawl never bursts the archive API quota. Detached inside the # backend container (compose exec -d), idempotent (skips already-cached cells), # so it never blocks the deploy or health check and is cheap on every deploy # after the first full warm. warm_city_archives() { echo "==> Warming city-page archives in the background (backend:/app/logs/warm-cities.log)" docker compose exec -d backend sh -c \ 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true } # Notify IndexNow (Bing / DuckDuckGo / Yandex) of the site's URLs, but only when # the set of pages actually changed (a new/removed city) — code-only deploys skip. # Best-effort: never fails the deploy. ping_indexnow() { echo "==> Pinging IndexNow (only if the URL set changed)" local base="${THERMOGRAPH_BASE_URL:-https://thermograph.org}" docker compose exec -T backend python indexnow.py --if-changed "$base" \ || echo "!! IndexNow ping failed (non-fatal)" >&2 } echo "==> Fetching $BRANCH" git fetch --prune origin "$BRANCH" git reset --hard "origin/$BRANCH" # Re-exec: git reset --hard just rewrote this very file's bytes on disk while # it's still running. bash reads a script via buffered, byte-offset I/O, so # anything AFTER this point in the OLD execution can read from the wrong # offset once the file's size/content changed underneath it -- a classic # self-modifying-script footgun. Confirmed live: after this PR added ~15 # lines above, one deploy ran with the OLD "Building images" log lines even # though `git status` showed the checkout correctly at the NEW commit -- # the file changed under a running interpreter, not the checkout. Restart # fresh from the now-updated file so everything after this line is # guaranteed self-consistent. Guarded so the second invocation doesn't # fetch+reset+re-exec forever. if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then export DEPLOY_SH_REEXECED=1 exec "$0" "$@" fi # Registry-pull cutover: pull the image each app repo's build-push.yml already # built and pushed, instead of building in place. This checkout is # thermograph-infra, not an app repo, so there's no "current commit" to derive # a tag from -- the caller (the deploying repo's deploy.yml) exports its own # service's tag (BACKEND_IMAGE_TAG or FRONTEND_IMAGE_TAG = sha-<12 hex> of the # app commit, or a semver tag). REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}" export REGISTRY_HOST BACKEND_IMAGE_PATH FRONTEND_IMAGE_PATH # Load the last-deployed tag for BOTH services first, so a single-service roll # still renders compose with the sibling's real, currently-running tag (never a # bare `local` that would recreate/downgrade it). The incoming env for the # service being deployed then overrides its line below. This file is untracked # (see .gitignore), so `git reset --hard` above leaves it in place. TAGS_FILE="$APP_DIR/deploy/.image-tags.env" if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a fi # Guard: the service(s) being rolled MUST have a concrete tag supplied now (the # sibling's may come from the persisted file). `all` needs both. case "$SERVICE" in backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> for a backend deploy}" ;; frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> for a frontend deploy}" ;; all) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}" : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}" ;; esac # Compose interpolates both vars for the whole file even when we act on one # service; default the not-yet-known sibling (first-ever deploy) to `local` so # interpolation doesn't warn -- harmless since --no-deps never touches it. export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" # Which compose services this run pulls/rolls. case "$SERVICE" in backend) TARGETS=(backend) ;; frontend) TARGETS=(frontend) ;; all) TARGETS=(backend frontend) ;; esac # Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml, # deploy-prod.yml, deploy-dev on the LAN runner) don't pass REGISTRY_TOKEN -- # the host is already `docker login`ed to the registry (persistent cred in # ~/.docker/config.json), so an unconditional login with an empty token would # abort the deploy under `set -e`. Use the token if present, else trust the # host's existing cred; a genuine auth problem then fails loudly at `pull`. if [ -n "${REGISTRY_TOKEN:-}" ]; then echo "==> Logging in to the registry ($REGISTRY_HOST)" echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin else echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST" fi echo "==> Pulling images (backend=$BACKEND_IMAGE_TAG frontend=$FRONTEND_IMAGE_TAG; rolling: ${TARGETS[*]})" # Retry: build-push.yml (triggered by the same push) has no ordering guarantee # against this deploy -- Forgejo Actions `needs:` only works between jobs in ONE # workflow file, not across the separate build-push.yml triggered by the same # event. Confirmed live: a deploy raced ahead of the push and failed with "not # found". A bounded retry (~5 min) covers a normal build; a genuine problem # (bad tag, registry down) still fails loudly after that. pull_ok=0 for i in $(seq 1 30); do if docker compose pull "${TARGETS[@]}"; then pull_ok=1 break fi echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2 sleep 10 done if [ "$pull_ok" != 1 ]; then echo "!! docker compose pull failed after 30 attempts" >&2 exit 1 fi # Roll only the target service(s). Backend schema migrations run inside its own # entrypoint (alembic upgrade head) before uvicorn, so there's no separate # migrate step; frontend is stateless. # # Single-service rolls use --no-deps so recreating backend doesn't also bounce # db, and recreating frontend doesn't touch backend -- that independence is the # whole point of the split. A full `all` deploy instead uses --remove-orphans, # which matters when the service topology itself changes (a renamed-away # service's old container would otherwise keep running and squat its port -- # confirmed live, this is what blocked beta's first dual-service deploy with # "port is already allocated"). echo "==> Rolling ${TARGETS[*]}" if [ "$SERVICE" = all ]; then docker compose up -d --remove-orphans else docker compose up -d --no-deps "${TARGETS[@]}" fi # Persist the now-live tags so the next single-service deploy knows the # sibling's real tag. Written after `up` so a failed pull never records a tag # that isn't actually running. mkdir -p "$(dirname "$TAGS_FILE")" cat > "$TAGS_FILE" < Health check: $svc ($url)" ok=0 for i in $(seq 1 30); do if curl -fsS -o /dev/null "$url"; then ok=1; break; fi sleep 1 done if [ "$ok" = 1 ]; then echo "==> OK: $svc is serving" else echo "!! Health check failed for $svc ($url)" >&2 health_ok=0 fi done if [ "$health_ok" != 1 ]; then docker compose ps || true for svc in "${TARGETS[@]}"; do docker compose logs --tail=50 "$svc" || true; done exit 1 fi # Post-deploy warm/IndexNow only make sense once the backend is (re)deployed -- # they exec inside the backend container. Skip them on a frontend-only roll. if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then warm_city_archives ping_indexnow fi exit 0