thermograph/infra/deploy/deploy.sh

#!/usr/bin/env bash
# Pull this INFRA repo's checkout up to date, then roll ONE (or all) of the
# docker-compose app services onto its separately-published image. Run on the
# VPS — each app repo's Forgejo Actions workflow invokes this over SSH (see
# .forgejo/workflows/deploy.yml in thermograph-backend / thermograph-frontend),
# and you can run it by hand too.
#
#   # roll just the backend onto a specific image:
#   ssh deploy@vps 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh'
#   # roll just the frontend:
#   ssh deploy@vps 'SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh'
#   # bring the whole stack up (both tags required):
#   ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha-<a> FRONTEND_IMAGE_TAG=sha-<b> /opt/thermograph/infra/deploy/deploy.sh'
#
# FE/BE CI-CD split: backend and frontend are published from separate repos as
# separate images (emi/thermograph/backend, emi/thermograph/frontend),
# so a deploy targets ONE service and leaves the other's running container +
# tag untouched. Each service's live tag is persisted host-side in
# deploy/.image-tags.env (untracked -- survives the git reset below) so a
# single-service roll re-renders compose with BOTH services' real tags and
# never accidentally recreates or downgrades the sibling.
#
# This checkout is the monorepo (compose + deploy live under infra/), not an app-only checkout: BRANCH is this repo's
# branch (compose files, db init, the secrets vault, this script itself);
# the *_IMAGE_TAG values are the separately-published app images to run -- the
# two axes are independent and rarely change together.
set -euo pipefail

# --- which environment is this? ------------------------------------------------
# vps2 runs beta AND prod, so "the host" no longer answers this — the caller does,
# via THERMOGRAPH_ENV (the deploy workflow always passes it). A by-hand run on a
# single-environment box still falls back to the host marker, and a box with
# neither still defaults to prod's historical paths, so nothing about an existing
# single-env host changes.
#
# Resolved from the SCRIPT'S OWN LOCATION, not a hardcoded /opt/thermograph:
# invoking /opt/thermograph-beta/infra/deploy/deploy.sh must act on the beta
# checkout even if something in the environment says otherwise. That is also the
# check that catches the one genuinely dangerous mistake on a two-environment
# host — running prod's checkout with THERMOGRAPH_ENV=beta, or the reverse.
SELF_DIR=$(cd "$(dirname "$0")" && pwd)
SELF_APP_DIR=$(cd "$SELF_DIR/../.." && pwd)

# Guarded exactly like render-secrets.sh below: the deploy that INTRODUCES this
# file runs with a checkout that predates it (it arrives with the git reset
# further down, after which deploy.sh re-execs). Missing => keep the pre-split
# behaviour rather than fail.
if [ -f "$SELF_DIR/env-topology.sh" ]; then
  # shellcheck source=infra/deploy/env-topology.sh
  . "$SELF_DIR/env-topology.sh"
  ENV_NAME=$(thermograph_env_name)
  # Nothing to go on anywhere: this is a pre-split host whose paths are prod's.
  [ -n "$ENV_NAME" ] || ENV_NAME=prod
  thermograph_topology "$ENV_NAME"
  if [ "$SELF_APP_DIR" != "$TG_APP_DIR" ] && [ -z "${APP_DIR:-}" ]; then
    echo "!! environment/checkout mismatch: THERMOGRAPH_ENV=$ENV_NAME expects" >&2
    echo "!!   $TG_APP_DIR but this script lives in $SELF_APP_DIR." >&2
    echo "!! On vps2 that means beta and prod have been crossed. Refusing to deploy." >&2
    echo "!! If this checkout really is $ENV_NAME (a rehearsal copy, a relocated" >&2
    echo "!! checkout), say so explicitly:  APP_DIR=$SELF_APP_DIR ..." >&2
    exit 2
  fi
  APP_DIR="${APP_DIR:-$TG_APP_DIR}"
  BRANCH="${BRANCH:-$TG_BRANCH}"
  ENV_FILE="$TG_ENV_FILE"
  DEPLOY_MODE="$TG_DEPLOY_MODE"
  [ "$TG_SKIP_COMMON" = 1 ] && export THERMOGRAPH_SECRETS_SKIP_COMMON=1
  # The second pass (after the re-exec) must resolve the SAME environment, and
  # the stack path needs it too.
  export THERMOGRAPH_ENV="$ENV_NAME"
else
  ENV_NAME=""
  APP_DIR="${APP_DIR:-/opt/thermograph}"
  BRANCH="${BRANCH:-main}"
  ENV_FILE=/etc/thermograph.env
  DEPLOY_MODE=""
fi

# Monorepo layout: git operations act on the checkout root ($APP_DIR); all
# compose files and deploy assets live under infra/.
INFRA_DIR="$APP_DIR/infra"
# Which service this deploy rolls: backend | frontend | all. Defaults to `all`
# (a full-stack bring-up) so a by-hand run with both tags still works; the
# per-repo deploy.yml workflows always pass an explicit single service.
SERVICE="${SERVICE:-all}"
HEALTH_PORT="${HEALTH_PORT:-8137}"
cd "$APP_DIR"

case "$SERVICE" in
  backend|frontend|all) ;;
  *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
esac

# Serialize deploys on this host. Backend and frontend deploy from SEPARATE
# repos whose workflows can fire for the same push within seconds of each
# other, and both SSH into this one checkout: concurrent runs race on the
# `git reset` below, the shared compose project, and the .image-tags.env
# read/modify/write (a lost update there re-rolls the sibling onto a stale
# tag). flock makes the second deploy wait its turn instead. The lock fd is
# inherited across the self re-exec below, so the lock spans the whole run;
# -w 600 bounds the wait (a deploy holding the lock >10 min is already
# broken), after which this exits non-zero and the CI job fails loudly.
DEPLOY_LOCK="$INFRA_DIR/deploy/.deploy.lock"
if [ -z "${DEPLOY_SH_FLOCKED:-}" ]; then
  export DEPLOY_SH_FLOCKED=1
  exec flock -w 600 "$DEPLOY_LOCK" "$0" "$@"
fi

# Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose
# interpolation and are also loaded into the backend container via env_file.
#
# When this host is configured for SOPS (an age key + /etc/thermograph/secrets-env),
# first render /etc/thermograph.env from the committed encrypted source of truth
# (deploy/secrets/*.yaml) so a key rotation is just an edit+commit+deploy. The guard
# on the helper's existence keeps the very deploy that INTRODUCES this file safe: on
# the first pass the checkout may predate it (it arrives with the git reset below,
# after which deploy.sh re-execs), so a missing helper simply falls back to the
# existing /etc/thermograph.env. Then source it so a by-hand run interpolates the
# same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/.
#
# $ENV_FILE, not a hardcoded /etc/thermograph.env: on vps2 prod renders
# prod.yaml there while beta renders beta.yaml to /etc/thermograph-beta.env.
# One file per environment, never shared.
if [ -f "$INFRA_DIR/deploy/render-secrets.sh" ]; then
  # shellcheck source=infra/deploy/render-secrets.sh
  . "$INFRA_DIR/deploy/render-secrets.sh"
  render_thermograph_secrets "$INFRA_DIR" "$ENV_NAME" "$ENV_FILE"
fi
# $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at
# lint time, so don't ask shellcheck to follow it.
set -a
# shellcheck source=/dev/null
. "$ENV_FILE" 2>/dev/null || true
set +a

# Pre-warm the ~750 city-page archives so /climate pages serve from cache and a
# search-engine crawl never bursts the archive API quota. Detached inside the
# backend container (compose exec -d), idempotent (skips already-cached cells),
# so it never blocks the deploy or health check and is cheap on every deploy
# after the first full warm.
warm_city_archives() {
  echo "==> Warming city-page archives in the background (backend:/app/logs/warm-cities.log)"
  docker compose exec -d backend sh -c \
    'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
}

# Notify IndexNow (Bing / DuckDuckGo / Yandex) of the site's URLs, but only when
# the set of pages actually changed (a new/removed city) — code-only deploys skip.
# Best-effort: never fails the deploy.
ping_indexnow() {
  echo "==> Pinging IndexNow (only if the URL set changed)"
  local base="${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
  docker compose exec -T backend python indexnow.py --if-changed "$base" \
    || echo "!! IndexNow ping failed (non-fatal)" >&2
}

echo "==> Fetching $BRANCH"
git fetch --prune origin "$BRANCH"
git reset --hard "origin/$BRANCH"

# Re-exec: git reset --hard just rewrote this very file's bytes on disk while
# it's still running. bash reads a script via buffered, byte-offset I/O, so
# anything AFTER this point in the OLD execution can read from the wrong
# offset once the file's size/content changed underneath it -- a classic
# self-modifying-script footgun. Confirmed live: after this PR added ~15
# lines above, one deploy ran with the OLD "Building images" log lines even
# though `git status` showed the checkout correctly at the NEW commit --
# the file changed under a running interpreter, not the checkout. Restart
# fresh from the now-updated file so everything after this line is
# guaranteed self-consistent. Guarded so the second invocation doesn't
# fetch+reset+re-exec forever.
if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then
  export DEPLOY_SH_REEXECED=1
  exec "$0" "$@"
fi

# Stack-mode routing: prod and beta are both Swarm stacks, dev is compose. The
# mode is a property of the ENVIRONMENT (env-topology.sh), not of the host --
# vps2 runs two stacks, and a host-wide /etc/thermograph/deploy-mode marker
# cannot describe a host that runs more than one environment. The marker is
# still honoured when the topology file is absent (a checkout that predates it).
# Checked AFTER the reset+re-exec so the stack script is always the freshly
# pulled one, and the SERVICE/tag contract passes through unchanged -- the
# deploy workflow never needs to know which mode an environment runs.
if [ -z "$DEPLOY_MODE" ]; then
  DEPLOY_MODE=$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)
fi
if [ "$DEPLOY_MODE" = "stack" ]; then
  exec bash "$INFRA_DIR/deploy/stack/deploy-stack.sh"
fi

# Monorepo: run every docker compose command from infra/ (the git operations
# above ran at the checkout root). The compose project name is pinned in the
# file itself (`name: thermograph`), so moving the working directory does NOT
# rename the project and recreate the stack under a second name.
cd "$INFRA_DIR"

# Compose-mode environments (dev) carry their project name, file list and bind
# address in the topology table. Set only when not already in the environment,
# so deploy-dev.sh's own exports and a by-hand override both still win.
if [ -n "${TG_COMPOSE_PROJECT:-}" ]; then
  export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-$TG_COMPOSE_PROJECT}"
  export COMPOSE_FILE="${COMPOSE_FILE:-$TG_COMPOSE_FILE}"
fi
# Which address the dev overlay publishes on. Defaults to loopback in the
# compose file; dev on vps1 sets the mesh address here, because a public VPS
# must never publish an unreviewed branch's stack on 0.0.0.0.
if [ -n "${TG_BIND_ADDR:-}" ]; then
  export DEV_BIND_ADDR="${DEV_BIND_ADDR:-$TG_BIND_ADDR}"
fi

# Registry-pull cutover: pull the image each app repo's build-push.yml already
# built and pushed, instead of building in place. This checkout is
# thermograph-infra, not an app repo, so there's no "current commit" to derive
# a tag from -- the caller (the deploying repo's deploy.yml) exports its own
# service's tag (BACKEND_IMAGE_TAG or FRONTEND_IMAGE_TAG = sha-<12 hex> of the
# app commit, or a semver tag).
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
export REGISTRY_HOST BACKEND_IMAGE_PATH FRONTEND_IMAGE_PATH

# Load the last-deployed tag for BOTH services first, so a single-service roll
# still renders compose with the sibling's real, currently-running tag (never a
# bare `local` that would recreate/downgrade it). The incoming env for the
# service being deployed then overrides its line below. This file is untracked
# (see .gitignore), so `git reset --hard` above leaves it in place.
TAGS_FILE="$INFRA_DIR/deploy/.image-tags.env"
# Capture the tags the caller explicitly passed BEFORE sourcing -- they must
# WIN. The file only supplies the *sibling's* last-known tag; sourcing it
# unconditionally would clobber an incoming tag (e.g. a frontend deploy whose
# FRONTEND_IMAGE_TAG got overwritten by the stale `local` the first backend-only
# deploy persisted for the not-yet-known sibling -> pull `:local` -> "manifest
# unknown"). So source for the sibling, then re-apply the caller's own value.
_incoming_backend="${BACKEND_IMAGE_TAG:-}"
_incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
if [ -f "$TAGS_FILE" ]; then
  set -a
  # shellcheck source=/dev/null # untracked runtime artifact this script writes below
  . "$TAGS_FILE"
  set +a
fi
[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"

# Guard: the service(s) being rolled MUST have a concrete tag supplied now (the
# sibling's may come from the persisted file). `all` needs both.
case "$SERVICE" in
  backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> for a backend deploy}" ;;
  frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> for a frontend deploy}" ;;
  all)
    : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
    : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
    ;;
esac
# Compose interpolates both vars for the whole file even when we act on one
# service; default the not-yet-known sibling (first-ever deploy) to `local` so
# interpolation doesn't warn -- harmless since --no-deps never touches it.
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"

# Which compose services this run pulls/rolls.
#
# `daemon` and `lake` ride with `backend` and are never targets on their own.
# Both run the SAME image at the SAME tag (a second binary and a second role
# inside the backend image): the daemon talks to the web process over the
# /internal/* contract, the lake serves it history — rolling one without the
# other is exactly the version skew those contracts have no negotiation for.
# Pairing them here is also what makes the services exist at all: a
# single-service deploy runs `up -d --no-deps <targets>`, so a service listed
# only in compose but absent from TARGETS would never be created on a
# backend-only deploy.
case "$SERVICE" in
  backend) TARGETS=(backend lake daemon) ;;
  frontend) TARGETS=(frontend) ;;
  all) TARGETS=(backend lake frontend daemon) ;;
esac

# Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml,
# deploy-prod.yml, deploy-dev on the LAN runner) don't pass REGISTRY_TOKEN --
# the host is already `docker login`ed to the registry (persistent cred in
# ~/.docker/config.json), so an unconditional login with an empty token would
# abort the deploy under `set -e`. Use the token if present, else trust the
# host's existing cred; a genuine auth problem then fails loudly at `pull`.
if [ -n "${REGISTRY_TOKEN:-}" ]; then
  echo "==> Logging in to the registry ($REGISTRY_HOST)"
  echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
else
  echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST"
fi

echo "==> Pulling images (backend=$BACKEND_IMAGE_TAG frontend=$FRONTEND_IMAGE_TAG; rolling: ${TARGETS[*]})"
# Retry: build-push.yml (triggered by the same push) has no ordering guarantee
# against this deploy -- Forgejo Actions `needs:` only works between jobs in ONE
# workflow file, not across the separate build-push.yml triggered by the same
# event. Confirmed live: a deploy raced ahead of the push and failed with "not
# found". A bounded retry (~5 min) covers a normal build; a genuine problem
# (bad tag, registry down) still fails loudly after that.
pull_ok=0
for i in $(seq 1 30); do
  if docker compose pull "${TARGETS[@]}"; then
    pull_ok=1
    break
  fi
  echo "   pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
  sleep 10
done
if [ "$pull_ok" != 1 ]; then
  echo "!! docker compose pull failed after 30 attempts" >&2
  exit 1
fi

# The daemon binary ships INSIDE the backend image, but infra and app images
# advance on different axes by design: this checkout tracks `main`, while image
# tags are env-staged (prod's come from `release`). So a host can legitimately
# be asked to roll a backend image OLDER than this compose file -- one built
# before the daemon existed and with no /usr/local/bin/thermograph-daemon in it.
# Creating the service anyway would leave a container crash-looping on a missing
# binary, on a deploy that otherwise succeeded. Probe the image we are actually
# about to roll and drop the daemon from this run if it can't support it; the
# next deploy of a tag that has it picks it up with no further action.
case " ${TARGETS[*]} " in
  *" daemon "*)
    # Built directly from the same vars docker-compose.yml's `daemon.image:`
    # interpolates (REGISTRY_HOST/BACKEND_IMAGE_PATH/BACKEND_IMAGE_TAG),
    # NOT via `docker compose config --images daemon`: that command does not
    # actually filter to the named service (confirmed live on Compose
    # v5.3.1 -- it prints every service's image, one per line, in file
    # order) so `| head -1` silently grabbed db's image instead. The probe
    # then always found no daemon binary in a Postgres image and dropped
    # daemon from EVERY backend deploy, regardless of what the real backend
    # image contained -- reproduced and confirmed against beta directly.
    daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG}"
    if ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then
      echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run"
      kept=()
      for t in "${TARGETS[@]}"; do
        [ "$t" = daemon ] || kept+=("$t")
      done
      TARGETS=("${kept[@]}")
    fi
    ;;
esac

# Roll only the target service(s). Backend schema migrations run inside its own
# entrypoint (alembic upgrade head) before uvicorn, so there's no separate
# migrate step; frontend is stateless.
#
# Single-service rolls use --no-deps so recreating backend doesn't also bounce
# db, and recreating frontend doesn't touch backend -- that independence is the
# whole point of the split. A full `all` deploy instead uses --remove-orphans,
# which matters when the service topology itself changes (a renamed-away
# service's old container would otherwise keep running and squat its port --
# confirmed live, this is what blocked beta's first dual-service deploy with
# "port is already allocated").
echo "==> Rolling ${TARGETS[*]}"
if [ "$SERVICE" = all ]; then
  docker compose up -d --remove-orphans
else
  docker compose up -d --no-deps "${TARGETS[@]}"
fi

# Persist the now-live tags so the next single-service deploy knows the
# sibling's real tag. Written after `up` so a failed pull never records a tag
# that isn't actually running.
mkdir -p "$(dirname "$TAGS_FILE")"
cat > "$TAGS_FILE" <<EOF
# Written by deploy.sh -- the image tag each service is currently running.
# Untracked (see .gitignore); lets a single-service deploy leave the other alone.
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
EOF

# Health check the service(s) we rolled: backend on 8137, frontend on 8080.
# (For `all`, backend's `/` serving is the readiness signal the old script used
# and the frontend depends_on backend anyway.)
# Health via each container's own HEALTHCHECK (docker inspect), NOT a host-port
# curl: the dev overlay leaves the frontend port UNpublished (reached through the
# backend's _proxy_to_frontend), so a localhost:8080 curl spuriously fails there.
# Both images HEALTHCHECK-curl /healthz internally, so this works published or not.
health_ok=1
for svc in "${TARGETS[@]}"; do
  cid=$(docker compose ps -q "$svc" 2>/dev/null)
  echo "==> Health check: $svc (container health)"
  ok=0; st=unknown
  for i in $(seq 1 40); do
    st=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null || echo gone)
    [ "$st" = healthy ] && { ok=1; break; }
    if [ "$st" = none ] && [ "$(docker inspect --format '{{.State.Status}}' "$cid" 2>/dev/null)" = running ]; then ok=1; break; fi
    sleep 2
  done
  if [ "$ok" = 1 ]; then
    echo "==> OK: $svc is healthy"
  else
    echo "!! Health check failed for $svc (status=$st)" >&2
    health_ok=0
  fi
done

if [ "$health_ok" != 1 ]; then
  docker compose ps || true
  for svc in "${TARGETS[@]}"; do docker compose logs --tail=50 "$svc" || true; done
  exit 1
fi

# Every deploy pulls a new sha-tagged image and nothing ever removed the old
# ones -- hosts accumulate gigabytes of dead tags at one per app commit. Keep
# only the tags recorded as now-live (both services) and delete other tags of
# the two app-image repos. Best-effort and after the health gate, so a failed
# roll never garbage-collects the image a rollback would need; docker also
# refuses to remove an image any container still uses.
echo "==> Pruning old app-image tags"
_be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}"
_fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}"
docker images --format '{{.Repository}}:{{.Tag}}' \
  | grep -E "^(${_be_repo}|${_fe_repo}):" \
  | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
  | xargs -r docker rmi 2>/dev/null || true

# Post-deploy warm/IndexNow only make sense once the backend is (re)deployed --
# they exec inside the backend container. Skip them on a frontend-only roll.
#
# TG_POST_DEPLOY gates them per ENVIRONMENT (env-topology.sh): prod does both,
# beta and dev do neither. Both are wrong outside prod. The warm spends the
# shared upstream archive quota to fill a cache nobody serves from, and the
# IndexNow ping announces URLs to Bing/DuckDuckGo/Yandex — from dev that means
# submitting a mesh address no crawler can reach.
#
# This gate existed in deploy/stack/deploy-stack.sh (beta, prod) but was missed
# here, on the compose path, which is the ONE environment where the default is
# wrong: dev. Standing dev up on vps1 duly warmed 1000 cities and tried to
# submit 14,007 URLs to IndexNow before this was fixed.
#
# Defaults to 1 when unset so a checkout predating env-topology.sh keeps prod's
# historical behaviour.
if { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; } && [ "${TG_POST_DEPLOY:-1}" = 1 ]; then
  warm_city_archives
  ping_indexnow
fi
exit 0