thermograph/infra/deploy/stack/deploy-stack.sh

#!/usr/bin/env bash
# Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh,
# speaking the SAME contract the app repos' workflows already use
# (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so
# switching a host to stack mode needs no workflow changes: deploy.sh execs
# this when /etc/thermograph/deploy-mode contains "stack".
#
# What a roll does here vs compose:
#   backend  -> one-shot migrate, then `docker service update --image` on
#               web AND worker (same image; start-first, health-gated,
#               auto-rollback on failure).
#   frontend -> `docker service update --image` on frontend.
#   all      -> full `docker stack deploy` (+ migrate first), which also
#               applies stack-file changes (new services, env, limits).
#
# TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with
# throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel
# rehearsal on the same host that cannot touch live data or ports.
set -euo pipefail

SERVICE="${SERVICE:-all}"

# Two stacks now run on vps2 (prod and beta), so nothing here may assume "the"
# stack: the name, the stack file, the loopback ports, the env file, the
# database role and the service-name prefix all come from env-topology.sh,
# keyed by the environment deploy.sh already resolved and exported.
SELF_DIR=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=infra/deploy/env-topology.sh
. "$SELF_DIR/../env-topology.sh"
ENV_NAME=$(thermograph_env_name)
[ -n "$ENV_NAME" ] || ENV_NAME=prod
thermograph_topology "$ENV_NAME"

APP_DIR="${APP_DIR:-$TG_APP_DIR}"
cd "$APP_DIR"

case "$SERVICE" in
  backend|frontend|all) ;;
  *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
esac

# Which stack-file services this environment's roll targets. Prod's are bare
# (web, worker, ...); beta's carry a `beta-` prefix so its short DNS names never
# collide with prod's on the shared overlay — see env-topology.sh.
SVC_WEB="${TG_SVC_PREFIX}web"
SVC_WORKER="${TG_SVC_PREFIX}worker"
SVC_FRONTEND="${TG_SVC_PREFIX}frontend"
SVC_LAKE="${TG_SVC_PREFIX}lake"
SVC_DAEMON="${TG_SVC_PREFIX}daemon"

STACK_FILE="$APP_DIR/infra/$TG_STACK_FILE"
ENV_FILE="$TG_ENV_FILE"
STACK_ENV_FILE="$TG_STACK_ENV_FILE"

if [ "${STACK_TEST:-0}" = "1" ]; then
  # Test mode rehearses PROD's stack only; it is not a second-environment path.
  STACK_NAME="thermograph-test"
  LB_NAME="thermograph-test-lb"
  LB_HTTP_PORT=18137; LB_FE_PORT=18080
  DATA_NETWORK="thermograph-test_internal"
  DB_SERVICE="thermograph-test_db"
  export PGDATA_VOLUME="thermograph-test_pgdata"
  export APPDATA_VOLUME="thermograph-test_appdata"
  export APPLOGS_VOLUME="thermograph-test_applogs"
  docker volume create "$PGDATA_VOLUME"  >/dev/null
  docker volume create "$APPDATA_VOLUME" >/dev/null
  docker volume create "$APPLOGS_VOLUME" >/dev/null
else
  STACK_NAME="${STACK_NAME:-$TG_STACK_NAME}"
  LB_NAME="$TG_LB_NAME"
  LB_HTTP_PORT="$TG_LB_HTTP_PORT"; LB_FE_PORT="$TG_LB_FE_PORT"
  # Where the database is. Prod owns it inside its own overlay; beta reaches
  # the same service across that overlay, which its stack declares external.
  DATA_NETWORK="$TG_DATA_NETWORK"
  DB_SERVICE="$TG_DB_SERVICE"
fi
export STACK_NAME
echo "==> Environment: $ENV_NAME (stack $STACK_NAME on $TG_HOST, checkout $APP_DIR)"

# --- secrets ------------------------------------------------------------------
# Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then
# install the uid-10001-readable copy the tasks' env-entrypoint shim sources.
# 10001 = the app images' `thermograph` user; the file is 0400 to that uid.
#
# Per-environment paths throughout: on vps2 prod uses /etc/thermograph.env +
# /etc/thermograph/stack.env while beta uses /etc/thermograph-beta.env +
# /etc/thermograph/beta-stack.env. Sharing either file would give beta prod's
# database credentials and vice versa.
if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then
  # shellcheck source=infra/deploy/render-secrets.sh
  . "$APP_DIR/infra/deploy/render-secrets.sh"
  render_thermograph_secrets "$APP_DIR/infra" "$ENV_NAME" "$ENV_FILE"
fi
# $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at
# lint time, so don't ask shellcheck to follow it.
set -a
# shellcheck source=/dev/null
. "$ENV_FILE" 2>/dev/null || true
set +a
sudo install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" \
  || install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE"
# The stack file bind-mounts this path into every task; export it so the
# `volumes:` interpolation resolves to the right environment's copy.
export STACK_ENV_FILE

# --- image tags -----------------------------------------------------------------
# Same persisted-tags contract as deploy.sh: incoming env wins, the file
# supplies the sibling. Stack mode keeps its own file so test/real never mix.
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
export REGISTRY_HOST
TAGS_FILE="$APP_DIR/infra/deploy/.stack-image-tags.env"
_incoming_backend="${BACKEND_IMAGE_TAG:-}"
_incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
# shellcheck source=/dev/null # untracked runtime artifact this script writes below
if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi
[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
case "$SERVICE" in
  backend)  : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;;
  frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;;
  all)
    : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}"
    : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;;
esac
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:$BACKEND_IMAGE_TAG"
FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:$FRONTEND_IMAGE_TAG"

# --- timescale image pin ---------------------------------------------------------
# Hazard #7: the db image under an existing volume must never drift. Resolve
# the digest-pinned ref from whatever is running (stack task or compose
# container), falling back to the local latest-pg18's digest on first bring-up.
#
# Only the environment that OWNS the database needs this — i.e. the one whose
# own stack declares the db service. Beta's stack declares none (it uses prod's
# instance), so resolving an image pin there answers a question beta's stack
# file never asks.
OWNS_DB=0
[ "$DB_SERVICE" = "${STACK_NAME}_db" ] && OWNS_DB=1
if [ -z "${TIMESCALEDB_IMAGE:-}" ] && [ "$OWNS_DB" = 1 ]; then
  cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
  [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1)
  if [ -n "$cid" ]; then
    img=$(docker inspect --format '{{.Image}}' "$cid")
  else
    img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}"
  fi
  TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1)
  [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img"
fi
export TIMESCALEDB_IMAGE
if [ "$OWNS_DB" = 1 ]; then
  echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE"
else
  echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE (db: shared $DB_SERVICE)"
fi

# --- registry ------------------------------------------------------------------
if [ -n "${REGISTRY_TOKEN:-}" ]; then
  echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
fi
echo "==> Pulling images"
pull_ok=0
for i in $(seq 1 30); do
  if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then
    pull_ok=1; break
  fi
  echo "   pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
  sleep 10
done
[ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; }

# --- one-shot migrations ---------------------------------------------------------
# Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0
# in the stack). Runs on the stack's overlay so `db` resolves. First-ever
# deploy: the network doesn't exist yet — create it exactly as the stack will
# (attachable overlay) so the name is adopted, then migrate, then deploy.
#
# The migrate task joins the network the DATABASE is on, which for beta is
# prod's overlay rather than beta's own — and it connects as this environment's
# own role to this environment's own database (beta: thermograph_beta on both
# counts), so a beta migration can never touch prod's schema.
NET="$DATA_NETWORK"
MIGRATE_URL="postgresql+asyncpg://${TG_DB_USER}:${POSTGRES_PASSWORD}@db:5432/${TG_DB_NAME}"
# Never pre-create $NET when this stack owns it: docker stack deploy must own it
# (a pre-existing unlabeled network makes it fail with "already exists"). On
# first deploy the migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE
# below). Beta's data network is prod's and always already exists.
if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then
  if docker service inspect "$DB_SERVICE" >/dev/null 2>&1; then
    echo "==> One-shot migrate ($BACKEND_IMAGE -> ${TG_DB_NAME})"
    docker run --rm --network "$NET" \
      -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \
      --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
  else
    echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below"
  fi
fi

# --- deploy ----------------------------------------------------------------------
FIRST_DEPLOY_MIGRATE=0
docker service inspect "$DB_SERVICE" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1
if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_${SVC_WEB}" >/dev/null 2>&1; then
  echo "==> docker stack deploy ($STACK_NAME)"
  docker stack deploy --with-registry-auth -c "$STACK_FILE" "$STACK_NAME"
  # First-ever deploy ran no migrate above (db didn't exist): wait for db,
  # migrate, then force web/worker to restart cleanly against the schema.
  if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then
    echo "==> Waiting for db, then first-boot migrate"
    for i in $(seq 1 60); do
      cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${DB_SERVICE}" | head -1)
      [ -n "$cid" ] && docker exec "$cid" pg_isready -U "$TG_DB_USER" -d "$TG_DB_NAME" >/dev/null 2>&1 && break
      sleep 5
    done
    docker run --rm --network "$NET" \
      -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \
      --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
    docker service update --force --detach=false "${STACK_NAME}_${SVC_WEB}"
    docker service update --force --detach=false "${STACK_NAME}_${SVC_WORKER}"
  fi
else
  case "$SERVICE" in
    backend)
      echo "==> Rolling $SVC_WEB + $SVC_WORKER + $SVC_LAKE to $BACKEND_IMAGE"
      docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WEB}"
      docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WORKER}"
      # lake and daemon ship in the same image; a stack file predating either
      # has no service yet — the next SERVICE=all stack deploy creates it, so
      # don't fail here. The daemon especially must roll with web: they share
      # the /internal/* contract, and a version skew between them is exactly
      # what pinning one BACKEND_IMAGE_TAG exists to prevent (seen live: the
      # first post-creation backend roll left the daemon a release behind).
      for extra in "$SVC_LAKE" "$SVC_DAEMON"; do
        if docker service inspect "${STACK_NAME}_${extra}" >/dev/null 2>&1; then
          docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${extra}"
        else
          echo "   (no ${STACK_NAME}_${extra} service yet; created on the next full stack deploy)"
        fi
      done
      ;;
    frontend)
      echo "==> Rolling $SVC_FRONTEND to $FRONTEND_IMAGE"
      docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_${SVC_FRONTEND}"
      ;;
  esac
fi

# --- loopback LB bridge -----------------------------------------------------------
# A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes
# 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated
# only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME`
# to force a refresh after editing lb/Caddyfile.
#
# Note the network: the LB proxies to THIS stack's own service VIPs, so it joins
# this stack's own overlay — which for beta is beta's `internal`, not the shared
# data network its tasks also sit on. And its config is per-environment, because
# beta's services are named beta-web/beta-frontend.
LB_NETWORK="${STACK_NAME}_internal"
LB_CONFIG="$APP_DIR/infra/${TG_LB_CONFIG:-deploy/stack/lb/Caddyfile}"
if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then
  docker rm -f "$LB_NAME" >/dev/null 2>&1 || true
  echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)"
  docker run -d --name "$LB_NAME" --restart unless-stopped \
    --network "$LB_NETWORK" \
    -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \
    -v "$LB_CONFIG:/etc/caddy/Caddyfile:ro" \
    caddy:2-alpine >/dev/null
fi

# --- verify -----------------------------------------------------------------------
echo "==> Health check via the LB"
ok=0
for i in $(seq 1 60); do
  if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi
  sleep 2
done
if [ "$ok" != 1 ]; then
  echo "!! stack health check failed" >&2
  docker stack ps "$STACK_NAME" --no-trunc | head -20
  exit 1
fi
echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}"

# Persist now-live tags (after health, like deploy.sh).
cat > "$TAGS_FILE" <<EOF
# Written by deploy-stack.sh -- the image tag each service is currently running.
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
EOF

# GC superseded app-image tags (keep the running pair), same as deploy.sh.
_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}"
_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}"
docker images --format '{{.Repository}}:{{.Tag}}' \
  | grep -E "^(${_be_repo}|${_fe_repo}):" \
  | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
  | xargs -r docker rmi 2>/dev/null || true

# Post-deploy warm + IndexNow, via any web task (skip in test mode: no data,
# and the warmer would burn upstream quota against an empty cache).
#
# TG_POST_DEPLOY gates this per environment: prod does both, beta does neither
# (IndexNow from beta would ask search engines to index beta.thermograph.org,
# and the warm would spend the shared upstream archive quota on a rehearsal
# cache). See env-topology.sh.
if [ "${STACK_TEST:-0}" != "1" ] && [ "${TG_POST_DEPLOY:-1}" = 1 ] \
   && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then
  wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_${SVC_WEB}" | head -1)
  if [ -n "$wcid" ]; then
    echo "==> Warming city archives (detached) + IndexNow"
    docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
    docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \
      || echo "!! IndexNow ping failed (non-fatal)" >&2
  fi
fi
exit 0