#!/usr/bin/env bash # Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh, # speaking the SAME contract the app repos' workflows already use # (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so # switching a host to stack mode needs no workflow changes: deploy.sh execs # this when /etc/thermograph/deploy-mode contains "stack". # # What a roll does here vs compose: # backend -> one-shot migrate, then `docker service update --image` on # web AND worker (same image; start-first, health-gated, # auto-rollback on failure). # frontend -> `docker service update --image` on frontend. # all -> full `docker stack deploy` (+ migrate first), which also # applies stack-file changes (new services, env, limits). # # TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with # throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel # rehearsal on the same host that cannot touch live data or ports. set -euo pipefail APP_DIR="${APP_DIR:-/opt/thermograph}" SERVICE="${SERVICE:-all}" cd "$APP_DIR" case "$SERVICE" in backend|frontend|all) ;; *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac if [ "${STACK_TEST:-0}" = "1" ]; then STACK_NAME="thermograph-test" LB_NAME="thermograph-test-lb" LB_HTTP_PORT=18137; LB_FE_PORT=18080 export PGDATA_VOLUME="thermograph-test_pgdata" export APPDATA_VOLUME="thermograph-test_appdata" export APPLOGS_VOLUME="thermograph-test_applogs" docker volume create "$PGDATA_VOLUME" >/dev/null docker volume create "$APPDATA_VOLUME" >/dev/null docker volume create "$APPLOGS_VOLUME" >/dev/null else STACK_NAME="${STACK_NAME:-thermograph}" LB_NAME="thermograph-lb" LB_HTTP_PORT=8137; LB_FE_PORT=8080 fi export STACK_NAME # --- secrets ------------------------------------------------------------------ # Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then # install the uid-10001-readable copy the tasks' env-entrypoint shim sources. # 10001 = the app images' `thermograph` user; the file is 0400 to that uid. if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$APP_DIR/infra/deploy/render-secrets.sh" render_thermograph_secrets "$APP_DIR/infra" fi # /etc/thermograph.env is rendered at deploy time from the SOPS vault — it # cannot exist at lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null . /etc/thermograph.env 2>/dev/null || true set +a sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \ || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env # --- image tags ----------------------------------------------------------------- # Same persisted-tags contract as deploy.sh: incoming env wins, the file # supplies the sibling. Stack mode keeps its own file so test/real never mix. REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}" export REGISTRY_HOST TAGS_FILE="$APP_DIR/infra/deploy/.stack-image-tags.env" _incoming_backend="${BACKEND_IMAGE_TAG:-}" _incoming_frontend="${FRONTEND_IMAGE_TAG:-}" # shellcheck source=/dev/null # untracked runtime artifact this script writes below if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend" [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend" case "$SERVICE" in backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;; frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;; all) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}" : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;; esac export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:$BACKEND_IMAGE_TAG" FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:$FRONTEND_IMAGE_TAG" # --- timescale image pin --------------------------------------------------------- # Hazard #7: the db image under an existing volume must never drift. Resolve # the digest-pinned ref from whatever is running (stack task or compose # container), falling back to the local latest-pg18's digest on first bring-up. if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1) if [ -n "$cid" ]; then img=$(docker inspect --format '{{.Image}}' "$cid") else img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}" fi TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1) [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img" fi export TIMESCALEDB_IMAGE echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" # --- registry ------------------------------------------------------------------ if [ -n "${REGISTRY_TOKEN:-}" ]; then echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin fi echo "==> Pulling images" pull_ok=0 for i in $(seq 1 30); do if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then pull_ok=1; break fi echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2 sleep 10 done [ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; } # --- one-shot migrations --------------------------------------------------------- # Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0 # in the stack). Runs on the stack's overlay so `db` resolves. First-ever # deploy: the network doesn't exist yet — create it exactly as the stack will # (attachable overlay) so the name is adopted, then migrate, then deploy. NET="${STACK_NAME}_internal" # Never pre-create $NET: docker stack deploy must own it (a pre-existing # unlabeled network makes it fail with "already exists"). On first deploy the # migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE below). if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then echo "==> One-shot migrate ($BACKEND_IMAGE)" docker run --rm --network "$NET" \ -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate else echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below" fi fi # --- deploy ---------------------------------------------------------------------- FIRST_DEPLOY_MIGRATE=0 docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then echo "==> docker stack deploy ($STACK_NAME)" docker stack deploy --with-registry-auth -c "$APP_DIR/infra/deploy/stack/thermograph-stack.yml" "$STACK_NAME" # First-ever deploy ran no migrate above (db didn't exist): wait for db, # migrate, then force web/worker to restart cleanly against the schema. if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then echo "==> Waiting for db, then first-boot migrate" for i in $(seq 1 60); do cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) [ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break sleep 5 done docker run --rm --network "$NET" \ -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate docker service update --force --detach=false "${STACK_NAME}_web" docker service update --force --detach=false "${STACK_NAME}_worker" fi else case "$SERVICE" in backend) echo "==> Rolling web + worker to $BACKEND_IMAGE" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker" ;; frontend) echo "==> Rolling frontend to $FRONTEND_IMAGE" docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend" ;; esac fi # --- loopback LB bridge ----------------------------------------------------------- # A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes # 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated # only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME` # to force a refresh after editing lb/Caddyfile. if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then docker rm -f "$LB_NAME" >/dev/null 2>&1 || true echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)" docker run -d --name "$LB_NAME" --restart unless-stopped \ --network "$NET" \ -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \ -v "$APP_DIR/infra/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \ caddy:2-alpine >/dev/null fi # --- verify ----------------------------------------------------------------------- echo "==> Health check via the LB" ok=0 for i in $(seq 1 60); do if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi sleep 2 done if [ "$ok" != 1 ]; then echo "!! stack health check failed" >&2 docker stack ps "$STACK_NAME" --no-trunc | head -20 exit 1 fi echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}" # Persist now-live tags (after health, like deploy.sh). cat > "$TAGS_FILE" </dev/null || true # Post-deploy warm + IndexNow, via any web task (skip in test mode: no data, # and the warmer would burn upstream quota against an empty cache). if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1) if [ -n "$wcid" ]; then echo "==> Warming city archives (detached) + IndexNow" docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \ || echo "!! IndexNow ping failed (non-fatal)" >&2 fi fi exit 0