#!/usr/bin/env bash # Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh, # speaking the SAME contract the app repos' workflows already use # (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so # switching a host to stack mode needs no workflow changes: deploy.sh execs # this when /etc/thermograph/deploy-mode contains "stack". # # What a roll does here vs compose: # backend -> one-shot migrate, then `docker service update --image` on # web AND worker (same image; start-first, health-gated, # auto-rollback on failure). # frontend -> `docker service update --image` on frontend. # all -> full `docker stack deploy` (+ migrate first), which also # applies stack-file changes (new services, env, limits). # # TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with # throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel # rehearsal on the same host that cannot touch live data or ports. set -euo pipefail SERVICE="${SERVICE:-all}" # Two stacks now run on vps2 (prod and beta), so nothing here may assume "the" # stack: the name, the stack file, the loopback ports, the env file, the # database role and the service-name prefix all come from env-topology.sh, # keyed by the environment deploy.sh already resolved and exported. SELF_DIR=$(cd "$(dirname "$0")" && pwd) # shellcheck source=infra/deploy/env-topology.sh . "$SELF_DIR/../env-topology.sh" ENV_NAME=$(thermograph_env_name) [ -n "$ENV_NAME" ] || ENV_NAME=prod thermograph_topology "$ENV_NAME" APP_DIR="${APP_DIR:-$TG_APP_DIR}" cd "$APP_DIR" case "$SERVICE" in backend|frontend|all) ;; *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac # Which stack-file services this environment's roll targets. Prod's are bare # (web, worker, ...); beta's carry a `beta-` prefix so its short DNS names never # collide with prod's on the shared overlay — see env-topology.sh. SVC_WEB="${TG_SVC_PREFIX}web" SVC_WORKER="${TG_SVC_PREFIX}worker" SVC_FRONTEND="${TG_SVC_PREFIX}frontend" SVC_LAKE="${TG_SVC_PREFIX}lake" SVC_DAEMON="${TG_SVC_PREFIX}daemon" STACK_FILE="$APP_DIR/infra/$TG_STACK_FILE" ENV_FILE="$TG_ENV_FILE" STACK_ENV_FILE="$TG_STACK_ENV_FILE" if [ "${STACK_TEST:-0}" = "1" ]; then # Test mode rehearses PROD's stack only; it is not a second-environment path. STACK_NAME="thermograph-test" LB_NAME="thermograph-test-lb" LB_HTTP_PORT=18137; LB_FE_PORT=18080 DATA_NETWORK="thermograph-test_internal" DB_SERVICE="thermograph-test_db" export PGDATA_VOLUME="thermograph-test_pgdata" export APPDATA_VOLUME="thermograph-test_appdata" export APPLOGS_VOLUME="thermograph-test_applogs" docker volume create "$PGDATA_VOLUME" >/dev/null docker volume create "$APPDATA_VOLUME" >/dev/null docker volume create "$APPLOGS_VOLUME" >/dev/null else STACK_NAME="${STACK_NAME:-$TG_STACK_NAME}" LB_NAME="$TG_LB_NAME" LB_HTTP_PORT="$TG_LB_HTTP_PORT"; LB_FE_PORT="$TG_LB_FE_PORT" # Where the database is. Prod owns it inside its own overlay; beta reaches # the same service across that overlay, which its stack declares external. DATA_NETWORK="$TG_DATA_NETWORK" DB_SERVICE="$TG_DB_SERVICE" fi export STACK_NAME echo "==> Environment: $ENV_NAME (stack $STACK_NAME on $TG_HOST, checkout $APP_DIR)" # --- secrets ------------------------------------------------------------------ # Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then # install the uid-10001-readable copy the tasks' env-entrypoint shim sources. # 10001 = the app images' `thermograph` user; the file is 0400 to that uid. # # Per-environment paths throughout: on vps2 prod uses /etc/thermograph.env + # /etc/thermograph/stack.env while beta uses /etc/thermograph-beta.env + # /etc/thermograph/beta-stack.env. Sharing either file would give beta prod's # database credentials and vice versa. if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$APP_DIR/infra/deploy/render-secrets.sh" render_thermograph_secrets "$APP_DIR/infra" "$ENV_NAME" "$ENV_FILE" fi # $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at # lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null . "$ENV_FILE" 2>/dev/null || true set +a sudo install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" \ || install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" # The stack file bind-mounts this path into every task; export it so the # `volumes:` interpolation resolves to the right environment's copy. export STACK_ENV_FILE # --- image tags ----------------------------------------------------------------- # Same persisted-tags contract as deploy.sh: incoming env wins, the file # supplies the sibling. Stack mode keeps its own file so test/real never mix. REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}" export REGISTRY_HOST TAGS_FILE="$APP_DIR/infra/deploy/.stack-image-tags.env" _incoming_backend="${BACKEND_IMAGE_TAG:-}" _incoming_frontend="${FRONTEND_IMAGE_TAG:-}" # shellcheck source=/dev/null # untracked runtime artifact this script writes below if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend" [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend" case "$SERVICE" in backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;; frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;; all) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}" : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;; esac export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:$BACKEND_IMAGE_TAG" FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:$FRONTEND_IMAGE_TAG" # --- timescale image pin --------------------------------------------------------- # Hazard #7: the db image under an existing volume must never drift. Resolve # the digest-pinned ref from whatever is running (stack task or compose # container), falling back to the local latest-pg18's digest on first bring-up. # # Only the environment that OWNS the database needs this — i.e. the one whose # own stack declares the db service. Beta's stack declares none (it uses prod's # instance), so resolving an image pin there answers a question beta's stack # file never asks. OWNS_DB=0 [ "$DB_SERVICE" = "${STACK_NAME}_db" ] && OWNS_DB=1 if [ -z "${TIMESCALEDB_IMAGE:-}" ] && [ "$OWNS_DB" = 1 ]; then cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1) if [ -n "$cid" ]; then img=$(docker inspect --format '{{.Image}}' "$cid") else img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}" fi TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1) [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img" fi export TIMESCALEDB_IMAGE if [ "$OWNS_DB" = 1 ]; then echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" else echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE (db: shared $DB_SERVICE)" fi # --- registry ------------------------------------------------------------------ if [ -n "${REGISTRY_TOKEN:-}" ]; then echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin fi echo "==> Pulling images" pull_ok=0 for i in $(seq 1 30); do if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then pull_ok=1; break fi echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2 sleep 10 done [ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; } # --- one-shot migrations --------------------------------------------------------- # Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0 # in the stack). Runs on the stack's overlay so `db` resolves. First-ever # deploy: the network doesn't exist yet — create it exactly as the stack will # (attachable overlay) so the name is adopted, then migrate, then deploy. # # The migrate task joins the network the DATABASE is on, which for beta is # prod's overlay rather than beta's own — and it connects as this environment's # own role to this environment's own database (beta: thermograph_beta on both # counts), so a beta migration can never touch prod's schema. NET="$DATA_NETWORK" MIGRATE_URL="postgresql+asyncpg://${TG_DB_USER}:${POSTGRES_PASSWORD}@db:5432/${TG_DB_NAME}" # Never pre-create $NET when this stack owns it: docker stack deploy must own it # (a pre-existing unlabeled network makes it fail with "already exists"). On # first deploy the migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE # below). Beta's data network is prod's and always already exists. if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then if docker service inspect "$DB_SERVICE" >/dev/null 2>&1; then echo "==> One-shot migrate ($BACKEND_IMAGE -> ${TG_DB_NAME})" docker run --rm --network "$NET" \ -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate else echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below" fi fi # --- deploy ---------------------------------------------------------------------- FIRST_DEPLOY_MIGRATE=0 docker service inspect "$DB_SERVICE" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_${SVC_WEB}" >/dev/null 2>&1; then echo "==> docker stack deploy ($STACK_NAME)" docker stack deploy --with-registry-auth -c "$STACK_FILE" "$STACK_NAME" # First-ever deploy ran no migrate above (db didn't exist): wait for db, # migrate, then force web/worker to restart cleanly against the schema. if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then echo "==> Waiting for db, then first-boot migrate" for i in $(seq 1 60); do cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${DB_SERVICE}" | head -1) [ -n "$cid" ] && docker exec "$cid" pg_isready -U "$TG_DB_USER" -d "$TG_DB_NAME" >/dev/null 2>&1 && break sleep 5 done docker run --rm --network "$NET" \ -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate docker service update --force --detach=false "${STACK_NAME}_${SVC_WEB}" docker service update --force --detach=false "${STACK_NAME}_${SVC_WORKER}" fi else case "$SERVICE" in backend) echo "==> Rolling $SVC_WEB + $SVC_WORKER + $SVC_LAKE to $BACKEND_IMAGE" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WEB}" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WORKER}" # lake and daemon ship in the same image; a stack file predating either # has no service yet — the next SERVICE=all stack deploy creates it, so # don't fail here. The daemon especially must roll with web: they share # the /internal/* contract, and a version skew between them is exactly # what pinning one BACKEND_IMAGE_TAG exists to prevent (seen live: the # first post-creation backend roll left the daemon a release behind). for extra in "$SVC_LAKE" "$SVC_DAEMON"; do if docker service inspect "${STACK_NAME}_${extra}" >/dev/null 2>&1; then docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${extra}" else echo " (no ${STACK_NAME}_${extra} service yet; created on the next full stack deploy)" fi done ;; frontend) echo "==> Rolling $SVC_FRONTEND to $FRONTEND_IMAGE" docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_${SVC_FRONTEND}" ;; esac fi # --- loopback LB bridge ----------------------------------------------------------- # A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes # 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated # only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME` # to force a refresh after editing lb/Caddyfile. # # Note the network: the LB proxies to THIS stack's own service VIPs, so it joins # this stack's own overlay — which for beta is beta's `internal`, not the shared # data network its tasks also sit on. And its config is per-environment, because # beta's services are named beta-web/beta-frontend. LB_NETWORK="${STACK_NAME}_internal" LB_CONFIG="$APP_DIR/infra/${TG_LB_CONFIG:-deploy/stack/lb/Caddyfile}" if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then docker rm -f "$LB_NAME" >/dev/null 2>&1 || true echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)" docker run -d --name "$LB_NAME" --restart unless-stopped \ --network "$LB_NETWORK" \ -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \ -v "$LB_CONFIG:/etc/caddy/Caddyfile:ro" \ caddy:2-alpine >/dev/null fi # --- verify ----------------------------------------------------------------------- echo "==> Health check via the LB" ok=0 for i in $(seq 1 60); do if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi sleep 2 done if [ "$ok" != 1 ]; then echo "!! stack health check failed" >&2 docker stack ps "$STACK_NAME" --no-trunc | head -20 exit 1 fi echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}" # Persist now-live tags (after health, like deploy.sh). cat > "$TAGS_FILE" </dev/null || true # Post-deploy warm + IndexNow, via any web task (skip in test mode: no data, # and the warmer would burn upstream quota against an empty cache). # # TG_POST_DEPLOY gates this per environment: prod does both, beta does neither # (IndexNow from beta would ask search engines to index beta.thermograph.org, # and the warm would spend the shared upstream archive quota on a rehearsal # cache). See env-topology.sh. if [ "${STACK_TEST:-0}" != "1" ] && [ "${TG_POST_DEPLOY:-1}" = 1 ] \ && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_${SVC_WEB}" | head -1) if [ -n "$wcid" ]; then echo "==> Warming city archives (detached) + IndexNow" docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \ || echo "!! IndexNow ping failed (non-fatal)" >&2 fi fi exit 0