Swarm stack for prod: autoscaled web tier (1-3), worker split, loopback LB (#9)

2026-07-23 04:13:12 +00:00 · 2026-07-23 04:13:12 +00:00 · 9cd24387f2
commit 9cd24387f2
parent d02c0f719f
8 changed files with 629 additions and 2 deletions
--- a/.forgejo/workflows/ops-cron.yml
+++ b/.forgejo/workflows/ops-cron.yml
@ -59,9 +59,15 @@ jobs:
            mkdir -p "$backup_dir"
            stamp="$(date -u +%Y%m%dT%H%M%SZ)"
            out="$backup_dir/thermograph-$stamp.dump"
+            # The db may run under plain compose OR as a Swarm stack task
+            # (prod post-cutover); resolve the container either way so the
+            # backup survives the deploy-mode switch.
+            dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1)
+            [ -z "$dbc" ] && dbc=$(cd /opt/thermograph && docker compose ps -q db 2>/dev/null | head -1)
+            [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; }
            # Write to a .partial and rename on success so a mid-dump failure can
            # never leave a truncated file that looks like a good backup.
-            docker compose exec -T db pg_dump -U thermograph -d thermograph \
+            docker exec "$dbc" pg_dump -U thermograph -d thermograph \
              --format=custom > "$out.partial"
            mv "$out.partial" "$out"
            echo "wrote $out ($(du -h "$out" | cut -f1))"
@ -91,5 +97,8 @@ jobs:
            set -euo pipefail
            cd /opt/thermograph
            set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
-            docker compose exec -T backend python indexnow.py --if-changed \
+            bec=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_web" | head -1)
+            [ -z "$bec" ] && bec=$(cd /opt/thermograph && docker compose ps -q backend 2>/dev/null | head -1)
+            [ -n "$bec" ] || { echo "!! no backend/web container found"; exit 1; }
+            docker exec "$bec" python indexnow.py --if-changed \
              "${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
--- a/.gitignore
+++ b/.gitignore
@ -22,3 +22,4 @@ age.key
 # `git clean` can't destroy the record of what's running).
 deploy/.image-tags.env
 deploy/.deploy.lock
+deploy/.stack-image-tags.env
--- a/deploy/deploy.sh
+++ b/deploy/deploy.sh
@ -114,6 +114,15 @@ if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then
  exec "$0" "$@"
 fi

+# Stack-mode routing: a host whose /etc/thermograph/deploy-mode says "stack"
+# (prod, after the Swarm cutover) deploys via the Swarm stack path instead of
+# compose. Checked AFTER the reset+re-exec so the stack script is always the
+# freshly-pulled one, and the SERVICE/tag contract passes through unchanged --
+# the app repos' workflows never need to know which mode a host runs.
+if [ "$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)" = "stack" ]; then
+  exec bash "$APP_DIR/deploy/stack/deploy-stack.sh"
+fi
+
 # Registry-pull cutover: pull the image each app repo's build-push.yml already
 # built and pushed, instead of building in place. This checkout is
 # thermograph-infra, not an app repo, so there's no "current commit" to derive
--- a/deploy/stack/autoscale.sh
+++ b/deploy/stack/autoscale.sh
@ -0,0 +1,93 @@
+#!/bin/sh
+# Autoscaler for the stack's `web` service: scale replicas between
+# MIN_REPLICAS and MAX_REPLICAS on sustained per-task CPU.
+#
+# Runs as a Swarm service on the manager with the docker socket mounted (see
+# thermograph-stack.yml). Every web task is placed on this node today, so
+# node-local `docker stats` sees them all — when a second app node exists,
+# this needs a per-node reader or a metrics-based signal instead; that's the
+# documented upgrade path, not a today problem.
+#
+# Semantics (deliberately boring):
+#   - Sample avg CPU% per web task every POLL_SECONDS (docker stats CPUPerc:
+#     100 = one full host core).
+#   - UP_SAMPLES consecutive samples above SCALE_UP_CPU   -> scale +1.
+#   - DOWN_SAMPLES consecutive samples below SCALE_DOWN_CPU -> scale -1.
+#     (Down is ~7x slower than up on defaults: flap-averse by construction.)
+#   - COOLDOWN_SECONDS after any change: samples are ignored entirely.
+#   - Clamped to [MIN_REPLICAS, MAX_REPLICAS]; scaling waits for convergence
+#     (--detach=false), so a stuck rollout blocks further changes rather than
+#     stacking them.
+set -eu
+
+STACK_NAME="${STACK_NAME:-thermograph}"
+SERVICE="${STACK_NAME}_web"
+MIN="${MIN_REPLICAS:-1}"
+MAX="${MAX_REPLICAS:-3}"
+UP_AT="${SCALE_UP_CPU:-220}"
+DOWN_AT="${SCALE_DOWN_CPU:-60}"
+POLL="${POLL_SECONDS:-15}"
+UP_N="${UP_SAMPLES:-3}"
+DOWN_N="${DOWN_SAMPLES:-20}"
+COOLDOWN="${COOLDOWN_SECONDS:-180}"
+
+up_hits=0
+down_hits=0
+last_change=0
+
+log() { echo "[autoscale] $(date -u +%H:%M:%S) $*"; }
+
+replicas() {
+  docker service inspect "$SERVICE" \
+    --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo ""
+}
+
+avg_cpu() {
+  # Mean CPUPerc across this node's web tasks, as an integer percent.
+  docker stats --no-stream --format '{{.Name}} {{.CPUPerc}}' 2>/dev/null \
+    | awk -v svc="$SERVICE" '
+        index($1, svc".") == 1 {
+          gsub(/%/, "", $2); sum += $2; n++
+        }
+        END { if (n > 0) printf "%d", sum / n; else print "" }'
+}
+
+log "watching $SERVICE: min=$MIN max=$MAX up>@${UP_AT}%x${UP_N} down<@${DOWN_AT}%x${DOWN_N} poll=${POLL}s cooldown=${COOLDOWN}s"
+
+while :; do
+  sleep "$POLL"
+
+  now=$(date +%s)
+  if [ $((now - last_change)) -lt "$COOLDOWN" ]; then
+    continue
+  fi
+
+  cur=$(replicas)
+  [ -n "$cur" ] || { log "service $SERVICE not found; waiting"; continue; }
+  cpu=$(avg_cpu)
+  [ -n "$cpu" ] || continue   # no running tasks visible this sample
+
+  if [ "$cpu" -gt "$UP_AT" ]; then
+    up_hits=$((up_hits + 1)); down_hits=0
+  elif [ "$cpu" -lt "$DOWN_AT" ]; then
+    down_hits=$((down_hits + 1)); up_hits=0
+  else
+    up_hits=0; down_hits=0
+  fi
+
+  if [ "$up_hits" -ge "$UP_N" ] && [ "$cur" -lt "$MAX" ]; then
+    target=$((cur + 1))
+    log "avg cpu ${cpu}% > ${UP_AT}% x${UP_N}: scaling $cur -> $target"
+    if docker service scale --detach=false "$SERVICE=$target"; then
+      last_change=$(date +%s)
+    fi
+    up_hits=0; down_hits=0
+  elif [ "$down_hits" -ge "$DOWN_N" ] && [ "$cur" -gt "$MIN" ]; then
+    target=$((cur - 1))
+    log "avg cpu ${cpu}% < ${DOWN_AT}% x${DOWN_N}: scaling $cur -> $target"
+    if docker service scale --detach=false "$SERVICE=$target"; then
+      last_change=$(date +%s)
+    fi
+    up_hits=0; down_hits=0
+  fi
+done
--- a/deploy/stack/deploy-stack.sh
+++ b/deploy/stack/deploy-stack.sh
@ -0,0 +1,225 @@
+#!/usr/bin/env bash
+# Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh,
+# speaking the SAME contract the app repos' workflows already use
+# (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so
+# switching a host to stack mode needs no workflow changes: deploy.sh execs
+# this when /etc/thermograph/deploy-mode contains "stack".
+#
+# What a roll does here vs compose:
+#   backend  -> one-shot migrate, then `docker service update --image` on
+#               web AND worker (same image; start-first, health-gated,
+#               auto-rollback on failure).
+#   frontend -> `docker service update --image` on frontend.
+#   all      -> full `docker stack deploy` (+ migrate first), which also
+#               applies stack-file changes (new services, env, limits).
+#
+# TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with
+# throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel
+# rehearsal on the same host that cannot touch live data or ports.
+set -euo pipefail
+
+APP_DIR="${APP_DIR:-/opt/thermograph}"
+SERVICE="${SERVICE:-all}"
+cd "$APP_DIR"
+
+case "$SERVICE" in
+  backend|frontend|all) ;;
+  *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
+esac
+
+if [ "${STACK_TEST:-0}" = "1" ]; then
+  STACK_NAME="thermograph-test"
+  LB_NAME="thermograph-test-lb"
+  LB_HTTP_PORT=18137; LB_FE_PORT=18080
+  export PGDATA_VOLUME="thermograph-test_pgdata"
+  export APPDATA_VOLUME="thermograph-test_appdata"
+  export APPLOGS_VOLUME="thermograph-test_applogs"
+  docker volume create "$PGDATA_VOLUME"  >/dev/null
+  docker volume create "$APPDATA_VOLUME" >/dev/null
+  docker volume create "$APPLOGS_VOLUME" >/dev/null
+else
+  STACK_NAME="${STACK_NAME:-thermograph}"
+  LB_NAME="thermograph-lb"
+  LB_HTTP_PORT=8137; LB_FE_PORT=8080
+fi
+export STACK_NAME
+
+# --- secrets ------------------------------------------------------------------
+# Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then
+# install the uid-10001-readable copy the tasks' env-entrypoint shim sources.
+# 10001 = the app images' `thermograph` user; the file is 0400 to that uid.
+if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then
+  # shellcheck source=deploy/render-secrets.sh
+  . "$APP_DIR/deploy/render-secrets.sh"
+  render_thermograph_secrets "$APP_DIR"
+fi
+set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
+sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \
+  || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env
+
+# --- image tags -----------------------------------------------------------------
+# Same persisted-tags contract as deploy.sh: incoming env wins, the file
+# supplies the sibling. Stack mode keeps its own file so test/real never mix.
+REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
+export REGISTRY_HOST
+TAGS_FILE="$APP_DIR/deploy/.stack-image-tags.env"
+_incoming_backend="${BACKEND_IMAGE_TAG:-}"
+_incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
+if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi
+[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
+[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
+case "$SERVICE" in
+  backend)  : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;;
+  frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;;
+  all)
+    : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}"
+    : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;;
+esac
+export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
+export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
+BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:$BACKEND_IMAGE_TAG"
+FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}:$FRONTEND_IMAGE_TAG"
+
+# --- timescale image pin ---------------------------------------------------------
+# Hazard #7: the db image under an existing volume must never drift. Resolve
+# the digest-pinned ref from whatever is running (stack task or compose
+# container), falling back to the local latest-pg18's digest on first bring-up.
+if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then
+  cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
+  [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1)
+  if [ -n "$cid" ]; then
+    img=$(docker inspect --format '{{.Image}}' "$cid")
+  else
+    img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}"
+  fi
+  TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1)
+  [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img"
+fi
+export TIMESCALEDB_IMAGE
+echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE"
+
+# --- registry ------------------------------------------------------------------
+if [ -n "${REGISTRY_TOKEN:-}" ]; then
+  echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
+fi
+echo "==> Pulling images"
+pull_ok=0
+for i in $(seq 1 30); do
+  if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then
+    pull_ok=1; break
+  fi
+  echo "   pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
+  sleep 10
+done
+[ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; }
+
+# --- one-shot migrations ---------------------------------------------------------
+# Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0
+# in the stack). Runs on the stack's overlay so `db` resolves. First-ever
+# deploy: the network doesn't exist yet — create it exactly as the stack will
+# (attachable overlay) so the name is adopted, then migrate, then deploy.
+NET="${STACK_NAME}_internal"
+docker network inspect "$NET" >/dev/null 2>&1 \
+  || docker network create --driver overlay --attachable --scope swarm "$NET" >/dev/null
+if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then
+  if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then
+    echo "==> One-shot migrate ($BACKEND_IMAGE)"
+    docker run --rm --network "$NET" \
+      -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \
+      --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
+  else
+    echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below"
+  fi
+fi
+
+# --- deploy ----------------------------------------------------------------------
+FIRST_DEPLOY_MIGRATE=0
+docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1
+if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then
+  echo "==> docker stack deploy ($STACK_NAME)"
+  docker stack deploy --with-registry-auth -c "$APP_DIR/deploy/stack/thermograph-stack.yml" "$STACK_NAME"
+  # First-ever deploy ran no migrate above (db didn't exist): wait for db,
+  # migrate, then force web/worker to restart cleanly against the schema.
+  if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then
+    echo "==> Waiting for db, then first-boot migrate"
+    for i in $(seq 1 60); do
+      cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
+      [ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break
+      sleep 5
+    done
+    docker run --rm --network "$NET" \
+      -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \
+      --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
+    docker service update --force --detach=false "${STACK_NAME}_web"
+    docker service update --force --detach=false "${STACK_NAME}_worker"
+  fi
+else
+  case "$SERVICE" in
+    backend)
+      echo "==> Rolling web + worker to $BACKEND_IMAGE"
+      docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web"
+      docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker"
+      ;;
+    frontend)
+      echo "==> Rolling frontend to $FRONTEND_IMAGE"
+      docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend"
+      ;;
+  esac
+fi
+
+# --- loopback LB bridge -----------------------------------------------------------
+# A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes
+# 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated
+# only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME`
+# to force a refresh after editing lb/Caddyfile.
+if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then
+  docker rm -f "$LB_NAME" >/dev/null 2>&1 || true
+  echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)"
+  docker run -d --name "$LB_NAME" --restart unless-stopped \
+    --network "$NET" \
+    -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \
+    -v "$APP_DIR/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \
+    caddy:2-alpine >/dev/null
+fi
+
+# --- verify -----------------------------------------------------------------------
+echo "==> Health check via the LB"
+ok=0
+for i in $(seq 1 60); do
+  if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi
+  sleep 2
+done
+if [ "$ok" != 1 ]; then
+  echo "!! stack health check failed" >&2
+  docker stack ps "$STACK_NAME" --no-trunc | head -20
+  exit 1
+fi
+echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}"
+
+# Persist now-live tags (after health, like deploy.sh).
+cat > "$TAGS_FILE" <<EOF
+# Written by deploy-stack.sh -- the image tag each service is currently running.
+BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
+FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
+EOF
+
+# GC superseded app-image tags (keep the running pair), same as deploy.sh.
+_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}"
+_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}"
+docker images --format '{{.Repository}}:{{.Tag}}' \
+  | grep -E "^(${_be_repo}|${_fe_repo}):" \
+  | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
+  | xargs -r docker rmi 2>/dev/null || true
+
+# Post-deploy warm + IndexNow, via any web task (skip in test mode: no data,
+# and the warmer would burn upstream quota against an empty cache).
+if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then
+  wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1)
+  if [ -n "$wcid" ]; then
+    echo "==> Warming city archives (detached) + IndexNow"
+    docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
+    docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \
+      || echo "!! IndexNow ping failed (non-fatal)" >&2
+  fi
+fi
+exit 0
--- a/deploy/stack/env-entrypoint.sh
+++ b/deploy/stack/env-entrypoint.sh
@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Stack-task entrypoint shim: source the host-rendered secrets env, then hand
+# off to the image's real entrypoint.
+#
+# Why: `docker stack deploy` does not support compose's `env_file:`, and
+# enumerating every vault key in the stack yml's `environment:` blocks would
+# drift the moment a key is added to deploy/secrets/. Instead deploy-stack.sh
+# installs a uid-10001-readable copy of the rendered env at
+# /etc/thermograph/stack.env, the stack bind-mounts it (with this script) into
+# every app task, and this shim exports each KEY=value — but ONLY for keys not
+# already set, so the yml's `environment:` blocks keep compose's env_file
+# precedence (environment always wins). Same set-if-unset contract as the
+# image's own /run/secrets shim in deploy/entrypoint.sh, which still runs
+# after this and stays a no-op here.
+set -euo pipefail
+
+ENV_FILE="${THERMOGRAPH_HOST_ENV:-/host/thermograph.env}"
+
+if [ -f "$ENV_FILE" ]; then
+  while IFS= read -r line || [ -n "$line" ]; do
+    case "$line" in
+      ''|'#'*) continue ;;
+      *=*)
+        key="${line%%=*}"
+        # Only sane identifiers; only if not already set by `environment:`.
+        case "$key" in
+          *[!A-Za-z0-9_]*|'') continue ;;
+        esac
+        if [ -z "${!key:-}" ]; then
+          export "$key=${line#*=}"
+        fi
+        ;;
+    esac
+  done < "$ENV_FILE"
+fi
+
+exec /app/deploy/entrypoint.sh "$@"
--- a/deploy/stack/lb/Caddyfile
+++ b/deploy/stack/lb/Caddyfile
@ -0,0 +1,30 @@
+# The loopback LB bridge's own config — NOT the host Caddy (that one still
+# terminates TLS for thermograph.org and proxies to 127.0.0.1:8137 exactly as
+# before; it needs no change for the stack cutover).
+#
+# This Caddy runs as a PLAIN container (deploy-stack.sh manages it) because
+# only plain containers can bind a specific host IP: Swarm port configs
+# publish on 0.0.0.0 (routing mesh or host mode alike), which would expose
+# the plaintext app un-fronted — hazard #6. It joins the stack's attachable
+# overlay and proxies to the service VIPs; Swarm's VIP round-robins across
+# however many web replicas the autoscaler is running, so this bridge never
+# needs to know the replica count.
+
+{
+	auto_https off
+	admin off
+}
+
+:8137 {
+	reverse_proxy web:8137 {
+		# Fail fast to the client if the VIP has no healthy task; Swarm's own
+		# task healthchecks handle ejecting dead replicas from the VIP.
+		lb_try_duration 5s
+	}
+}
+
+:8080 {
+	reverse_proxy frontend:8080 {
+		lb_try_duration 5s
+	}
+}
--- a/deploy/stack/thermograph-stack.yml
+++ b/deploy/stack/thermograph-stack.yml
@ -0,0 +1,223 @@
+# Docker Swarm stack for prod — the autoscaling successor to docker-compose.yml
+# on that host (beta and LAN dev stay on plain compose; deploy.sh routes by the
+# /etc/thermograph/deploy-mode marker). Two-image world: web/worker run the
+# backend image, frontend its own — per-service tags, same contract as compose.
+#
+# Deployed by deploy/stack/deploy-stack.sh, which:
+#   - sources /etc/thermograph.env (SOPS-rendered) so ${VARS} here interpolate,
+#   - installs a uid-10001-readable copy at /etc/thermograph/stack.env that
+#     deploy/stack/env-entrypoint.sh sources inside each app task (set-if-unset,
+#     so `environment:` blocks below always win) — this replaces compose's
+#     env_file:, which `docker stack deploy` does not support,
+#   - runs migrations as a ONE-SHOT task before rolling (RUN_MIGRATIONS=0 in
+#     every replica — N replicas must never race Alembic),
+#   - manages the loopback LB bridge (see lb/README note below): Swarm's mesh
+#     can only publish on 0.0.0.0 (would expose the plaintext app un-fronted),
+#     so nothing here has `ports:`. A plain container on this attachable
+#     overlay binds 127.0.0.1:8137/8080 for the host Caddy and proxies to the
+#     service VIPs — Swarm's VIP does the actual load balancing across
+#     replicas.
+#
+# Scaling model: `web` is stateless (ROLE=web never runs the notifier) and
+# scales 1..N — the autoscaler service adjusts replicas between
+# WEB_MIN_REPLICAS/WEB_MAX_REPLICAS on task CPU. `worker` owns the notifier +
+# scheduler: exactly 1 replica, with the cluster-wide Postgres advisory lock
+# (THERMOGRAPH_SINGLETON_PG) as belt-and-suspenders. `db` is exactly 1 — a
+# database does not scale by container count on one host; its levers are
+# DB_CPUS/DB_MEMORY (and, multi-host later, Patroni replicas per the topology
+# doc). Everything is pinned to the manager node: all volumes are local to
+# prod today. That constraint is the ONLY thing to relax when a second app
+# node joins.
+#
+# TIMESCALEDB_IMAGE must be the exact image (digest-pinned) the compose stack
+# was running — see hazard #7 in the hop-1 runbook: a floating tag can change
+# the extension minor under an existing volume. deploy-stack.sh resolves it
+# from the running/last-known container automatically.
+
+services:
+  db:
+    image: ${TIMESCALEDB_IMAGE:?set TIMESCALEDB_IMAGE to the exact (digest-pinned) image the volume was created with}
+    environment:
+      POSTGRES_USER: thermograph
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
+      POSTGRES_DB: thermograph
+      DB_MEMORY: ${DB_MEMORY:-16g}
+    volumes:
+      - pgdata:/var/lib/postgresql
+      - /opt/thermograph/deploy/db/init:/docker-entrypoint-initdb.d:ro
+    networks:
+      - internal
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U thermograph -d thermograph"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    deploy:
+      replicas: 1
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "${DB_CPUS:-4}"
+          memory: ${DB_MEMORY:-16g}
+      restart_policy:
+        condition: on-failure
+
+  web:
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG}
+    entrypoint: ["/host/env-entrypoint.sh"]
+    environment:
+      THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph
+      THERMOGRAPH_BASE: /
+      PORT: 8137
+      THERMOGRAPH_SERVICE_ROLE: backend
+      THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://frontend:8080
+      WORKERS: ${WEB_WORKERS:-4}
+      THERMOGRAPH_DATA_DIR: /state
+      # web NEVER runs the notifier/scheduler, even if it would win election —
+      # that's the worker service's job. This is what makes web replicas safe.
+      THERMOGRAPH_ROLE: web
+      RUN_MIGRATIONS: "0"
+      # Overlay tasks reach the HOST's Postfix via the docker_gwbridge gateway,
+      # not the compose bridge's 172.19.0.1 (an overlay has no host gateway).
+      # provision-mail.sh's DOCKER_MAIL_GATEWAY/SUBNET cover this listener.
+      THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1}
+    volumes:
+      - appdata:/state
+      - applogs:/app/logs
+      - /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
+      - /etc/thermograph/stack.env:/host/thermograph.env:ro
+    networks:
+      - internal
+    deploy:
+      replicas: ${WEB_MIN_REPLICAS:-1}
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "${WEB_CPUS:-4}"
+      restart_policy:
+        condition: on-failure
+      update_config:
+        # New task must pass the image's own HEALTHCHECK before the old one
+        # stops — zero-downtime single-service rolls.
+        order: start-first
+        failure_action: rollback
+
+  worker:
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG}
+    entrypoint: ["/host/env-entrypoint.sh"]
+    environment:
+      THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph
+      THERMOGRAPH_BASE: /
+      PORT: 8137
+      THERMOGRAPH_SERVICE_ROLE: backend
+      THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://frontend:8080
+      WORKERS: "1"
+      THERMOGRAPH_DATA_DIR: /state
+      THERMOGRAPH_ROLE: worker
+      # Cluster-wide Postgres advisory lock, not the host flock: correct at
+      # replicas=1 today and stays correct if a second worker ever appears.
+      THERMOGRAPH_SINGLETON_PG: "1"
+      RUN_MIGRATIONS: "0"
+      THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1}
+    volumes:
+      - appdata:/state
+      - applogs:/app/logs
+      - /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
+      - /etc/thermograph/stack.env:/host/thermograph.env:ro
+    networks:
+      - internal
+    deploy:
+      replicas: 1
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "${WORKER_CPUS:-2}"
+      restart_policy:
+        condition: on-failure
+
+  frontend:
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}:${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG}
+    entrypoint: ["/host/env-entrypoint.sh"]
+    environment:
+      THERMOGRAPH_BASE: /
+      PORT: 8080
+      THERMOGRAPH_SERVICE_ROLE: frontend
+      THERMOGRAPH_API_BASE_INTERNAL: http://web:8137
+    volumes:
+      - /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
+      - /etc/thermograph/stack.env:/host/thermograph.env:ro
+    networks:
+      - internal
+    deploy:
+      replicas: 1
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "${FRONTEND_CPUS:-2}"
+      restart_policy:
+        condition: on-failure
+      update_config:
+        order: start-first
+        failure_action: rollback
+
+  # Scales `web` between WEB_MIN_REPLICAS and WEB_MAX_REPLICAS on sustained
+  # task CPU (docker stats on this node — every task is pinned here today).
+  # Deliberately a dumb shell loop with hysteresis + cooldown, not an
+  # autoscaling framework: Swarm has no native autoscaler and this workload
+  # doesn't justify one (topology doc §6 — declarative scaling as a feature).
+  autoscaler:
+    image: docker:27-cli
+    entrypoint: ["/bin/sh", "/host/autoscale.sh"]
+    environment:
+      STACK_NAME: ${STACK_NAME:-thermograph}
+      MIN_REPLICAS: ${WEB_MIN_REPLICAS:-1}
+      MAX_REPLICAS: ${WEB_MAX_REPLICAS:-3}
+      # Thresholds are avg docker-stats CPU% PER TASK (host-core-relative: 100
+      # = one full core). Up fast, down slow.
+      SCALE_UP_CPU: ${SCALE_UP_CPU:-220}
+      SCALE_DOWN_CPU: ${SCALE_DOWN_CPU:-60}
+      POLL_SECONDS: ${POLL_SECONDS:-15}
+      UP_SAMPLES: ${UP_SAMPLES:-3}
+      DOWN_SAMPLES: ${DOWN_SAMPLES:-20}
+      COOLDOWN_SECONDS: ${COOLDOWN_SECONDS:-180}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /opt/thermograph/deploy/stack/autoscale.sh:/host/autoscale.sh:ro
+    networks:
+      - internal
+    deploy:
+      replicas: 1
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "0.2"
+          memory: 64m
+      restart_policy:
+        condition: any
+
+networks:
+  internal:
+    driver: overlay
+    # Attachable so the loopback LB bridge (a PLAIN container — only plain
+    # containers can bind 127.0.0.1; Swarm port configs cannot) can join and
+    # reach the service VIPs.
+    attachable: true
+
+volumes:
+  # External and explicitly named: the real stack REUSES the volumes the
+  # compose stack created (same data, zero migration). deploy-stack.sh's test
+  # mode points these at throwaway names instead.
+  pgdata:
+    external: true
+    name: ${PGDATA_VOLUME:-thermograph_pgdata}
+  appdata:
+    external: true
+    name: ${APPDATA_VOLUME:-thermograph_appdata}
+  applogs:
+    external: true
+    name: ${APPLOGS_VOLUME:-thermograph_applogs}