From 9cd24387f288eca5767230be81d13f7bd992b90d Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 04:13:12 +0000 Subject: [PATCH] Swarm stack for prod: autoscaled web tier (1-3), worker split, loopback LB (#9) --- .forgejo/workflows/ops-cron.yml | 13 +- .gitignore | 1 + deploy/deploy.sh | 9 ++ deploy/stack/autoscale.sh | 93 ++++++++++++ deploy/stack/deploy-stack.sh | 225 +++++++++++++++++++++++++++++ deploy/stack/env-entrypoint.sh | 37 +++++ deploy/stack/lb/Caddyfile | 30 ++++ deploy/stack/thermograph-stack.yml | 223 ++++++++++++++++++++++++++++ 8 files changed, 629 insertions(+), 2 deletions(-) create mode 100755 deploy/stack/autoscale.sh create mode 100755 deploy/stack/deploy-stack.sh create mode 100755 deploy/stack/env-entrypoint.sh create mode 100644 deploy/stack/lb/Caddyfile create mode 100644 deploy/stack/thermograph-stack.yml diff --git a/.forgejo/workflows/ops-cron.yml b/.forgejo/workflows/ops-cron.yml index c641a3e..76f8947 100644 --- a/.forgejo/workflows/ops-cron.yml +++ b/.forgejo/workflows/ops-cron.yml @@ -59,9 +59,15 @@ jobs: mkdir -p "$backup_dir" stamp="$(date -u +%Y%m%dT%H%M%SZ)" out="$backup_dir/thermograph-$stamp.dump" + # The db may run under plain compose OR as a Swarm stack task + # (prod post-cutover); resolve the container either way so the + # backup survives the deploy-mode switch. + dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1) + [ -z "$dbc" ] && dbc=$(cd /opt/thermograph && docker compose ps -q db 2>/dev/null | head -1) + [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; } # Write to a .partial and rename on success so a mid-dump failure can # never leave a truncated file that looks like a good backup. - docker compose exec -T db pg_dump -U thermograph -d thermograph \ + docker exec "$dbc" pg_dump -U thermograph -d thermograph \ --format=custom > "$out.partial" mv "$out.partial" "$out" echo "wrote $out ($(du -h "$out" | cut -f1))" @@ -91,5 +97,8 @@ jobs: set -euo pipefail cd /opt/thermograph set -a; . /etc/thermograph.env 2>/dev/null || true; set +a - docker compose exec -T backend python indexnow.py --if-changed \ + bec=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_web" | head -1) + [ -z "$bec" ] && bec=$(cd /opt/thermograph && docker compose ps -q backend 2>/dev/null | head -1) + [ -n "$bec" ] || { echo "!! no backend/web container found"; exit 1; } + docker exec "$bec" python indexnow.py --if-changed \ "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" diff --git a/.gitignore b/.gitignore index 6347e64..5795e87 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ age.key # `git clean` can't destroy the record of what's running). deploy/.image-tags.env deploy/.deploy.lock +deploy/.stack-image-tags.env diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 6b03b35..192761b 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -114,6 +114,15 @@ if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then exec "$0" "$@" fi +# Stack-mode routing: a host whose /etc/thermograph/deploy-mode says "stack" +# (prod, after the Swarm cutover) deploys via the Swarm stack path instead of +# compose. Checked AFTER the reset+re-exec so the stack script is always the +# freshly-pulled one, and the SERVICE/tag contract passes through unchanged -- +# the app repos' workflows never need to know which mode a host runs. +if [ "$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)" = "stack" ]; then + exec bash "$APP_DIR/deploy/stack/deploy-stack.sh" +fi + # Registry-pull cutover: pull the image each app repo's build-push.yml already # built and pushed, instead of building in place. This checkout is # thermograph-infra, not an app repo, so there's no "current commit" to derive diff --git a/deploy/stack/autoscale.sh b/deploy/stack/autoscale.sh new file mode 100755 index 0000000..2a36ade --- /dev/null +++ b/deploy/stack/autoscale.sh @@ -0,0 +1,93 @@ +#!/bin/sh +# Autoscaler for the stack's `web` service: scale replicas between +# MIN_REPLICAS and MAX_REPLICAS on sustained per-task CPU. +# +# Runs as a Swarm service on the manager with the docker socket mounted (see +# thermograph-stack.yml). Every web task is placed on this node today, so +# node-local `docker stats` sees them all — when a second app node exists, +# this needs a per-node reader or a metrics-based signal instead; that's the +# documented upgrade path, not a today problem. +# +# Semantics (deliberately boring): +# - Sample avg CPU% per web task every POLL_SECONDS (docker stats CPUPerc: +# 100 = one full host core). +# - UP_SAMPLES consecutive samples above SCALE_UP_CPU -> scale +1. +# - DOWN_SAMPLES consecutive samples below SCALE_DOWN_CPU -> scale -1. +# (Down is ~7x slower than up on defaults: flap-averse by construction.) +# - COOLDOWN_SECONDS after any change: samples are ignored entirely. +# - Clamped to [MIN_REPLICAS, MAX_REPLICAS]; scaling waits for convergence +# (--detach=false), so a stuck rollout blocks further changes rather than +# stacking them. +set -eu + +STACK_NAME="${STACK_NAME:-thermograph}" +SERVICE="${STACK_NAME}_web" +MIN="${MIN_REPLICAS:-1}" +MAX="${MAX_REPLICAS:-3}" +UP_AT="${SCALE_UP_CPU:-220}" +DOWN_AT="${SCALE_DOWN_CPU:-60}" +POLL="${POLL_SECONDS:-15}" +UP_N="${UP_SAMPLES:-3}" +DOWN_N="${DOWN_SAMPLES:-20}" +COOLDOWN="${COOLDOWN_SECONDS:-180}" + +up_hits=0 +down_hits=0 +last_change=0 + +log() { echo "[autoscale] $(date -u +%H:%M:%S) $*"; } + +replicas() { + docker service inspect "$SERVICE" \ + --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "" +} + +avg_cpu() { + # Mean CPUPerc across this node's web tasks, as an integer percent. + docker stats --no-stream --format '{{.Name}} {{.CPUPerc}}' 2>/dev/null \ + | awk -v svc="$SERVICE" ' + index($1, svc".") == 1 { + gsub(/%/, "", $2); sum += $2; n++ + } + END { if (n > 0) printf "%d", sum / n; else print "" }' +} + +log "watching $SERVICE: min=$MIN max=$MAX up>@${UP_AT}%x${UP_N} down<@${DOWN_AT}%x${DOWN_N} poll=${POLL}s cooldown=${COOLDOWN}s" + +while :; do + sleep "$POLL" + + now=$(date +%s) + if [ $((now - last_change)) -lt "$COOLDOWN" ]; then + continue + fi + + cur=$(replicas) + [ -n "$cur" ] || { log "service $SERVICE not found; waiting"; continue; } + cpu=$(avg_cpu) + [ -n "$cpu" ] || continue # no running tasks visible this sample + + if [ "$cpu" -gt "$UP_AT" ]; then + up_hits=$((up_hits + 1)); down_hits=0 + elif [ "$cpu" -lt "$DOWN_AT" ]; then + down_hits=$((down_hits + 1)); up_hits=0 + else + up_hits=0; down_hits=0 + fi + + if [ "$up_hits" -ge "$UP_N" ] && [ "$cur" -lt "$MAX" ]; then + target=$((cur + 1)) + log "avg cpu ${cpu}% > ${UP_AT}% x${UP_N}: scaling $cur -> $target" + if docker service scale --detach=false "$SERVICE=$target"; then + last_change=$(date +%s) + fi + up_hits=0; down_hits=0 + elif [ "$down_hits" -ge "$DOWN_N" ] && [ "$cur" -gt "$MIN" ]; then + target=$((cur - 1)) + log "avg cpu ${cpu}% < ${DOWN_AT}% x${DOWN_N}: scaling $cur -> $target" + if docker service scale --detach=false "$SERVICE=$target"; then + last_change=$(date +%s) + fi + up_hits=0; down_hits=0 + fi +done diff --git a/deploy/stack/deploy-stack.sh b/deploy/stack/deploy-stack.sh new file mode 100755 index 0000000..f2df221 --- /dev/null +++ b/deploy/stack/deploy-stack.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh, +# speaking the SAME contract the app repos' workflows already use +# (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so +# switching a host to stack mode needs no workflow changes: deploy.sh execs +# this when /etc/thermograph/deploy-mode contains "stack". +# +# What a roll does here vs compose: +# backend -> one-shot migrate, then `docker service update --image` on +# web AND worker (same image; start-first, health-gated, +# auto-rollback on failure). +# frontend -> `docker service update --image` on frontend. +# all -> full `docker stack deploy` (+ migrate first), which also +# applies stack-file changes (new services, env, limits). +# +# TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with +# throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel +# rehearsal on the same host that cannot touch live data or ports. +set -euo pipefail + +APP_DIR="${APP_DIR:-/opt/thermograph}" +SERVICE="${SERVICE:-all}" +cd "$APP_DIR" + +case "$SERVICE" in + backend|frontend|all) ;; + *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; +esac + +if [ "${STACK_TEST:-0}" = "1" ]; then + STACK_NAME="thermograph-test" + LB_NAME="thermograph-test-lb" + LB_HTTP_PORT=18137; LB_FE_PORT=18080 + export PGDATA_VOLUME="thermograph-test_pgdata" + export APPDATA_VOLUME="thermograph-test_appdata" + export APPLOGS_VOLUME="thermograph-test_applogs" + docker volume create "$PGDATA_VOLUME" >/dev/null + docker volume create "$APPDATA_VOLUME" >/dev/null + docker volume create "$APPLOGS_VOLUME" >/dev/null +else + STACK_NAME="${STACK_NAME:-thermograph}" + LB_NAME="thermograph-lb" + LB_HTTP_PORT=8137; LB_FE_PORT=8080 +fi +export STACK_NAME + +# --- secrets ------------------------------------------------------------------ +# Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then +# install the uid-10001-readable copy the tasks' env-entrypoint shim sources. +# 10001 = the app images' `thermograph` user; the file is 0400 to that uid. +if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then + # shellcheck source=deploy/render-secrets.sh + . "$APP_DIR/deploy/render-secrets.sh" + render_thermograph_secrets "$APP_DIR" +fi +set -a; . /etc/thermograph.env 2>/dev/null || true; set +a +sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \ + || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env + +# --- image tags ----------------------------------------------------------------- +# Same persisted-tags contract as deploy.sh: incoming env wins, the file +# supplies the sibling. Stack mode keeps its own file so test/real never mix. +REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}" +export REGISTRY_HOST +TAGS_FILE="$APP_DIR/deploy/.stack-image-tags.env" +_incoming_backend="${BACKEND_IMAGE_TAG:-}" +_incoming_frontend="${FRONTEND_IMAGE_TAG:-}" +if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi +[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend" +[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend" +case "$SERVICE" in + backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;; + frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;; + all) + : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}" + : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;; +esac +export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" +export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" +BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:$BACKEND_IMAGE_TAG" +FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}:$FRONTEND_IMAGE_TAG" + +# --- timescale image pin --------------------------------------------------------- +# Hazard #7: the db image under an existing volume must never drift. Resolve +# the digest-pinned ref from whatever is running (stack task or compose +# container), falling back to the local latest-pg18's digest on first bring-up. +if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then + cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) + [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1) + if [ -n "$cid" ]; then + img=$(docker inspect --format '{{.Image}}' "$cid") + else + img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}" + fi + TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1) + [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img" +fi +export TIMESCALEDB_IMAGE +echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" + +# --- registry ------------------------------------------------------------------ +if [ -n "${REGISTRY_TOKEN:-}" ]; then + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin +fi +echo "==> Pulling images" +pull_ok=0 +for i in $(seq 1 30); do + if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then + pull_ok=1; break + fi + echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2 + sleep 10 +done +[ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; } + +# --- one-shot migrations --------------------------------------------------------- +# Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0 +# in the stack). Runs on the stack's overlay so `db` resolves. First-ever +# deploy: the network doesn't exist yet — create it exactly as the stack will +# (attachable overlay) so the name is adopted, then migrate, then deploy. +NET="${STACK_NAME}_internal" +docker network inspect "$NET" >/dev/null 2>&1 \ + || docker network create --driver overlay --attachable --scope swarm "$NET" >/dev/null +if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then + if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then + echo "==> One-shot migrate ($BACKEND_IMAGE)" + docker run --rm --network "$NET" \ + -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ + --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate + else + echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below" + fi +fi + +# --- deploy ---------------------------------------------------------------------- +FIRST_DEPLOY_MIGRATE=0 +docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 +if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then + echo "==> docker stack deploy ($STACK_NAME)" + docker stack deploy --with-registry-auth -c "$APP_DIR/deploy/stack/thermograph-stack.yml" "$STACK_NAME" + # First-ever deploy ran no migrate above (db didn't exist): wait for db, + # migrate, then force web/worker to restart cleanly against the schema. + if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then + echo "==> Waiting for db, then first-boot migrate" + for i in $(seq 1 60); do + cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) + [ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break + sleep 5 + done + docker run --rm --network "$NET" \ + -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ + --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate + docker service update --force --detach=false "${STACK_NAME}_web" + docker service update --force --detach=false "${STACK_NAME}_worker" + fi +else + case "$SERVICE" in + backend) + echo "==> Rolling web + worker to $BACKEND_IMAGE" + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web" + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker" + ;; + frontend) + echo "==> Rolling frontend to $FRONTEND_IMAGE" + docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend" + ;; + esac +fi + +# --- loopback LB bridge ----------------------------------------------------------- +# A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes +# 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated +# only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME` +# to force a refresh after editing lb/Caddyfile. +if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then + docker rm -f "$LB_NAME" >/dev/null 2>&1 || true + echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)" + docker run -d --name "$LB_NAME" --restart unless-stopped \ + --network "$NET" \ + -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \ + -v "$APP_DIR/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \ + caddy:2-alpine >/dev/null +fi + +# --- verify ----------------------------------------------------------------------- +echo "==> Health check via the LB" +ok=0 +for i in $(seq 1 60); do + if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi + sleep 2 +done +if [ "$ok" != 1 ]; then + echo "!! stack health check failed" >&2 + docker stack ps "$STACK_NAME" --no-trunc | head -20 + exit 1 +fi +echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}" + +# Persist now-live tags (after health, like deploy.sh). +cat > "$TAGS_FILE" </dev/null || true + +# Post-deploy warm + IndexNow, via any web task (skip in test mode: no data, +# and the warmer would burn upstream quota against an empty cache). +if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then + wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1) + if [ -n "$wcid" ]; then + echo "==> Warming city archives (detached) + IndexNow" + docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true + docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \ + || echo "!! IndexNow ping failed (non-fatal)" >&2 + fi +fi +exit 0 diff --git a/deploy/stack/env-entrypoint.sh b/deploy/stack/env-entrypoint.sh new file mode 100755 index 0000000..7905b01 --- /dev/null +++ b/deploy/stack/env-entrypoint.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Stack-task entrypoint shim: source the host-rendered secrets env, then hand +# off to the image's real entrypoint. +# +# Why: `docker stack deploy` does not support compose's `env_file:`, and +# enumerating every vault key in the stack yml's `environment:` blocks would +# drift the moment a key is added to deploy/secrets/. Instead deploy-stack.sh +# installs a uid-10001-readable copy of the rendered env at +# /etc/thermograph/stack.env, the stack bind-mounts it (with this script) into +# every app task, and this shim exports each KEY=value — but ONLY for keys not +# already set, so the yml's `environment:` blocks keep compose's env_file +# precedence (environment always wins). Same set-if-unset contract as the +# image's own /run/secrets shim in deploy/entrypoint.sh, which still runs +# after this and stays a no-op here. +set -euo pipefail + +ENV_FILE="${THERMOGRAPH_HOST_ENV:-/host/thermograph.env}" + +if [ -f "$ENV_FILE" ]; then + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + *=*) + key="${line%%=*}" + # Only sane identifiers; only if not already set by `environment:`. + case "$key" in + *[!A-Za-z0-9_]*|'') continue ;; + esac + if [ -z "${!key:-}" ]; then + export "$key=${line#*=}" + fi + ;; + esac + done < "$ENV_FILE" +fi + +exec /app/deploy/entrypoint.sh "$@" diff --git a/deploy/stack/lb/Caddyfile b/deploy/stack/lb/Caddyfile new file mode 100644 index 0000000..46402ef --- /dev/null +++ b/deploy/stack/lb/Caddyfile @@ -0,0 +1,30 @@ +# The loopback LB bridge's own config — NOT the host Caddy (that one still +# terminates TLS for thermograph.org and proxies to 127.0.0.1:8137 exactly as +# before; it needs no change for the stack cutover). +# +# This Caddy runs as a PLAIN container (deploy-stack.sh manages it) because +# only plain containers can bind a specific host IP: Swarm port configs +# publish on 0.0.0.0 (routing mesh or host mode alike), which would expose +# the plaintext app un-fronted — hazard #6. It joins the stack's attachable +# overlay and proxies to the service VIPs; Swarm's VIP round-robins across +# however many web replicas the autoscaler is running, so this bridge never +# needs to know the replica count. + +{ + auto_https off + admin off +} + +:8137 { + reverse_proxy web:8137 { + # Fail fast to the client if the VIP has no healthy task; Swarm's own + # task healthchecks handle ejecting dead replicas from the VIP. + lb_try_duration 5s + } +} + +:8080 { + reverse_proxy frontend:8080 { + lb_try_duration 5s + } +} diff --git a/deploy/stack/thermograph-stack.yml b/deploy/stack/thermograph-stack.yml new file mode 100644 index 0000000..5916337 --- /dev/null +++ b/deploy/stack/thermograph-stack.yml @@ -0,0 +1,223 @@ +# Docker Swarm stack for prod — the autoscaling successor to docker-compose.yml +# on that host (beta and LAN dev stay on plain compose; deploy.sh routes by the +# /etc/thermograph/deploy-mode marker). Two-image world: web/worker run the +# backend image, frontend its own — per-service tags, same contract as compose. +# +# Deployed by deploy/stack/deploy-stack.sh, which: +# - sources /etc/thermograph.env (SOPS-rendered) so ${VARS} here interpolate, +# - installs a uid-10001-readable copy at /etc/thermograph/stack.env that +# deploy/stack/env-entrypoint.sh sources inside each app task (set-if-unset, +# so `environment:` blocks below always win) — this replaces compose's +# env_file:, which `docker stack deploy` does not support, +# - runs migrations as a ONE-SHOT task before rolling (RUN_MIGRATIONS=0 in +# every replica — N replicas must never race Alembic), +# - manages the loopback LB bridge (see lb/README note below): Swarm's mesh +# can only publish on 0.0.0.0 (would expose the plaintext app un-fronted), +# so nothing here has `ports:`. A plain container on this attachable +# overlay binds 127.0.0.1:8137/8080 for the host Caddy and proxies to the +# service VIPs — Swarm's VIP does the actual load balancing across +# replicas. +# +# Scaling model: `web` is stateless (ROLE=web never runs the notifier) and +# scales 1..N — the autoscaler service adjusts replicas between +# WEB_MIN_REPLICAS/WEB_MAX_REPLICAS on task CPU. `worker` owns the notifier + +# scheduler: exactly 1 replica, with the cluster-wide Postgres advisory lock +# (THERMOGRAPH_SINGLETON_PG) as belt-and-suspenders. `db` is exactly 1 — a +# database does not scale by container count on one host; its levers are +# DB_CPUS/DB_MEMORY (and, multi-host later, Patroni replicas per the topology +# doc). Everything is pinned to the manager node: all volumes are local to +# prod today. That constraint is the ONLY thing to relax when a second app +# node joins. +# +# TIMESCALEDB_IMAGE must be the exact image (digest-pinned) the compose stack +# was running — see hazard #7 in the hop-1 runbook: a floating tag can change +# the extension minor under an existing volume. deploy-stack.sh resolves it +# from the running/last-known container automatically. + +services: + db: + image: ${TIMESCALEDB_IMAGE:?set TIMESCALEDB_IMAGE to the exact (digest-pinned) image the volume was created with} + environment: + POSTGRES_USER: thermograph + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + POSTGRES_DB: thermograph + DB_MEMORY: ${DB_MEMORY:-16g} + volumes: + - pgdata:/var/lib/postgresql + - /opt/thermograph/deploy/db/init:/docker-entrypoint-initdb.d:ro + networks: + - internal + healthcheck: + test: ["CMD-SHELL", "pg_isready -U thermograph -d thermograph"] + interval: 5s + timeout: 5s + retries: 10 + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${DB_CPUS:-4}" + memory: ${DB_MEMORY:-16g} + restart_policy: + condition: on-failure + + web: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph + THERMOGRAPH_BASE: / + PORT: 8137 + THERMOGRAPH_SERVICE_ROLE: backend + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://frontend:8080 + WORKERS: ${WEB_WORKERS:-4} + THERMOGRAPH_DATA_DIR: /state + # web NEVER runs the notifier/scheduler, even if it would win election — + # that's the worker service's job. This is what makes web replicas safe. + THERMOGRAPH_ROLE: web + RUN_MIGRATIONS: "0" + # Overlay tasks reach the HOST's Postfix via the docker_gwbridge gateway, + # not the compose bridge's 172.19.0.1 (an overlay has no host gateway). + # provision-mail.sh's DOCKER_MAIL_GATEWAY/SUBNET cover this listener. + THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + volumes: + - appdata:/state + - applogs:/app/logs + - /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + replicas: ${WEB_MIN_REPLICAS:-1} + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${WEB_CPUS:-4}" + restart_policy: + condition: on-failure + update_config: + # New task must pass the image's own HEALTHCHECK before the old one + # stops — zero-downtime single-service rolls. + order: start-first + failure_action: rollback + + worker: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph + THERMOGRAPH_BASE: / + PORT: 8137 + THERMOGRAPH_SERVICE_ROLE: backend + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://frontend:8080 + WORKERS: "1" + THERMOGRAPH_DATA_DIR: /state + THERMOGRAPH_ROLE: worker + # Cluster-wide Postgres advisory lock, not the host flock: correct at + # replicas=1 today and stays correct if a second worker ever appears. + THERMOGRAPH_SINGLETON_PG: "1" + RUN_MIGRATIONS: "0" + THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + volumes: + - appdata:/state + - applogs:/app/logs + - /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${WORKER_CPUS:-2}" + restart_policy: + condition: on-failure + + frontend: + image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}:${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_BASE: / + PORT: 8080 + THERMOGRAPH_SERVICE_ROLE: frontend + THERMOGRAPH_API_BASE_INTERNAL: http://web:8137 + volumes: + - /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${FRONTEND_CPUS:-2}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + + # Scales `web` between WEB_MIN_REPLICAS and WEB_MAX_REPLICAS on sustained + # task CPU (docker stats on this node — every task is pinned here today). + # Deliberately a dumb shell loop with hysteresis + cooldown, not an + # autoscaling framework: Swarm has no native autoscaler and this workload + # doesn't justify one (topology doc §6 — declarative scaling as a feature). + autoscaler: + image: docker:27-cli + entrypoint: ["/bin/sh", "/host/autoscale.sh"] + environment: + STACK_NAME: ${STACK_NAME:-thermograph} + MIN_REPLICAS: ${WEB_MIN_REPLICAS:-1} + MAX_REPLICAS: ${WEB_MAX_REPLICAS:-3} + # Thresholds are avg docker-stats CPU% PER TASK (host-core-relative: 100 + # = one full core). Up fast, down slow. + SCALE_UP_CPU: ${SCALE_UP_CPU:-220} + SCALE_DOWN_CPU: ${SCALE_DOWN_CPU:-60} + POLL_SECONDS: ${POLL_SECONDS:-15} + UP_SAMPLES: ${UP_SAMPLES:-3} + DOWN_SAMPLES: ${DOWN_SAMPLES:-20} + COOLDOWN_SECONDS: ${COOLDOWN_SECONDS:-180} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /opt/thermograph/deploy/stack/autoscale.sh:/host/autoscale.sh:ro + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "0.2" + memory: 64m + restart_policy: + condition: any + +networks: + internal: + driver: overlay + # Attachable so the loopback LB bridge (a PLAIN container — only plain + # containers can bind 127.0.0.1; Swarm port configs cannot) can join and + # reach the service VIPs. + attachable: true + +volumes: + # External and explicitly named: the real stack REUSES the volumes the + # compose stack created (same data, zero migration). deploy-stack.sh's test + # mode points these at throwaway names instead. + pgdata: + external: true + name: ${PGDATA_VOLUME:-thermograph_pgdata} + appdata: + external: true + name: ${APPDATA_VOLUME:-thermograph_appdata} + applogs: + external: true + name: ${APPLOGS_VOLUME:-thermograph_applogs}