Swarm stack for prod: autoscaled web tier (1-3), worker split, loopback LB (#9)
This commit is contained in:
parent
d02c0f719f
commit
9cd24387f2
8 changed files with 629 additions and 2 deletions
|
|
@ -59,9 +59,15 @@ jobs:
|
||||||
mkdir -p "$backup_dir"
|
mkdir -p "$backup_dir"
|
||||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
out="$backup_dir/thermograph-$stamp.dump"
|
out="$backup_dir/thermograph-$stamp.dump"
|
||||||
|
# The db may run under plain compose OR as a Swarm stack task
|
||||||
|
# (prod post-cutover); resolve the container either way so the
|
||||||
|
# backup survives the deploy-mode switch.
|
||||||
|
dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1)
|
||||||
|
[ -z "$dbc" ] && dbc=$(cd /opt/thermograph && docker compose ps -q db 2>/dev/null | head -1)
|
||||||
|
[ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; }
|
||||||
# Write to a .partial and rename on success so a mid-dump failure can
|
# Write to a .partial and rename on success so a mid-dump failure can
|
||||||
# never leave a truncated file that looks like a good backup.
|
# never leave a truncated file that looks like a good backup.
|
||||||
docker compose exec -T db pg_dump -U thermograph -d thermograph \
|
docker exec "$dbc" pg_dump -U thermograph -d thermograph \
|
||||||
--format=custom > "$out.partial"
|
--format=custom > "$out.partial"
|
||||||
mv "$out.partial" "$out"
|
mv "$out.partial" "$out"
|
||||||
echo "wrote $out ($(du -h "$out" | cut -f1))"
|
echo "wrote $out ($(du -h "$out" | cut -f1))"
|
||||||
|
|
@ -91,5 +97,8 @@ jobs:
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd /opt/thermograph
|
cd /opt/thermograph
|
||||||
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
|
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
|
||||||
docker compose exec -T backend python indexnow.py --if-changed \
|
bec=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_web" | head -1)
|
||||||
|
[ -z "$bec" ] && bec=$(cd /opt/thermograph && docker compose ps -q backend 2>/dev/null | head -1)
|
||||||
|
[ -n "$bec" ] || { echo "!! no backend/web container found"; exit 1; }
|
||||||
|
docker exec "$bec" python indexnow.py --if-changed \
|
||||||
"${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
|
"${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
|
||||||
|
|
|
||||||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -22,3 +22,4 @@ age.key
|
||||||
# `git clean` can't destroy the record of what's running).
|
# `git clean` can't destroy the record of what's running).
|
||||||
deploy/.image-tags.env
|
deploy/.image-tags.env
|
||||||
deploy/.deploy.lock
|
deploy/.deploy.lock
|
||||||
|
deploy/.stack-image-tags.env
|
||||||
|
|
|
||||||
|
|
@ -114,6 +114,15 @@ if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then
|
||||||
exec "$0" "$@"
|
exec "$0" "$@"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Stack-mode routing: a host whose /etc/thermograph/deploy-mode says "stack"
|
||||||
|
# (prod, after the Swarm cutover) deploys via the Swarm stack path instead of
|
||||||
|
# compose. Checked AFTER the reset+re-exec so the stack script is always the
|
||||||
|
# freshly-pulled one, and the SERVICE/tag contract passes through unchanged --
|
||||||
|
# the app repos' workflows never need to know which mode a host runs.
|
||||||
|
if [ "$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)" = "stack" ]; then
|
||||||
|
exec bash "$APP_DIR/deploy/stack/deploy-stack.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
# Registry-pull cutover: pull the image each app repo's build-push.yml already
|
# Registry-pull cutover: pull the image each app repo's build-push.yml already
|
||||||
# built and pushed, instead of building in place. This checkout is
|
# built and pushed, instead of building in place. This checkout is
|
||||||
# thermograph-infra, not an app repo, so there's no "current commit" to derive
|
# thermograph-infra, not an app repo, so there's no "current commit" to derive
|
||||||
|
|
|
||||||
93
deploy/stack/autoscale.sh
Executable file
93
deploy/stack/autoscale.sh
Executable file
|
|
@ -0,0 +1,93 @@
|
||||||
|
#!/bin/sh
|
||||||
|
# Autoscaler for the stack's `web` service: scale replicas between
|
||||||
|
# MIN_REPLICAS and MAX_REPLICAS on sustained per-task CPU.
|
||||||
|
#
|
||||||
|
# Runs as a Swarm service on the manager with the docker socket mounted (see
|
||||||
|
# thermograph-stack.yml). Every web task is placed on this node today, so
|
||||||
|
# node-local `docker stats` sees them all — when a second app node exists,
|
||||||
|
# this needs a per-node reader or a metrics-based signal instead; that's the
|
||||||
|
# documented upgrade path, not a today problem.
|
||||||
|
#
|
||||||
|
# Semantics (deliberately boring):
|
||||||
|
# - Sample avg CPU% per web task every POLL_SECONDS (docker stats CPUPerc:
|
||||||
|
# 100 = one full host core).
|
||||||
|
# - UP_SAMPLES consecutive samples above SCALE_UP_CPU -> scale +1.
|
||||||
|
# - DOWN_SAMPLES consecutive samples below SCALE_DOWN_CPU -> scale -1.
|
||||||
|
# (Down is ~7x slower than up on defaults: flap-averse by construction.)
|
||||||
|
# - COOLDOWN_SECONDS after any change: samples are ignored entirely.
|
||||||
|
# - Clamped to [MIN_REPLICAS, MAX_REPLICAS]; scaling waits for convergence
|
||||||
|
# (--detach=false), so a stuck rollout blocks further changes rather than
|
||||||
|
# stacking them.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
STACK_NAME="${STACK_NAME:-thermograph}"
|
||||||
|
SERVICE="${STACK_NAME}_web"
|
||||||
|
MIN="${MIN_REPLICAS:-1}"
|
||||||
|
MAX="${MAX_REPLICAS:-3}"
|
||||||
|
UP_AT="${SCALE_UP_CPU:-220}"
|
||||||
|
DOWN_AT="${SCALE_DOWN_CPU:-60}"
|
||||||
|
POLL="${POLL_SECONDS:-15}"
|
||||||
|
UP_N="${UP_SAMPLES:-3}"
|
||||||
|
DOWN_N="${DOWN_SAMPLES:-20}"
|
||||||
|
COOLDOWN="${COOLDOWN_SECONDS:-180}"
|
||||||
|
|
||||||
|
up_hits=0
|
||||||
|
down_hits=0
|
||||||
|
last_change=0
|
||||||
|
|
||||||
|
log() { echo "[autoscale] $(date -u +%H:%M:%S) $*"; }
|
||||||
|
|
||||||
|
replicas() {
|
||||||
|
docker service inspect "$SERVICE" \
|
||||||
|
--format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo ""
|
||||||
|
}
|
||||||
|
|
||||||
|
avg_cpu() {
|
||||||
|
# Mean CPUPerc across this node's web tasks, as an integer percent.
|
||||||
|
docker stats --no-stream --format '{{.Name}} {{.CPUPerc}}' 2>/dev/null \
|
||||||
|
| awk -v svc="$SERVICE" '
|
||||||
|
index($1, svc".") == 1 {
|
||||||
|
gsub(/%/, "", $2); sum += $2; n++
|
||||||
|
}
|
||||||
|
END { if (n > 0) printf "%d", sum / n; else print "" }'
|
||||||
|
}
|
||||||
|
|
||||||
|
log "watching $SERVICE: min=$MIN max=$MAX up>@${UP_AT}%x${UP_N} down<@${DOWN_AT}%x${DOWN_N} poll=${POLL}s cooldown=${COOLDOWN}s"
|
||||||
|
|
||||||
|
while :; do
|
||||||
|
sleep "$POLL"
|
||||||
|
|
||||||
|
now=$(date +%s)
|
||||||
|
if [ $((now - last_change)) -lt "$COOLDOWN" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
cur=$(replicas)
|
||||||
|
[ -n "$cur" ] || { log "service $SERVICE not found; waiting"; continue; }
|
||||||
|
cpu=$(avg_cpu)
|
||||||
|
[ -n "$cpu" ] || continue # no running tasks visible this sample
|
||||||
|
|
||||||
|
if [ "$cpu" -gt "$UP_AT" ]; then
|
||||||
|
up_hits=$((up_hits + 1)); down_hits=0
|
||||||
|
elif [ "$cpu" -lt "$DOWN_AT" ]; then
|
||||||
|
down_hits=$((down_hits + 1)); up_hits=0
|
||||||
|
else
|
||||||
|
up_hits=0; down_hits=0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$up_hits" -ge "$UP_N" ] && [ "$cur" -lt "$MAX" ]; then
|
||||||
|
target=$((cur + 1))
|
||||||
|
log "avg cpu ${cpu}% > ${UP_AT}% x${UP_N}: scaling $cur -> $target"
|
||||||
|
if docker service scale --detach=false "$SERVICE=$target"; then
|
||||||
|
last_change=$(date +%s)
|
||||||
|
fi
|
||||||
|
up_hits=0; down_hits=0
|
||||||
|
elif [ "$down_hits" -ge "$DOWN_N" ] && [ "$cur" -gt "$MIN" ]; then
|
||||||
|
target=$((cur - 1))
|
||||||
|
log "avg cpu ${cpu}% < ${DOWN_AT}% x${DOWN_N}: scaling $cur -> $target"
|
||||||
|
if docker service scale --detach=false "$SERVICE=$target"; then
|
||||||
|
last_change=$(date +%s)
|
||||||
|
fi
|
||||||
|
up_hits=0; down_hits=0
|
||||||
|
fi
|
||||||
|
done
|
||||||
225
deploy/stack/deploy-stack.sh
Executable file
225
deploy/stack/deploy-stack.sh
Executable file
|
|
@ -0,0 +1,225 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh,
|
||||||
|
# speaking the SAME contract the app repos' workflows already use
|
||||||
|
# (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so
|
||||||
|
# switching a host to stack mode needs no workflow changes: deploy.sh execs
|
||||||
|
# this when /etc/thermograph/deploy-mode contains "stack".
|
||||||
|
#
|
||||||
|
# What a roll does here vs compose:
|
||||||
|
# backend -> one-shot migrate, then `docker service update --image` on
|
||||||
|
# web AND worker (same image; start-first, health-gated,
|
||||||
|
# auto-rollback on failure).
|
||||||
|
# frontend -> `docker service update --image` on frontend.
|
||||||
|
# all -> full `docker stack deploy` (+ migrate first), which also
|
||||||
|
# applies stack-file changes (new services, env, limits).
|
||||||
|
#
|
||||||
|
# TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with
|
||||||
|
# throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel
|
||||||
|
# rehearsal on the same host that cannot touch live data or ports.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
APP_DIR="${APP_DIR:-/opt/thermograph}"
|
||||||
|
SERVICE="${SERVICE:-all}"
|
||||||
|
cd "$APP_DIR"
|
||||||
|
|
||||||
|
case "$SERVICE" in
|
||||||
|
backend|frontend|all) ;;
|
||||||
|
*) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [ "${STACK_TEST:-0}" = "1" ]; then
|
||||||
|
STACK_NAME="thermograph-test"
|
||||||
|
LB_NAME="thermograph-test-lb"
|
||||||
|
LB_HTTP_PORT=18137; LB_FE_PORT=18080
|
||||||
|
export PGDATA_VOLUME="thermograph-test_pgdata"
|
||||||
|
export APPDATA_VOLUME="thermograph-test_appdata"
|
||||||
|
export APPLOGS_VOLUME="thermograph-test_applogs"
|
||||||
|
docker volume create "$PGDATA_VOLUME" >/dev/null
|
||||||
|
docker volume create "$APPDATA_VOLUME" >/dev/null
|
||||||
|
docker volume create "$APPLOGS_VOLUME" >/dev/null
|
||||||
|
else
|
||||||
|
STACK_NAME="${STACK_NAME:-thermograph}"
|
||||||
|
LB_NAME="thermograph-lb"
|
||||||
|
LB_HTTP_PORT=8137; LB_FE_PORT=8080
|
||||||
|
fi
|
||||||
|
export STACK_NAME
|
||||||
|
|
||||||
|
# --- secrets ------------------------------------------------------------------
|
||||||
|
# Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then
|
||||||
|
# install the uid-10001-readable copy the tasks' env-entrypoint shim sources.
|
||||||
|
# 10001 = the app images' `thermograph` user; the file is 0400 to that uid.
|
||||||
|
if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then
|
||||||
|
# shellcheck source=deploy/render-secrets.sh
|
||||||
|
. "$APP_DIR/deploy/render-secrets.sh"
|
||||||
|
render_thermograph_secrets "$APP_DIR"
|
||||||
|
fi
|
||||||
|
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
|
||||||
|
sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \
|
||||||
|
|| install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env
|
||||||
|
|
||||||
|
# --- image tags -----------------------------------------------------------------
|
||||||
|
# Same persisted-tags contract as deploy.sh: incoming env wins, the file
|
||||||
|
# supplies the sibling. Stack mode keeps its own file so test/real never mix.
|
||||||
|
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
|
||||||
|
export REGISTRY_HOST
|
||||||
|
TAGS_FILE="$APP_DIR/deploy/.stack-image-tags.env"
|
||||||
|
_incoming_backend="${BACKEND_IMAGE_TAG:-}"
|
||||||
|
_incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
|
||||||
|
if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi
|
||||||
|
[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
|
||||||
|
[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
|
||||||
|
case "$SERVICE" in
|
||||||
|
backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;;
|
||||||
|
frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;;
|
||||||
|
all)
|
||||||
|
: "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}"
|
||||||
|
: "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;;
|
||||||
|
esac
|
||||||
|
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
|
||||||
|
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
|
||||||
|
BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:$BACKEND_IMAGE_TAG"
|
||||||
|
FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}:$FRONTEND_IMAGE_TAG"
|
||||||
|
|
||||||
|
# --- timescale image pin ---------------------------------------------------------
|
||||||
|
# Hazard #7: the db image under an existing volume must never drift. Resolve
|
||||||
|
# the digest-pinned ref from whatever is running (stack task or compose
|
||||||
|
# container), falling back to the local latest-pg18's digest on first bring-up.
|
||||||
|
if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then
|
||||||
|
cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
|
||||||
|
[ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1)
|
||||||
|
if [ -n "$cid" ]; then
|
||||||
|
img=$(docker inspect --format '{{.Image}}' "$cid")
|
||||||
|
else
|
||||||
|
img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}"
|
||||||
|
fi
|
||||||
|
TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1)
|
||||||
|
[ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img"
|
||||||
|
fi
|
||||||
|
export TIMESCALEDB_IMAGE
|
||||||
|
echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE"
|
||||||
|
|
||||||
|
# --- registry ------------------------------------------------------------------
|
||||||
|
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||||
|
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
||||||
|
fi
|
||||||
|
echo "==> Pulling images"
|
||||||
|
pull_ok=0
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then
|
||||||
|
pull_ok=1; break
|
||||||
|
fi
|
||||||
|
echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
|
[ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; }
|
||||||
|
|
||||||
|
# --- one-shot migrations ---------------------------------------------------------
|
||||||
|
# Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0
|
||||||
|
# in the stack). Runs on the stack's overlay so `db` resolves. First-ever
|
||||||
|
# deploy: the network doesn't exist yet — create it exactly as the stack will
|
||||||
|
# (attachable overlay) so the name is adopted, then migrate, then deploy.
|
||||||
|
NET="${STACK_NAME}_internal"
|
||||||
|
docker network inspect "$NET" >/dev/null 2>&1 \
|
||||||
|
|| docker network create --driver overlay --attachable --scope swarm "$NET" >/dev/null
|
||||||
|
if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then
|
||||||
|
if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then
|
||||||
|
echo "==> One-shot migrate ($BACKEND_IMAGE)"
|
||||||
|
docker run --rm --network "$NET" \
|
||||||
|
-e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \
|
||||||
|
--entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
|
||||||
|
else
|
||||||
|
echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- deploy ----------------------------------------------------------------------
|
||||||
|
FIRST_DEPLOY_MIGRATE=0
|
||||||
|
docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1
|
||||||
|
if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then
|
||||||
|
echo "==> docker stack deploy ($STACK_NAME)"
|
||||||
|
docker stack deploy --with-registry-auth -c "$APP_DIR/deploy/stack/thermograph-stack.yml" "$STACK_NAME"
|
||||||
|
# First-ever deploy ran no migrate above (db didn't exist): wait for db,
|
||||||
|
# migrate, then force web/worker to restart cleanly against the schema.
|
||||||
|
if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then
|
||||||
|
echo "==> Waiting for db, then first-boot migrate"
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
|
||||||
|
[ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
docker run --rm --network "$NET" \
|
||||||
|
-e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \
|
||||||
|
--entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
|
||||||
|
docker service update --force --detach=false "${STACK_NAME}_web"
|
||||||
|
docker service update --force --detach=false "${STACK_NAME}_worker"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
case "$SERVICE" in
|
||||||
|
backend)
|
||||||
|
echo "==> Rolling web + worker to $BACKEND_IMAGE"
|
||||||
|
docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web"
|
||||||
|
docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker"
|
||||||
|
;;
|
||||||
|
frontend)
|
||||||
|
echo "==> Rolling frontend to $FRONTEND_IMAGE"
|
||||||
|
docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- loopback LB bridge -----------------------------------------------------------
|
||||||
|
# A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes
|
||||||
|
# 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated
|
||||||
|
# only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME`
|
||||||
|
# to force a refresh after editing lb/Caddyfile.
|
||||||
|
if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then
|
||||||
|
docker rm -f "$LB_NAME" >/dev/null 2>&1 || true
|
||||||
|
echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)"
|
||||||
|
docker run -d --name "$LB_NAME" --restart unless-stopped \
|
||||||
|
--network "$NET" \
|
||||||
|
-p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \
|
||||||
|
-v "$APP_DIR/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \
|
||||||
|
caddy:2-alpine >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- verify -----------------------------------------------------------------------
|
||||||
|
echo "==> Health check via the LB"
|
||||||
|
ok=0
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if [ "$ok" != 1 ]; then
|
||||||
|
echo "!! stack health check failed" >&2
|
||||||
|
docker stack ps "$STACK_NAME" --no-trunc | head -20
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}"
|
||||||
|
|
||||||
|
# Persist now-live tags (after health, like deploy.sh).
|
||||||
|
cat > "$TAGS_FILE" <<EOF
|
||||||
|
# Written by deploy-stack.sh -- the image tag each service is currently running.
|
||||||
|
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
|
||||||
|
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# GC superseded app-image tags (keep the running pair), same as deploy.sh.
|
||||||
|
_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}"
|
||||||
|
_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}"
|
||||||
|
docker images --format '{{.Repository}}:{{.Tag}}' \
|
||||||
|
| grep -E "^(${_be_repo}|${_fe_repo}):" \
|
||||||
|
| grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
|
||||||
|
| xargs -r docker rmi 2>/dev/null || true
|
||||||
|
|
||||||
|
# Post-deploy warm + IndexNow, via any web task (skip in test mode: no data,
|
||||||
|
# and the warmer would burn upstream quota against an empty cache).
|
||||||
|
if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then
|
||||||
|
wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1)
|
||||||
|
if [ -n "$wcid" ]; then
|
||||||
|
echo "==> Warming city archives (detached) + IndexNow"
|
||||||
|
docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
|
||||||
|
docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \
|
||||||
|
|| echo "!! IndexNow ping failed (non-fatal)" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
37
deploy/stack/env-entrypoint.sh
Executable file
37
deploy/stack/env-entrypoint.sh
Executable file
|
|
@ -0,0 +1,37 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Stack-task entrypoint shim: source the host-rendered secrets env, then hand
|
||||||
|
# off to the image's real entrypoint.
|
||||||
|
#
|
||||||
|
# Why: `docker stack deploy` does not support compose's `env_file:`, and
|
||||||
|
# enumerating every vault key in the stack yml's `environment:` blocks would
|
||||||
|
# drift the moment a key is added to deploy/secrets/. Instead deploy-stack.sh
|
||||||
|
# installs a uid-10001-readable copy of the rendered env at
|
||||||
|
# /etc/thermograph/stack.env, the stack bind-mounts it (with this script) into
|
||||||
|
# every app task, and this shim exports each KEY=value — but ONLY for keys not
|
||||||
|
# already set, so the yml's `environment:` blocks keep compose's env_file
|
||||||
|
# precedence (environment always wins). Same set-if-unset contract as the
|
||||||
|
# image's own /run/secrets shim in deploy/entrypoint.sh, which still runs
|
||||||
|
# after this and stays a no-op here.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ENV_FILE="${THERMOGRAPH_HOST_ENV:-/host/thermograph.env}"
|
||||||
|
|
||||||
|
if [ -f "$ENV_FILE" ]; then
|
||||||
|
while IFS= read -r line || [ -n "$line" ]; do
|
||||||
|
case "$line" in
|
||||||
|
''|'#'*) continue ;;
|
||||||
|
*=*)
|
||||||
|
key="${line%%=*}"
|
||||||
|
# Only sane identifiers; only if not already set by `environment:`.
|
||||||
|
case "$key" in
|
||||||
|
*[!A-Za-z0-9_]*|'') continue ;;
|
||||||
|
esac
|
||||||
|
if [ -z "${!key:-}" ]; then
|
||||||
|
export "$key=${line#*=}"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done < "$ENV_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec /app/deploy/entrypoint.sh "$@"
|
||||||
30
deploy/stack/lb/Caddyfile
Normal file
30
deploy/stack/lb/Caddyfile
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
# The loopback LB bridge's own config — NOT the host Caddy (that one still
|
||||||
|
# terminates TLS for thermograph.org and proxies to 127.0.0.1:8137 exactly as
|
||||||
|
# before; it needs no change for the stack cutover).
|
||||||
|
#
|
||||||
|
# This Caddy runs as a PLAIN container (deploy-stack.sh manages it) because
|
||||||
|
# only plain containers can bind a specific host IP: Swarm port configs
|
||||||
|
# publish on 0.0.0.0 (routing mesh or host mode alike), which would expose
|
||||||
|
# the plaintext app un-fronted — hazard #6. It joins the stack's attachable
|
||||||
|
# overlay and proxies to the service VIPs; Swarm's VIP round-robins across
|
||||||
|
# however many web replicas the autoscaler is running, so this bridge never
|
||||||
|
# needs to know the replica count.
|
||||||
|
|
||||||
|
{
|
||||||
|
auto_https off
|
||||||
|
admin off
|
||||||
|
}
|
||||||
|
|
||||||
|
:8137 {
|
||||||
|
reverse_proxy web:8137 {
|
||||||
|
# Fail fast to the client if the VIP has no healthy task; Swarm's own
|
||||||
|
# task healthchecks handle ejecting dead replicas from the VIP.
|
||||||
|
lb_try_duration 5s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
:8080 {
|
||||||
|
reverse_proxy frontend:8080 {
|
||||||
|
lb_try_duration 5s
|
||||||
|
}
|
||||||
|
}
|
||||||
223
deploy/stack/thermograph-stack.yml
Normal file
223
deploy/stack/thermograph-stack.yml
Normal file
|
|
@ -0,0 +1,223 @@
|
||||||
|
# Docker Swarm stack for prod — the autoscaling successor to docker-compose.yml
|
||||||
|
# on that host (beta and LAN dev stay on plain compose; deploy.sh routes by the
|
||||||
|
# /etc/thermograph/deploy-mode marker). Two-image world: web/worker run the
|
||||||
|
# backend image, frontend its own — per-service tags, same contract as compose.
|
||||||
|
#
|
||||||
|
# Deployed by deploy/stack/deploy-stack.sh, which:
|
||||||
|
# - sources /etc/thermograph.env (SOPS-rendered) so ${VARS} here interpolate,
|
||||||
|
# - installs a uid-10001-readable copy at /etc/thermograph/stack.env that
|
||||||
|
# deploy/stack/env-entrypoint.sh sources inside each app task (set-if-unset,
|
||||||
|
# so `environment:` blocks below always win) — this replaces compose's
|
||||||
|
# env_file:, which `docker stack deploy` does not support,
|
||||||
|
# - runs migrations as a ONE-SHOT task before rolling (RUN_MIGRATIONS=0 in
|
||||||
|
# every replica — N replicas must never race Alembic),
|
||||||
|
# - manages the loopback LB bridge (see lb/README note below): Swarm's mesh
|
||||||
|
# can only publish on 0.0.0.0 (would expose the plaintext app un-fronted),
|
||||||
|
# so nothing here has `ports:`. A plain container on this attachable
|
||||||
|
# overlay binds 127.0.0.1:8137/8080 for the host Caddy and proxies to the
|
||||||
|
# service VIPs — Swarm's VIP does the actual load balancing across
|
||||||
|
# replicas.
|
||||||
|
#
|
||||||
|
# Scaling model: `web` is stateless (ROLE=web never runs the notifier) and
|
||||||
|
# scales 1..N — the autoscaler service adjusts replicas between
|
||||||
|
# WEB_MIN_REPLICAS/WEB_MAX_REPLICAS on task CPU. `worker` owns the notifier +
|
||||||
|
# scheduler: exactly 1 replica, with the cluster-wide Postgres advisory lock
|
||||||
|
# (THERMOGRAPH_SINGLETON_PG) as belt-and-suspenders. `db` is exactly 1 — a
|
||||||
|
# database does not scale by container count on one host; its levers are
|
||||||
|
# DB_CPUS/DB_MEMORY (and, multi-host later, Patroni replicas per the topology
|
||||||
|
# doc). Everything is pinned to the manager node: all volumes are local to
|
||||||
|
# prod today. That constraint is the ONLY thing to relax when a second app
|
||||||
|
# node joins.
|
||||||
|
#
|
||||||
|
# TIMESCALEDB_IMAGE must be the exact image (digest-pinned) the compose stack
|
||||||
|
# was running — see hazard #7 in the hop-1 runbook: a floating tag can change
|
||||||
|
# the extension minor under an existing volume. deploy-stack.sh resolves it
|
||||||
|
# from the running/last-known container automatically.
|
||||||
|
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: ${TIMESCALEDB_IMAGE:?set TIMESCALEDB_IMAGE to the exact (digest-pinned) image the volume was created with}
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: thermograph
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
|
||||||
|
POSTGRES_DB: thermograph
|
||||||
|
DB_MEMORY: ${DB_MEMORY:-16g}
|
||||||
|
volumes:
|
||||||
|
- pgdata:/var/lib/postgresql
|
||||||
|
- /opt/thermograph/deploy/db/init:/docker-entrypoint-initdb.d:ro
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U thermograph -d thermograph"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
placement:
|
||||||
|
constraints: ["node.role == manager"]
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: "${DB_CPUS:-4}"
|
||||||
|
memory: ${DB_MEMORY:-16g}
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
|
||||||
|
web:
|
||||||
|
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG}
|
||||||
|
entrypoint: ["/host/env-entrypoint.sh"]
|
||||||
|
environment:
|
||||||
|
THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph
|
||||||
|
THERMOGRAPH_BASE: /
|
||||||
|
PORT: 8137
|
||||||
|
THERMOGRAPH_SERVICE_ROLE: backend
|
||||||
|
THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://frontend:8080
|
||||||
|
WORKERS: ${WEB_WORKERS:-4}
|
||||||
|
THERMOGRAPH_DATA_DIR: /state
|
||||||
|
# web NEVER runs the notifier/scheduler, even if it would win election —
|
||||||
|
# that's the worker service's job. This is what makes web replicas safe.
|
||||||
|
THERMOGRAPH_ROLE: web
|
||||||
|
RUN_MIGRATIONS: "0"
|
||||||
|
# Overlay tasks reach the HOST's Postfix via the docker_gwbridge gateway,
|
||||||
|
# not the compose bridge's 172.19.0.1 (an overlay has no host gateway).
|
||||||
|
# provision-mail.sh's DOCKER_MAIL_GATEWAY/SUBNET cover this listener.
|
||||||
|
THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1}
|
||||||
|
volumes:
|
||||||
|
- appdata:/state
|
||||||
|
- applogs:/app/logs
|
||||||
|
- /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
|
||||||
|
- /etc/thermograph/stack.env:/host/thermograph.env:ro
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
deploy:
|
||||||
|
replicas: ${WEB_MIN_REPLICAS:-1}
|
||||||
|
placement:
|
||||||
|
constraints: ["node.role == manager"]
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: "${WEB_CPUS:-4}"
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
update_config:
|
||||||
|
# New task must pass the image's own HEALTHCHECK before the old one
|
||||||
|
# stops — zero-downtime single-service rolls.
|
||||||
|
order: start-first
|
||||||
|
failure_action: rollback
|
||||||
|
|
||||||
|
worker:
|
||||||
|
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG}
|
||||||
|
entrypoint: ["/host/env-entrypoint.sh"]
|
||||||
|
environment:
|
||||||
|
THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph
|
||||||
|
THERMOGRAPH_BASE: /
|
||||||
|
PORT: 8137
|
||||||
|
THERMOGRAPH_SERVICE_ROLE: backend
|
||||||
|
THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://frontend:8080
|
||||||
|
WORKERS: "1"
|
||||||
|
THERMOGRAPH_DATA_DIR: /state
|
||||||
|
THERMOGRAPH_ROLE: worker
|
||||||
|
# Cluster-wide Postgres advisory lock, not the host flock: correct at
|
||||||
|
# replicas=1 today and stays correct if a second worker ever appears.
|
||||||
|
THERMOGRAPH_SINGLETON_PG: "1"
|
||||||
|
RUN_MIGRATIONS: "0"
|
||||||
|
THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1}
|
||||||
|
volumes:
|
||||||
|
- appdata:/state
|
||||||
|
- applogs:/app/logs
|
||||||
|
- /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
|
||||||
|
- /etc/thermograph/stack.env:/host/thermograph.env:ro
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
placement:
|
||||||
|
constraints: ["node.role == manager"]
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: "${WORKER_CPUS:-2}"
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}:${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG}
|
||||||
|
entrypoint: ["/host/env-entrypoint.sh"]
|
||||||
|
environment:
|
||||||
|
THERMOGRAPH_BASE: /
|
||||||
|
PORT: 8080
|
||||||
|
THERMOGRAPH_SERVICE_ROLE: frontend
|
||||||
|
THERMOGRAPH_API_BASE_INTERNAL: http://web:8137
|
||||||
|
volumes:
|
||||||
|
- /opt/thermograph/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
|
||||||
|
- /etc/thermograph/stack.env:/host/thermograph.env:ro
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
placement:
|
||||||
|
constraints: ["node.role == manager"]
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: "${FRONTEND_CPUS:-2}"
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
update_config:
|
||||||
|
order: start-first
|
||||||
|
failure_action: rollback
|
||||||
|
|
||||||
|
# Scales `web` between WEB_MIN_REPLICAS and WEB_MAX_REPLICAS on sustained
|
||||||
|
# task CPU (docker stats on this node — every task is pinned here today).
|
||||||
|
# Deliberately a dumb shell loop with hysteresis + cooldown, not an
|
||||||
|
# autoscaling framework: Swarm has no native autoscaler and this workload
|
||||||
|
# doesn't justify one (topology doc §6 — declarative scaling as a feature).
|
||||||
|
autoscaler:
|
||||||
|
image: docker:27-cli
|
||||||
|
entrypoint: ["/bin/sh", "/host/autoscale.sh"]
|
||||||
|
environment:
|
||||||
|
STACK_NAME: ${STACK_NAME:-thermograph}
|
||||||
|
MIN_REPLICAS: ${WEB_MIN_REPLICAS:-1}
|
||||||
|
MAX_REPLICAS: ${WEB_MAX_REPLICAS:-3}
|
||||||
|
# Thresholds are avg docker-stats CPU% PER TASK (host-core-relative: 100
|
||||||
|
# = one full core). Up fast, down slow.
|
||||||
|
SCALE_UP_CPU: ${SCALE_UP_CPU:-220}
|
||||||
|
SCALE_DOWN_CPU: ${SCALE_DOWN_CPU:-60}
|
||||||
|
POLL_SECONDS: ${POLL_SECONDS:-15}
|
||||||
|
UP_SAMPLES: ${UP_SAMPLES:-3}
|
||||||
|
DOWN_SAMPLES: ${DOWN_SAMPLES:-20}
|
||||||
|
COOLDOWN_SECONDS: ${COOLDOWN_SECONDS:-180}
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
- /opt/thermograph/deploy/stack/autoscale.sh:/host/autoscale.sh:ro
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
deploy:
|
||||||
|
replicas: 1
|
||||||
|
placement:
|
||||||
|
constraints: ["node.role == manager"]
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: "0.2"
|
||||||
|
memory: 64m
|
||||||
|
restart_policy:
|
||||||
|
condition: any
|
||||||
|
|
||||||
|
networks:
|
||||||
|
internal:
|
||||||
|
driver: overlay
|
||||||
|
# Attachable so the loopback LB bridge (a PLAIN container — only plain
|
||||||
|
# containers can bind 127.0.0.1; Swarm port configs cannot) can join and
|
||||||
|
# reach the service VIPs.
|
||||||
|
attachable: true
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
# External and explicitly named: the real stack REUSES the volumes the
|
||||||
|
# compose stack created (same data, zero migration). deploy-stack.sh's test
|
||||||
|
# mode points these at throwaway names instead.
|
||||||
|
pgdata:
|
||||||
|
external: true
|
||||||
|
name: ${PGDATA_VOLUME:-thermograph_pgdata}
|
||||||
|
appdata:
|
||||||
|
external: true
|
||||||
|
name: ${APPDATA_VOLUME:-thermograph_appdata}
|
||||||
|
applogs:
|
||||||
|
external: true
|
||||||
|
name: ${APPLOGS_VOLUME:-thermograph_applogs}
|
||||||
Loading…
Reference in a new issue