233 lines
11 KiB
Bash
Executable file
233 lines
11 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Swarm-stack deploy for prod — the stack-mode counterpart of deploy/deploy.sh,
|
|
# speaking the SAME contract the app repos' workflows already use
|
|
# (SERVICE=backend|frontend|all + BACKEND_IMAGE_TAG/FRONTEND_IMAGE_TAG), so
|
|
# switching a host to stack mode needs no workflow changes: deploy.sh execs
|
|
# this when /etc/thermograph/deploy-mode contains "stack".
|
|
#
|
|
# What a roll does here vs compose:
|
|
# backend -> one-shot migrate, then `docker service update --image` on
|
|
# web AND worker (same image; start-first, health-gated,
|
|
# auto-rollback on failure).
|
|
# frontend -> `docker service update --image` on frontend.
|
|
# all -> full `docker stack deploy` (+ migrate first), which also
|
|
# applies stack-file changes (new services, env, limits).
|
|
#
|
|
# TEST MODE (STACK_TEST=1): deploys under stack name thermograph-test with
|
|
# throwaway volumes and the LB on 127.0.0.1:18137/18080 — a full parallel
|
|
# rehearsal on the same host that cannot touch live data or ports.
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thermograph}"
|
|
SERVICE="${SERVICE:-all}"
|
|
cd "$APP_DIR"
|
|
|
|
case "$SERVICE" in
|
|
backend|frontend|all) ;;
|
|
*) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
|
|
esac
|
|
|
|
if [ "${STACK_TEST:-0}" = "1" ]; then
|
|
STACK_NAME="thermograph-test"
|
|
LB_NAME="thermograph-test-lb"
|
|
LB_HTTP_PORT=18137; LB_FE_PORT=18080
|
|
export PGDATA_VOLUME="thermograph-test_pgdata"
|
|
export APPDATA_VOLUME="thermograph-test_appdata"
|
|
export APPLOGS_VOLUME="thermograph-test_applogs"
|
|
docker volume create "$PGDATA_VOLUME" >/dev/null
|
|
docker volume create "$APPDATA_VOLUME" >/dev/null
|
|
docker volume create "$APPLOGS_VOLUME" >/dev/null
|
|
else
|
|
STACK_NAME="${STACK_NAME:-thermograph}"
|
|
LB_NAME="thermograph-lb"
|
|
LB_HTTP_PORT=8137; LB_FE_PORT=8080
|
|
fi
|
|
export STACK_NAME
|
|
|
|
# --- secrets ------------------------------------------------------------------
|
|
# Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then
|
|
# install the uid-10001-readable copy the tasks' env-entrypoint shim sources.
|
|
# 10001 = the app images' `thermograph` user; the file is 0400 to that uid.
|
|
if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then
|
|
# shellcheck source=deploy/render-secrets.sh
|
|
. "$APP_DIR/infra/deploy/render-secrets.sh"
|
|
render_thermograph_secrets "$APP_DIR/infra"
|
|
fi
|
|
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
|
|
sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \
|
|
|| install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env
|
|
|
|
# --- image tags -----------------------------------------------------------------
|
|
# Same persisted-tags contract as deploy.sh: incoming env wins, the file
|
|
# supplies the sibling. Stack mode keeps its own file so test/real never mix.
|
|
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
|
|
export REGISTRY_HOST
|
|
TAGS_FILE="$APP_DIR/infra/deploy/.stack-image-tags.env"
|
|
_incoming_backend="${BACKEND_IMAGE_TAG:-}"
|
|
_incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
|
|
if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi
|
|
[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
|
|
[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
|
|
case "$SERVICE" in
|
|
backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12hex>}" ;;
|
|
frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12hex>}" ;;
|
|
all)
|
|
: "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG (SERVICE=all needs both)}"
|
|
: "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG (SERVICE=all needs both)}" ;;
|
|
esac
|
|
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
|
|
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
|
|
BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:$BACKEND_IMAGE_TAG"
|
|
FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:$FRONTEND_IMAGE_TAG"
|
|
|
|
# --- timescale image pin ---------------------------------------------------------
|
|
# Hazard #7: the db image under an existing volume must never drift. Resolve
|
|
# the digest-pinned ref from whatever is running (stack task or compose
|
|
# container), falling back to the local latest-pg18's digest on first bring-up.
|
|
if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then
|
|
cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
|
|
[ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1)
|
|
if [ -n "$cid" ]; then
|
|
img=$(docker inspect --format '{{.Image}}' "$cid")
|
|
else
|
|
img="timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18}"
|
|
fi
|
|
TIMESCALEDB_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null | head -1)
|
|
[ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img"
|
|
fi
|
|
export TIMESCALEDB_IMAGE
|
|
echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE"
|
|
|
|
# --- registry ------------------------------------------------------------------
|
|
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
|
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
|
fi
|
|
echo "==> Pulling images"
|
|
pull_ok=0
|
|
for i in $(seq 1 30); do
|
|
if docker pull -q "$BACKEND_IMAGE" >/dev/null && docker pull -q "$FRONTEND_IMAGE" >/dev/null; then
|
|
pull_ok=1; break
|
|
fi
|
|
echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
|
|
sleep 10
|
|
done
|
|
[ "$pull_ok" = 1 ] || { echo "!! image pull failed after 30 attempts" >&2; exit 1; }
|
|
|
|
# --- one-shot migrations ---------------------------------------------------------
|
|
# Before any backend roll: N replicas must never race Alembic (RUN_MIGRATIONS=0
|
|
# in the stack). Runs on the stack's overlay so `db` resolves. First-ever
|
|
# deploy: the network doesn't exist yet — create it exactly as the stack will
|
|
# (attachable overlay) so the name is adopted, then migrate, then deploy.
|
|
NET="${STACK_NAME}_internal"
|
|
# Never pre-create $NET: docker stack deploy must own it (a pre-existing
|
|
# unlabeled network makes it fail with "already exists"). On first deploy the
|
|
# migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE below).
|
|
if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then
|
|
if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then
|
|
echo "==> One-shot migrate ($BACKEND_IMAGE)"
|
|
docker run --rm --network "$NET" \
|
|
-e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \
|
|
--entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
|
|
else
|
|
echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below"
|
|
fi
|
|
fi
|
|
|
|
# --- deploy ----------------------------------------------------------------------
|
|
FIRST_DEPLOY_MIGRATE=0
|
|
docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1
|
|
if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then
|
|
echo "==> docker stack deploy ($STACK_NAME)"
|
|
docker stack deploy --with-registry-auth -c "$APP_DIR/infra/deploy/stack/thermograph-stack.yml" "$STACK_NAME"
|
|
# First-ever deploy ran no migrate above (db didn't exist): wait for db,
|
|
# migrate, then force web/worker to restart cleanly against the schema.
|
|
if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then
|
|
echo "==> Waiting for db, then first-boot migrate"
|
|
for i in $(seq 1 60); do
|
|
cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1)
|
|
[ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break
|
|
sleep 5
|
|
done
|
|
docker run --rm --network "$NET" \
|
|
-e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \
|
|
--entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate
|
|
docker service update --force --detach=false "${STACK_NAME}_web"
|
|
docker service update --force --detach=false "${STACK_NAME}_worker"
|
|
fi
|
|
else
|
|
case "$SERVICE" in
|
|
backend)
|
|
echo "==> Rolling web + worker + lake to $BACKEND_IMAGE"
|
|
docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web"
|
|
docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker"
|
|
# lake ships in the same image; a stack file predating it has no service
|
|
# yet — the next SERVICE=all stack deploy creates it, so don't fail here.
|
|
if docker service inspect "${STACK_NAME}_lake" >/dev/null 2>&1; then
|
|
docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_lake"
|
|
else
|
|
echo " (no ${STACK_NAME}_lake service yet; created on the next full stack deploy)"
|
|
fi
|
|
;;
|
|
frontend)
|
|
echo "==> Rolling frontend to $FRONTEND_IMAGE"
|
|
docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend"
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
# --- loopback LB bridge -----------------------------------------------------------
|
|
# A PLAIN container (only plain containers can bind 127.0.0.1; Swarm publishes
|
|
# 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated
|
|
# only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME`
|
|
# to force a refresh after editing lb/Caddyfile.
|
|
if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then
|
|
docker rm -f "$LB_NAME" >/dev/null 2>&1 || true
|
|
echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)"
|
|
docker run -d --name "$LB_NAME" --restart unless-stopped \
|
|
--network "$NET" \
|
|
-p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \
|
|
-v "$APP_DIR/infra/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \
|
|
caddy:2-alpine >/dev/null
|
|
fi
|
|
|
|
# --- verify -----------------------------------------------------------------------
|
|
echo "==> Health check via the LB"
|
|
ok=0
|
|
for i in $(seq 1 60); do
|
|
if curl -fsS -m 3 -o /dev/null "http://127.0.0.1:${LB_HTTP_PORT}/healthz"; then ok=1; break; fi
|
|
sleep 2
|
|
done
|
|
if [ "$ok" != 1 ]; then
|
|
echo "!! stack health check failed" >&2
|
|
docker stack ps "$STACK_NAME" --no-trunc | head -20
|
|
exit 1
|
|
fi
|
|
echo "==> OK: $STACK_NAME serving on 127.0.0.1:${LB_HTTP_PORT}"
|
|
|
|
# Persist now-live tags (after health, like deploy.sh).
|
|
cat > "$TAGS_FILE" <<EOF
|
|
# Written by deploy-stack.sh -- the image tag each service is currently running.
|
|
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
|
|
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
|
|
EOF
|
|
|
|
# GC superseded app-image tags (keep the running pair), same as deploy.sh.
|
|
_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}"
|
|
_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}"
|
|
docker images --format '{{.Repository}}:{{.Tag}}' \
|
|
| grep -E "^(${_be_repo}|${_fe_repo}):" \
|
|
| grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
|
|
| xargs -r docker rmi 2>/dev/null || true
|
|
|
|
# Post-deploy warm + IndexNow, via any web task (skip in test mode: no data,
|
|
# and the warmer would burn upstream quota against an empty cache).
|
|
if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then
|
|
wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1)
|
|
if [ -n "$wcid" ]; then
|
|
echo "==> Warming city archives (detached) + IndexNow"
|
|
docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
|
|
docker exec "$wcid" python indexnow.py --if-changed "${THERMOGRAPH_BASE_URL:-https://thermograph.org}" \
|
|
|| echo "!! IndexNow ping failed (non-fatal)" >&2
|
|
fi
|
|
fi
|
|
exit 0
|