59e7517747
Two bugs surfaced wiring dev auto-deploy: 1. Sourcing .image-tags.env clobbered the caller's incoming *_IMAGE_TAG. The first backend-only deploy persists FRONTEND_IMAGE_TAG=local (sibling unknown); the next frontend deploy then sourced that and pulled :local -> 'manifest unknown'. Now the incoming env is captured before sourcing and re-applied. 2. Health-checked services via a host-port curl, but the dev overlay leaves the frontend port unpublished (reached via the backend proxy) -> false failure. Now polls each container's own HEALTHCHECK status via docker inspect.
255 lines
12 KiB
Bash
Executable file
255 lines
12 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Pull this INFRA repo's checkout up to date, then roll ONE (or all) of the
|
|
# docker-compose app services onto its separately-published image. Run on the
|
|
# VPS — each app repo's Forgejo Actions workflow invokes this over SSH (see
|
|
# .forgejo/workflows/deploy.yml in thermograph-backend / thermograph-frontend),
|
|
# and you can run it by hand too.
|
|
#
|
|
# # roll just the backend onto a specific image:
|
|
# ssh deploy@vps 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh'
|
|
# # roll just the frontend:
|
|
# ssh deploy@vps 'SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh'
|
|
# # bring the whole stack up (both tags required):
|
|
# ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha-<a> FRONTEND_IMAGE_TAG=sha-<b> /opt/thermograph/deploy/deploy.sh'
|
|
#
|
|
# FE/BE CI-CD split: backend and frontend are published from separate repos as
|
|
# separate images (emi/thermograph-backend/app, emi/thermograph-frontend/app),
|
|
# so a deploy targets ONE service and leaves the other's running container +
|
|
# tag untouched. Each service's live tag is persisted host-side in
|
|
# deploy/.image-tags.env (untracked -- survives the git reset below) so a
|
|
# single-service roll re-renders compose with BOTH services' real tags and
|
|
# never accidentally recreates or downgrades the sibling.
|
|
#
|
|
# This checkout is thermograph-infra, not an app repo: BRANCH is this repo's
|
|
# branch (compose files, db init, the secrets vault, this script itself);
|
|
# the *_IMAGE_TAG values are the separately-published app images to run -- the
|
|
# two axes are independent and rarely change together.
|
|
set -euo pipefail
|
|
|
|
APP_DIR="${APP_DIR:-/opt/thermograph}"
|
|
BRANCH="${BRANCH:-main}"
|
|
# Which service this deploy rolls: backend | frontend | all. Defaults to `all`
|
|
# (a full-stack bring-up) so a by-hand run with both tags still works; the
|
|
# per-repo deploy.yml workflows always pass an explicit single service.
|
|
SERVICE="${SERVICE:-all}"
|
|
HEALTH_PORT="${HEALTH_PORT:-8137}"
|
|
cd "$APP_DIR"
|
|
|
|
case "$SERVICE" in
|
|
backend|frontend|all) ;;
|
|
*) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;;
|
|
esac
|
|
|
|
# Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose
|
|
# interpolation and are also loaded into the backend container via env_file.
|
|
#
|
|
# When this host is configured for SOPS (an age key + /etc/thermograph/secrets-env),
|
|
# first render /etc/thermograph.env from the committed encrypted source of truth
|
|
# (deploy/secrets/*.yaml) so a key rotation is just an edit+commit+deploy. The guard
|
|
# on the helper's existence keeps the very deploy that INTRODUCES this file safe: on
|
|
# the first pass the checkout may predate it (it arrives with the git reset below,
|
|
# after which deploy.sh re-execs), so a missing helper simply falls back to the
|
|
# existing /etc/thermograph.env. Then source it so a by-hand run interpolates the
|
|
# same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/.
|
|
if [ -f "$APP_DIR/deploy/render-secrets.sh" ]; then
|
|
# shellcheck source=deploy/render-secrets.sh
|
|
. "$APP_DIR/deploy/render-secrets.sh"
|
|
render_thermograph_secrets "$APP_DIR"
|
|
fi
|
|
set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
|
|
|
|
# Pre-warm the ~750 city-page archives so /climate pages serve from cache and a
|
|
# search-engine crawl never bursts the archive API quota. Detached inside the
|
|
# backend container (compose exec -d), idempotent (skips already-cached cells),
|
|
# so it never blocks the deploy or health check and is cheap on every deploy
|
|
# after the first full warm.
|
|
warm_city_archives() {
|
|
echo "==> Warming city-page archives in the background (backend:/app/logs/warm-cities.log)"
|
|
docker compose exec -d backend sh -c \
|
|
'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true
|
|
}
|
|
|
|
# Notify IndexNow (Bing / DuckDuckGo / Yandex) of the site's URLs, but only when
|
|
# the set of pages actually changed (a new/removed city) — code-only deploys skip.
|
|
# Best-effort: never fails the deploy.
|
|
ping_indexnow() {
|
|
echo "==> Pinging IndexNow (only if the URL set changed)"
|
|
local base="${THERMOGRAPH_BASE_URL:-https://thermograph.org}"
|
|
docker compose exec -T backend python indexnow.py --if-changed "$base" \
|
|
|| echo "!! IndexNow ping failed (non-fatal)" >&2
|
|
}
|
|
|
|
echo "==> Fetching $BRANCH"
|
|
git fetch --prune origin "$BRANCH"
|
|
git reset --hard "origin/$BRANCH"
|
|
|
|
# Re-exec: git reset --hard just rewrote this very file's bytes on disk while
|
|
# it's still running. bash reads a script via buffered, byte-offset I/O, so
|
|
# anything AFTER this point in the OLD execution can read from the wrong
|
|
# offset once the file's size/content changed underneath it -- a classic
|
|
# self-modifying-script footgun. Confirmed live: after this PR added ~15
|
|
# lines above, one deploy ran with the OLD "Building images" log lines even
|
|
# though `git status` showed the checkout correctly at the NEW commit --
|
|
# the file changed under a running interpreter, not the checkout. Restart
|
|
# fresh from the now-updated file so everything after this line is
|
|
# guaranteed self-consistent. Guarded so the second invocation doesn't
|
|
# fetch+reset+re-exec forever.
|
|
if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then
|
|
export DEPLOY_SH_REEXECED=1
|
|
exec "$0" "$@"
|
|
fi
|
|
|
|
# Registry-pull cutover: pull the image each app repo's build-push.yml already
|
|
# built and pushed, instead of building in place. This checkout is
|
|
# thermograph-infra, not an app repo, so there's no "current commit" to derive
|
|
# a tag from -- the caller (the deploying repo's deploy.yml) exports its own
|
|
# service's tag (BACKEND_IMAGE_TAG or FRONTEND_IMAGE_TAG = sha-<12 hex> of the
|
|
# app commit, or a semver tag).
|
|
REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}"
|
|
export REGISTRY_HOST BACKEND_IMAGE_PATH FRONTEND_IMAGE_PATH
|
|
|
|
# Load the last-deployed tag for BOTH services first, so a single-service roll
|
|
# still renders compose with the sibling's real, currently-running tag (never a
|
|
# bare `local` that would recreate/downgrade it). The incoming env for the
|
|
# service being deployed then overrides its line below. This file is untracked
|
|
# (see .gitignore), so `git reset --hard` above leaves it in place.
|
|
TAGS_FILE="$APP_DIR/deploy/.image-tags.env"
|
|
# Capture the tags the caller explicitly passed BEFORE sourcing -- they must
|
|
# WIN. The file only supplies the *sibling's* last-known tag; sourcing it
|
|
# unconditionally would clobber an incoming tag (e.g. a frontend deploy whose
|
|
# FRONTEND_IMAGE_TAG got overwritten by the stale `local` the first backend-only
|
|
# deploy persisted for the not-yet-known sibling -> pull `:local` -> "manifest
|
|
# unknown"). So source for the sibling, then re-apply the caller's own value.
|
|
_incoming_backend="${BACKEND_IMAGE_TAG:-}"
|
|
_incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
|
|
if [ -f "$TAGS_FILE" ]; then
|
|
set -a; . "$TAGS_FILE"; set +a
|
|
fi
|
|
[ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
|
|
[ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
|
|
|
|
# Guard: the service(s) being rolled MUST have a concrete tag supplied now (the
|
|
# sibling's may come from the persisted file). `all` needs both.
|
|
case "$SERVICE" in
|
|
backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> for a backend deploy}" ;;
|
|
frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> for a frontend deploy}" ;;
|
|
all)
|
|
: "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
|
|
: "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}"
|
|
;;
|
|
esac
|
|
# Compose interpolates both vars for the whole file even when we act on one
|
|
# service; default the not-yet-known sibling (first-ever deploy) to `local` so
|
|
# interpolation doesn't warn -- harmless since --no-deps never touches it.
|
|
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
|
|
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
|
|
|
|
# Which compose services this run pulls/rolls.
|
|
case "$SERVICE" in
|
|
backend) TARGETS=(backend) ;;
|
|
frontend) TARGETS=(frontend) ;;
|
|
all) TARGETS=(backend frontend) ;;
|
|
esac
|
|
|
|
# Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml,
|
|
# deploy-prod.yml, deploy-dev on the LAN runner) don't pass REGISTRY_TOKEN --
|
|
# the host is already `docker login`ed to the registry (persistent cred in
|
|
# ~/.docker/config.json), so an unconditional login with an empty token would
|
|
# abort the deploy under `set -e`. Use the token if present, else trust the
|
|
# host's existing cred; a genuine auth problem then fails loudly at `pull`.
|
|
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
|
echo "==> Logging in to the registry ($REGISTRY_HOST)"
|
|
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
|
else
|
|
echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST"
|
|
fi
|
|
|
|
echo "==> Pulling images (backend=$BACKEND_IMAGE_TAG frontend=$FRONTEND_IMAGE_TAG; rolling: ${TARGETS[*]})"
|
|
# Retry: build-push.yml (triggered by the same push) has no ordering guarantee
|
|
# against this deploy -- Forgejo Actions `needs:` only works between jobs in ONE
|
|
# workflow file, not across the separate build-push.yml triggered by the same
|
|
# event. Confirmed live: a deploy raced ahead of the push and failed with "not
|
|
# found". A bounded retry (~5 min) covers a normal build; a genuine problem
|
|
# (bad tag, registry down) still fails loudly after that.
|
|
pull_ok=0
|
|
for i in $(seq 1 30); do
|
|
if docker compose pull "${TARGETS[@]}"; then
|
|
pull_ok=1
|
|
break
|
|
fi
|
|
echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2
|
|
sleep 10
|
|
done
|
|
if [ "$pull_ok" != 1 ]; then
|
|
echo "!! docker compose pull failed after 30 attempts" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Roll only the target service(s). Backend schema migrations run inside its own
|
|
# entrypoint (alembic upgrade head) before uvicorn, so there's no separate
|
|
# migrate step; frontend is stateless.
|
|
#
|
|
# Single-service rolls use --no-deps so recreating backend doesn't also bounce
|
|
# db, and recreating frontend doesn't touch backend -- that independence is the
|
|
# whole point of the split. A full `all` deploy instead uses --remove-orphans,
|
|
# which matters when the service topology itself changes (a renamed-away
|
|
# service's old container would otherwise keep running and squat its port --
|
|
# confirmed live, this is what blocked beta's first dual-service deploy with
|
|
# "port is already allocated").
|
|
echo "==> Rolling ${TARGETS[*]}"
|
|
if [ "$SERVICE" = all ]; then
|
|
docker compose up -d --remove-orphans
|
|
else
|
|
docker compose up -d --no-deps "${TARGETS[@]}"
|
|
fi
|
|
|
|
# Persist the now-live tags so the next single-service deploy knows the
|
|
# sibling's real tag. Written after `up` so a failed pull never records a tag
|
|
# that isn't actually running.
|
|
mkdir -p "$(dirname "$TAGS_FILE")"
|
|
cat > "$TAGS_FILE" <<EOF
|
|
# Written by deploy.sh -- the image tag each service is currently running.
|
|
# Untracked (see .gitignore); lets a single-service deploy leave the other alone.
|
|
BACKEND_IMAGE_TAG=$BACKEND_IMAGE_TAG
|
|
FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
|
|
EOF
|
|
|
|
# Health check the service(s) we rolled: backend on 8137, frontend on 8080.
|
|
# (For `all`, backend's `/` serving is the readiness signal the old script used
|
|
# and the frontend depends_on backend anyway.)
|
|
# Health via each container's own HEALTHCHECK (docker inspect), NOT a host-port
|
|
# curl: the dev overlay leaves the frontend port UNpublished (reached through the
|
|
# backend's _proxy_to_frontend), so a localhost:8080 curl spuriously fails there.
|
|
# Both images HEALTHCHECK-curl /healthz internally, so this works published or not.
|
|
health_ok=1
|
|
for svc in "${TARGETS[@]}"; do
|
|
cid=$(docker compose ps -q "$svc" 2>/dev/null)
|
|
echo "==> Health check: $svc (container health)"
|
|
ok=0; st=unknown
|
|
for i in $(seq 1 40); do
|
|
st=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null || echo gone)
|
|
[ "$st" = healthy ] && { ok=1; break; }
|
|
if [ "$st" = none ] && [ "$(docker inspect --format '{{.State.Status}}' "$cid" 2>/dev/null)" = running ]; then ok=1; break; fi
|
|
sleep 2
|
|
done
|
|
if [ "$ok" = 1 ]; then
|
|
echo "==> OK: $svc is healthy"
|
|
else
|
|
echo "!! Health check failed for $svc (status=$st)" >&2
|
|
health_ok=0
|
|
fi
|
|
done
|
|
|
|
if [ "$health_ok" != 1 ]; then
|
|
docker compose ps || true
|
|
for svc in "${TARGETS[@]}"; do docker compose logs --tail=50 "$svc" || true; done
|
|
exit 1
|
|
fi
|
|
|
|
# Post-deploy warm/IndexNow only make sense once the backend is (re)deployed --
|
|
# they exec inside the backend container. Skip them on a frontend-only roll.
|
|
if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then
|
|
warm_city_archives
|
|
ping_indexnow
|
|
fi
|
|
exit 0
|