#!/usr/bin/env bash # Pull this INFRA repo's checkout up to date, then roll ONE (or all) of the # docker-compose app services onto its separately-published image. Run on the # VPS — each app repo's Forgejo Actions workflow invokes this over SSH (see # .forgejo/workflows/deploy.yml in thermograph-backend / thermograph-frontend), # and you can run it by hand too. # # # roll just the backend onto a specific image: # ssh deploy@vps 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh' # # roll just the frontend: # ssh deploy@vps 'SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh' # # bring the whole stack up (both tags required): # ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- /opt/thermograph/infra/deploy/deploy.sh' # # FE/BE CI-CD split: backend and frontend are published from separate repos as # separate images (emi/thermograph/backend, emi/thermograph/frontend), # so a deploy targets ONE service and leaves the other's running container + # tag untouched. Each service's live tag is persisted host-side in # deploy/.image-tags.env (untracked -- survives the git reset below) so a # single-service roll re-renders compose with BOTH services' real tags and # never accidentally recreates or downgrades the sibling. # # This checkout is the monorepo (compose + deploy live under infra/), not an app-only checkout: BRANCH is this repo's # branch (compose files, db init, the secrets vault, this script itself); # the *_IMAGE_TAG values are the separately-published app images to run -- the # two axes are independent and rarely change together. set -euo pipefail APP_DIR="${APP_DIR:-/opt/thermograph}" # Monorepo layout: git operations act on the checkout root ($APP_DIR); all # compose files and deploy assets live under infra/. INFRA_DIR="$APP_DIR/infra" BRANCH="${BRANCH:-main}" # Which service this deploy rolls: backend | frontend | all. Defaults to `all` # (a full-stack bring-up) so a by-hand run with both tags still works; the # per-repo deploy.yml workflows always pass an explicit single service. SERVICE="${SERVICE:-all}" HEALTH_PORT="${HEALTH_PORT:-8137}" cd "$APP_DIR" case "$SERVICE" in backend|frontend|all) ;; *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac # Serialize deploys on this host. Backend and frontend deploy from SEPARATE # repos whose workflows can fire for the same push within seconds of each # other, and both SSH into this one checkout: concurrent runs race on the # `git reset` below, the shared compose project, and the .image-tags.env # read/modify/write (a lost update there re-rolls the sibling onto a stale # tag). flock makes the second deploy wait its turn instead. The lock fd is # inherited across the self re-exec below, so the lock spans the whole run; # -w 600 bounds the wait (a deploy holding the lock >10 min is already # broken), after which this exits non-zero and the CI job fails loudly. DEPLOY_LOCK="$INFRA_DIR/deploy/.deploy.lock" if [ -z "${DEPLOY_SH_FLOCKED:-}" ]; then export DEPLOY_SH_FLOCKED=1 exec flock -w 600 "$DEPLOY_LOCK" "$0" "$@" fi # Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose # interpolation and are also loaded into the backend container via env_file. # # When this host is configured for SOPS (an age key + /etc/thermograph/secrets-env), # first render /etc/thermograph.env from the committed encrypted source of truth # (deploy/secrets/*.yaml) so a key rotation is just an edit+commit+deploy. The guard # on the helper's existence keeps the very deploy that INTRODUCES this file safe: on # the first pass the checkout may predate it (it arrives with the git reset below, # after which deploy.sh re-execs), so a missing helper simply falls back to the # existing /etc/thermograph.env. Then source it so a by-hand run interpolates the # same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/. if [ -f "$INFRA_DIR/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$INFRA_DIR/deploy/render-secrets.sh" render_thermograph_secrets "$INFRA_DIR" fi # /etc/thermograph.env is rendered at deploy time from the SOPS vault — it # cannot exist at lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null . /etc/thermograph.env 2>/dev/null || true set +a # Pre-warm the ~750 city-page archives so /climate pages serve from cache and a # search-engine crawl never bursts the archive API quota. Detached inside the # backend container (compose exec -d), idempotent (skips already-cached cells), # so it never blocks the deploy or health check and is cheap on every deploy # after the first full warm. warm_city_archives() { echo "==> Warming city-page archives in the background (backend:/app/logs/warm-cities.log)" docker compose exec -d backend sh -c \ 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true } # Notify IndexNow (Bing / DuckDuckGo / Yandex) of the site's URLs, but only when # the set of pages actually changed (a new/removed city) — code-only deploys skip. # Best-effort: never fails the deploy. ping_indexnow() { echo "==> Pinging IndexNow (only if the URL set changed)" local base="${THERMOGRAPH_BASE_URL:-https://thermograph.org}" docker compose exec -T backend python indexnow.py --if-changed "$base" \ || echo "!! IndexNow ping failed (non-fatal)" >&2 } echo "==> Fetching $BRANCH" git fetch --prune origin "$BRANCH" git reset --hard "origin/$BRANCH" # Re-exec: git reset --hard just rewrote this very file's bytes on disk while # it's still running. bash reads a script via buffered, byte-offset I/O, so # anything AFTER this point in the OLD execution can read from the wrong # offset once the file's size/content changed underneath it -- a classic # self-modifying-script footgun. Confirmed live: after this PR added ~15 # lines above, one deploy ran with the OLD "Building images" log lines even # though `git status` showed the checkout correctly at the NEW commit -- # the file changed under a running interpreter, not the checkout. Restart # fresh from the now-updated file so everything after this line is # guaranteed self-consistent. Guarded so the second invocation doesn't # fetch+reset+re-exec forever. if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then export DEPLOY_SH_REEXECED=1 exec "$0" "$@" fi # Stack-mode routing: a host whose /etc/thermograph/deploy-mode says "stack" # (prod, after the Swarm cutover) deploys via the Swarm stack path instead of # compose. Checked AFTER the reset+re-exec so the stack script is always the # freshly-pulled one, and the SERVICE/tag contract passes through unchanged -- # the app repos' workflows never need to know which mode a host runs. if [ "$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)" = "stack" ]; then exec bash "$INFRA_DIR/deploy/stack/deploy-stack.sh" fi # Monorepo: run every docker compose command from infra/ (the git operations # above ran at the checkout root). The compose project name is pinned in the # file itself (`name: thermograph`), so moving the working directory does NOT # rename the project and recreate the stack under a second name. cd "$INFRA_DIR" # Registry-pull cutover: pull the image each app repo's build-push.yml already # built and pushed, instead of building in place. This checkout is # thermograph-infra, not an app repo, so there's no "current commit" to derive # a tag from -- the caller (the deploying repo's deploy.yml) exports its own # service's tag (BACKEND_IMAGE_TAG or FRONTEND_IMAGE_TAG = sha-<12 hex> of the # app commit, or a semver tag). REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}" export REGISTRY_HOST BACKEND_IMAGE_PATH FRONTEND_IMAGE_PATH # Load the last-deployed tag for BOTH services first, so a single-service roll # still renders compose with the sibling's real, currently-running tag (never a # bare `local` that would recreate/downgrade it). The incoming env for the # service being deployed then overrides its line below. This file is untracked # (see .gitignore), so `git reset --hard` above leaves it in place. TAGS_FILE="$INFRA_DIR/deploy/.image-tags.env" # Capture the tags the caller explicitly passed BEFORE sourcing -- they must # WIN. The file only supplies the *sibling's* last-known tag; sourcing it # unconditionally would clobber an incoming tag (e.g. a frontend deploy whose # FRONTEND_IMAGE_TAG got overwritten by the stale `local` the first backend-only # deploy persisted for the not-yet-known sibling -> pull `:local` -> "manifest # unknown"). So source for the sibling, then re-apply the caller's own value. _incoming_backend="${BACKEND_IMAGE_TAG:-}" _incoming_frontend="${FRONTEND_IMAGE_TAG:-}" if [ -f "$TAGS_FILE" ]; then set -a # shellcheck source=/dev/null # untracked runtime artifact this script writes below . "$TAGS_FILE" set +a fi [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend" [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend" # Guard: the service(s) being rolled MUST have a concrete tag supplied now (the # sibling's may come from the persisted file). `all` needs both. case "$SERVICE" in backend) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> for a backend deploy}" ;; frontend) : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> for a frontend deploy}" ;; all) : "${BACKEND_IMAGE_TAG:?set BACKEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}" : "${FRONTEND_IMAGE_TAG:?set FRONTEND_IMAGE_TAG=sha-<12-hex> (SERVICE=all needs both)}" ;; esac # Compose interpolates both vars for the whole file even when we act on one # service; default the not-yet-known sibling (first-ever deploy) to `local` so # interpolation doesn't warn -- harmless since --no-deps never touches it. export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" # Which compose services this run pulls/rolls. # # `daemon` and `lake` ride with `backend` and are never targets on their own. # Both run the SAME image at the SAME tag (a second binary and a second role # inside the backend image): the daemon talks to the web process over the # /internal/* contract, the lake serves it history — rolling one without the # other is exactly the version skew those contracts have no negotiation for. # Pairing them here is also what makes the services exist at all: a # single-service deploy runs `up -d --no-deps `, so a service listed # only in compose but absent from TARGETS would never be created on a # backend-only deploy. case "$SERVICE" in backend) TARGETS=(backend lake daemon) ;; frontend) TARGETS=(frontend) ;; all) TARGETS=(backend lake frontend daemon) ;; esac # Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml, # deploy-prod.yml, deploy-dev on the LAN runner) don't pass REGISTRY_TOKEN -- # the host is already `docker login`ed to the registry (persistent cred in # ~/.docker/config.json), so an unconditional login with an empty token would # abort the deploy under `set -e`. Use the token if present, else trust the # host's existing cred; a genuine auth problem then fails loudly at `pull`. if [ -n "${REGISTRY_TOKEN:-}" ]; then echo "==> Logging in to the registry ($REGISTRY_HOST)" echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin else echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST" fi echo "==> Pulling images (backend=$BACKEND_IMAGE_TAG frontend=$FRONTEND_IMAGE_TAG; rolling: ${TARGETS[*]})" # Retry: build-push.yml (triggered by the same push) has no ordering guarantee # against this deploy -- Forgejo Actions `needs:` only works between jobs in ONE # workflow file, not across the separate build-push.yml triggered by the same # event. Confirmed live: a deploy raced ahead of the push and failed with "not # found". A bounded retry (~5 min) covers a normal build; a genuine problem # (bad tag, registry down) still fails loudly after that. pull_ok=0 for i in $(seq 1 30); do if docker compose pull "${TARGETS[@]}"; then pull_ok=1 break fi echo " pull attempt $i/30 failed (image may not be pushed yet); retrying in 10s..." >&2 sleep 10 done if [ "$pull_ok" != 1 ]; then echo "!! docker compose pull failed after 30 attempts" >&2 exit 1 fi # The daemon binary ships INSIDE the backend image, but infra and app images # advance on different axes by design: this checkout tracks `main`, while image # tags are env-staged (prod's come from `release`). So a host can legitimately # be asked to roll a backend image OLDER than this compose file -- one built # before the daemon existed and with no /usr/local/bin/thermograph-daemon in it. # Creating the service anyway would leave a container crash-looping on a missing # binary, on a deploy that otherwise succeeded. Probe the image we are actually # about to roll and drop the daemon from this run if it can't support it; the # next deploy of a tag that has it picks it up with no further action. case " ${TARGETS[*]} " in *" daemon "*) # Built directly from the same vars docker-compose.yml's `daemon.image:` # interpolates (REGISTRY_HOST/BACKEND_IMAGE_PATH/BACKEND_IMAGE_TAG), # NOT via `docker compose config --images daemon`: that command does not # actually filter to the named service (confirmed live on Compose # v5.3.1 -- it prints every service's image, one per line, in file # order) so `| head -1` silently grabbed db's image instead. The probe # then always found no daemon binary in a Postgres image and dropped # daemon from EVERY backend deploy, regardless of what the real backend # image contained -- reproduced and confirmed against beta directly. daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG}" if ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run" kept=() for t in "${TARGETS[@]}"; do [ "$t" = daemon ] || kept+=("$t") done TARGETS=("${kept[@]}") fi ;; esac # Roll only the target service(s). Backend schema migrations run inside its own # entrypoint (alembic upgrade head) before uvicorn, so there's no separate # migrate step; frontend is stateless. # # Single-service rolls use --no-deps so recreating backend doesn't also bounce # db, and recreating frontend doesn't touch backend -- that independence is the # whole point of the split. A full `all` deploy instead uses --remove-orphans, # which matters when the service topology itself changes (a renamed-away # service's old container would otherwise keep running and squat its port -- # confirmed live, this is what blocked beta's first dual-service deploy with # "port is already allocated"). echo "==> Rolling ${TARGETS[*]}" if [ "$SERVICE" = all ]; then docker compose up -d --remove-orphans else docker compose up -d --no-deps "${TARGETS[@]}" fi # Persist the now-live tags so the next single-service deploy knows the # sibling's real tag. Written after `up` so a failed pull never records a tag # that isn't actually running. mkdir -p "$(dirname "$TAGS_FILE")" cat > "$TAGS_FILE" </dev/null) echo "==> Health check: $svc (container health)" ok=0; st=unknown for i in $(seq 1 40); do st=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null || echo gone) [ "$st" = healthy ] && { ok=1; break; } if [ "$st" = none ] && [ "$(docker inspect --format '{{.State.Status}}' "$cid" 2>/dev/null)" = running ]; then ok=1; break; fi sleep 2 done if [ "$ok" = 1 ]; then echo "==> OK: $svc is healthy" else echo "!! Health check failed for $svc (status=$st)" >&2 health_ok=0 fi done if [ "$health_ok" != 1 ]; then docker compose ps || true for svc in "${TARGETS[@]}"; do docker compose logs --tail=50 "$svc" || true; done exit 1 fi # Every deploy pulls a new sha-tagged image and nothing ever removed the old # ones -- hosts accumulate gigabytes of dead tags at one per app commit. Keep # only the tags recorded as now-live (both services) and delete other tags of # the two app-image repos. Best-effort and after the health gate, so a failed # roll never garbage-collects the image a rollback would need; docker also # refuses to remove an image any container still uses. echo "==> Pruning old app-image tags" _be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}" _fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}" docker images --format '{{.Repository}}:{{.Tag}}' \ | grep -E "^(${_be_repo}|${_fe_repo}):" \ | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \ | xargs -r docker rmi 2>/dev/null || true # Post-deploy warm/IndexNow only make sense once the backend is (re)deployed -- # they exec inside the backend container. Skip them on a frontend-only roll. if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then warm_city_archives ping_indexnow fi exit 0