From e44d1603b6fa247bb2dd0acf85e734f6b9b33434 Mon Sep 17 00:00:00 2001 From: emi Date: Wed, 22 Jul 2026 23:39:29 +0000 Subject: [PATCH] deploy.sh: serialize concurrent deploys with flock; GC old app-image tags (#6) --- .gitignore | 8 ++++++++ deploy/deploy.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.gitignore b/.gitignore index 2a051c8..6347e64 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,11 @@ age.key *.age.key .DS_Store + +# Host-side deploy state (deploy.sh): the live per-service image tags and the +# cross-repo deploy lock. Untracked on purpose -- they must survive the +# `git reset --hard` at the top of every deploy (deploy.sh's comments already +# assumed .image-tags.env was ignored; make it actually true so a stray +# `git clean` can't destroy the record of what's running). +deploy/.image-tags.env +deploy/.deploy.lock diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 87295b1..6b03b35 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -40,6 +40,21 @@ case "$SERVICE" in *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac +# Serialize deploys on this host. Backend and frontend deploy from SEPARATE +# repos whose workflows can fire for the same push within seconds of each +# other, and both SSH into this one checkout: concurrent runs race on the +# `git reset` below, the shared compose project, and the .image-tags.env +# read/modify/write (a lost update there re-rolls the sibling onto a stale +# tag). flock makes the second deploy wait its turn instead. The lock fd is +# inherited across the self re-exec below, so the lock spans the whole run; +# -w 600 bounds the wait (a deploy holding the lock >10 min is already +# broken), after which this exits non-zero and the CI job fails loudly. +DEPLOY_LOCK="$APP_DIR/deploy/.deploy.lock" +if [ -z "${DEPLOY_SH_FLOCKED:-}" ]; then + export DEPLOY_SH_FLOCKED=1 + exec flock -w 600 "$DEPLOY_LOCK" "$0" "$@" +fi + # Secrets (POSTGRES_PASSWORD, VAPID keys, AUTH_SECRET, ...) drive compose # interpolation and are also loaded into the backend container via env_file. # @@ -246,6 +261,20 @@ if [ "$health_ok" != 1 ]; then exit 1 fi +# Every deploy pulls a new sha-tagged image and nothing ever removed the old +# ones -- hosts accumulate gigabytes of dead tags at one per app commit. Keep +# only the tags recorded as now-live (both services) and delete other tags of +# the two app-image repos. Best-effort and after the health gate, so a failed +# roll never garbage-collects the image a rollback would need; docker also +# refuses to remove an image any container still uses. +echo "==> Pruning old app-image tags" +_be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}" +_fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-emi/thermograph-frontend/app}" +docker images --format '{{.Repository}}:{{.Tag}}' \ + | grep -E "^(${_be_repo}|${_fe_repo}):" \ + | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \ + | xargs -r docker rmi 2>/dev/null || true + # Post-deploy warm/IndexNow only make sense once the backend is (re)deployed -- # they exec inside the backend container. Skip them on a frontend-only roll. if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then