From dde342f01b63265be5667309de39db3bc183a9a0 Mon Sep 17 00:00:00 2001 From: emi Date: Wed, 22 Jul 2026 23:26:35 +0000 Subject: [PATCH] Port the fixed ops-cron: prod backups died with the app-repo archive (#3) --- .forgejo/workflows/ops-cron.yml | 42 ++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/.forgejo/workflows/ops-cron.yml b/.forgejo/workflows/ops-cron.yml index 9302e7c..c641a3e 100644 --- a/.forgejo/workflows/ops-cron.yml +++ b/.forgejo/workflows/ops-cron.yml @@ -10,11 +10,15 @@ name: Ops cron (backup + IndexNow) # stack (docker compose exec), the same way deploy.sh already runs its own # post-deploy IndexNow ping -- no new network exposure, no separate dependency # install. Runs on the `docker` label (the always-on Swarm-hosted runner -# deploy/forgejo/ stood up) and reuses the same SSH secrets deploy.yml already -# has (SSH_HOST/SSH_USER/SSH_KEY/SSH_PORT) -- inherits, and will automatically -# benefit from a fix to, deploy.yml's own flagged branch/host mismatch (that -# workflow's header comment: it still targets `main`, while -# terraform.tfvars.example names prod's branch `release`). +# deploy/forgejo/ stood up). +# +# Targets PROD via the PROD_SSH_* secrets (prod = 169.58.46.181, `agent` user, +# in the docker group so no sudo needed; /etc/thermograph.env is agent-readable). +# These are the same secrets deploy-prod.yml uses for the release->prod deploy -- +# NOT the SSH_* secrets, which point at BETA (deploy.yml's `main`->beta path). An +# earlier revision reused SSH_* here, so the "prod" backup was silently dumping +# beta; prod itself had no backup at all. The prod database is the one that must +# be backed up, so both jobs use PROD_SSH_*. on: schedule: @@ -38,23 +42,33 @@ jobs: - name: Dump the prod database over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} + host: ${{ secrets.PROD_SSH_HOST }} + username: ${{ secrets.PROD_SSH_USER }} + key: ${{ secrets.PROD_SSH_KEY }} + port: ${{ secrets.PROD_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph + # Source the env so `docker compose` can interpolate POSTGRES_PASSWORD; + # without it compose fails to parse docker-compose.yml, the redirect + # still creates the target, and the job leaves a 0-byte .dump and exits + # non-zero (exactly how this backup was silently failing). Mirrors the + # IndexNow job below, which already sources it. + set -a; . /etc/thermograph.env 2>/dev/null || true; set +a backup_dir="$HOME/thermograph-backups" mkdir -p "$backup_dir" stamp="$(date -u +%Y%m%dT%H%M%SZ)" out="$backup_dir/thermograph-$stamp.dump" + # Write to a .partial and rename on success so a mid-dump failure can + # never leave a truncated file that looks like a good backup. docker compose exec -T db pg_dump -U thermograph -d thermograph \ - --format=custom > "$out" + --format=custom > "$out.partial" + mv "$out.partial" "$out" echo "wrote $out ($(du -h "$out" | cut -f1))" # The dumps are the disaster-recovery copy, not a versioned # archive -- keep the last 14 days and let the rest age out. find "$backup_dir" -name 'thermograph-*.dump' -mtime +14 -delete + find "$backup_dir" -name 'thermograph-*.dump.partial' -mtime +1 -delete indexnow: name: IndexNow ping @@ -69,10 +83,10 @@ jobs: - name: Ping IndexNow if the URL set changed uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} + host: ${{ secrets.PROD_SSH_HOST }} + username: ${{ secrets.PROD_SSH_USER }} + key: ${{ secrets.PROD_SSH_KEY }} + port: ${{ secrets.PROD_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph