name: Ops cron (backup + IndexNow) # Scheduled operational jobs that don't belong in the app's own worker-tier # scheduler (notifications/scheduler.py, Track A chunk 5) because they're # infra/ops concerns rather than app-domain background work -- see the job # classification table in # thermograph-docs/architecture/repo-topology-and-infrastructure.md §7. # # The jobs SSH into a host and run inside the already-running stack (docker # exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new # network exposure, no separate dependency install. Runs on the `docker` label, # served by the vps2 runner (infra/deploy/forgejo/runner-vps2/) and, when it is # up, the desktop. # # THIS WORKFLOW IS THE ESTATE'S ONLY BACKUP, so its dependency on a runner is # load-bearing. On 2026-07-31 the desktop was the only registered runner and # went offline for 21 hours; this schedule did not fire at 03:00Z. Forgejo held # the run in `waiting` and it completed on reconnect, so nothing was lost that # time. The vps2 runner exists so the next outage is not a coin flip. # # WHICH BOX EACH JOB TALKS TO, and why the secret names say so: # # backup, indexnow -> VPS2_SSH_* (vps2 = 169.58.46.181: prod AND beta, the # shared TimescaleDB, Postfix, the backups) # forgejo-backup -> VPS1_SSH_* (vps1 = 75.119.132.91: Forgejo, Grafana, # Loki, and the dev environment) # # The secrets are keyed by HOST rather than by environment because an # environment no longer implies a machine: vps2 runs two of them. The previous # names encoded the opposite assumption and had already caused one incident -- # an early revision used SSH_* here, which meant "beta", so the job labelled # "prod backup" was silently dumping beta while prod had no backup at all. The # same conflation is why Forgejo's backup used to be described as running "on # beta": beta and Forgejo happened to share a box, so one secret served both # meanings. It does not any more. # # BOTH APPLICATION DATABASES ARE BACKED UP. prod and beta are separate databases # (`thermograph` and `thermograph_beta`) on one shared instance. Dumping only # the prod database would recreate the original failure in a new shape: a job # that looks like "the backup" while one environment's data is silently # uncovered. The loop below dumps each, to its own off-box prefix. # Monorepo port: the compose file now lives under infra/ of the host's # /opt/thermograph monorepo checkout, so every docker-compose invocation cd's # into /opt/thermograph/infra (compose also pins `name: thermograph` so the # project name no longer depends on the directory). Everything else unchanged. on: schedule: # 03:00 UTC daily -- a low-traffic window for both jobs. - cron: '0 3 * * *' workflow_dispatch: {} jobs: backup: name: pg_dump backup runs-on: docker # Guards against the schedule and a manual workflow_dispatch landing # close together (or two manual triggers): queue rather than cancel, so # a workflow_dispatch never aborts a pg_dump mid-write and leaves a # truncated .dump file -- same cancel-in-progress:false rationale as # deploy.yml/deploy-dev.yml's own SSH-script jobs. concurrency: group: ops-backup cancel-in-progress: false steps: - name: Dump both application databases over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed # into the remote script via `envs:` (a host env file isn't a reliable # source across boxes -- on vps1 it isn't readable by the deploy user -- # so the CI secret is the uniform home, like the VPS*_SSH_* pairs). env: S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} S3_BUCKET: ${{ secrets.S3_BUCKET }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: host: ${{ secrets.VPS2_SSH_HOST }} username: ${{ secrets.VPS2_SSH_USER }} key: ${{ secrets.VPS2_SSH_KEY }} port: ${{ secrets.VPS2_SSH_PORT }} envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail cd /opt/thermograph/infra # Source the env so `docker compose` can interpolate POSTGRES_PASSWORD; # without it compose fails to parse docker-compose.yml, the redirect # still creates the target, and the job leaves a 0-byte .dump and exits # non-zero (exactly how this backup was silently failing). Mirrors the # IndexNow job below, which already sources it. set -a; . /etc/thermograph.env 2>/dev/null || true; set +a backup_dir="$HOME/thermograph-backups" mkdir -p "$backup_dir" stamp="$(date -u +%Y%m%dT%H%M%SZ)" # The db may run under plain compose OR as a Swarm stack task; # resolve the container either way so the backup survives a # deploy-mode switch. One instance now serves both environments. dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1) [ -z "$dbc" ] && dbc=$(cd /opt/thermograph/infra && docker compose ps -q db 2>/dev/null | head -1) [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; } # env:database pairs. Both are dumped every night: they are separate # databases on one instance, and backing up only one would leave the # other silently uncovered. for pair in prod:thermograph beta:thermograph_beta; do envname="${pair%%:*}"; dbname="${pair##*:}" # A missing database is a HARD failure, never a quiet skip -- a # backup job that shrugs off an absent database is exactly how an # environment ends up with no backups and nobody noticing. if ! docker exec "$dbc" psql -U thermograph -d postgres -tAc \ "select 1 from pg_database where datname='$dbname'" | grep -q 1; then echo "!! database '$dbname' ($envname) does not exist on this instance." if [ "$envname" = prod ]; then echo "!! That is prod's own database on prod's own instance — this is not a" echo "!! provisioning gap, something is badly wrong. Do not 'fix' it by creating" echo "!! an empty database; find out where the real one went." else echo "!! provision it first: sudo bash /opt/thermograph-beta/infra/deploy/db/provision-env-db.sh $envname" fi exit 1 fi out="$backup_dir/$dbname-$stamp.dump" # Write to a .partial and rename on success so a mid-dump failure can # never leave a truncated file that looks like a good backup. docker exec "$dbc" pg_dump -U thermograph -d "$dbname" \ --format=custom > "$out.partial" mv "$out.partial" "$out" echo "wrote $out ($(du -h "$out" | cut -f1))" dumps="${dumps:-} $envname:$out" done # The dumps are the disaster-recovery copy, not a versioned # archive -- keep the last 14 days and let the rest age out. find "$backup_dir" -name '*.dump' -mtime +14 -delete find "$backup_dir" -name '*.dump.partial' -mtime +1 -delete # --- off-box copy to Contabo Object Storage (age-encrypted) --- # Streams the just-written dump through age (to the vault's age # recipient -- the same key each host renders secrets with, so # /etc/thermograph/age.key decrypts it for restore) straight to S3. No # local intermediate, no plaintext leaves the box. Skips cleanly if the # S3 secrets aren't set. Keeps 30 days of encrypted dumps off-box. if [ -n "${S3_ACCESS_KEY:-}" ]; then command -v rclone >/dev/null 2>&1 || sudo apt-get install -y -qq rclone command -v age >/dev/null 2>&1 || sudo apt-get install -y -qq age export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \ RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \ RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 # One prefix per environment, so a restore never has to guess which # database a dump came from: backups/db/prod/ and backups/db/beta/. for entry in $dumps; do envname="${entry%%:*}"; f="${entry##*:}" s3base="$S3_BUCKET/backups/db/$envname" age -r "$recip" < "$f" | rclone rcat "archive:$s3base/$(basename "$f").age" echo "off-box: uploaded $(basename "$f").age to $s3base" rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true done else echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)" fi forgejo-backup: name: Forgejo backup (vps1) -> S3 runs-on: docker # Forgejo (the git host + all CI history) lives on vps1 and had NO backup at # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS, # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as # vps1's SSH user (VPS1_SSH_*), which is in the docker group -- so docker # exec/run need no host sudo, and the data tar runs inside a throwaway # container (the volume dir is root-owned on the host). rclone + age must be # present on vps1. # # This job follows FORGEJO, not beta. It used to use the same secret as the # beta deploy purely because Forgejo and beta shared a box; beta has since # moved to vps2 and Forgejo has not moved at all. concurrency: group: ops-forgejo-backup cancel-in-progress: false steps: - name: Back up Forgejo (db + data volume) over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 env: S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} S3_BUCKET: ${{ secrets.S3_BUCKET }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: host: ${{ secrets.VPS1_SSH_HOST }} username: ${{ secrets.VPS1_SSH_USER }} key: ${{ secrets.VPS1_SSH_KEY }} port: ${{ secrets.VPS1_SSH_PORT }} envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; } command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on vps1"; exit 1; } command -v age >/dev/null 2>&1 || { echo "!! age missing on vps1"; exit 1; } export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \ RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \ RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 stamp="$(date -u +%Y%m%dT%H%M%SZ)" base="$S3_BUCKET/backups/forgejo" fdb=$(docker ps -qf name=forgejo_db | head -1) [ -n "$fdb" ] || { echo "!! no forgejo_db container"; exit 1; } # 1) Forgejo Postgres (consistent dump); creds read inside the container docker exec "$fdb" sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \ | age -r "$recip" | rclone rcat "archive:$base/db/forgejo-$stamp.dump.age" # 2) Forgejo data volume via a throwaway container (no host sudo needed) docker run --rm -v forgejo_forgejo_data:/data:ro busybox tar -C /data -cf - . \ | age -r "$recip" | rclone rcat "archive:$base/data/forgejo-data-$stamp.tar.age" echo "forgejo backup uploaded ($stamp)" rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true secrets-parity: name: OpenBao/SOPS parity gate runs-on: docker # The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive # green parity runs" -- and which, until now, nothing implemented. A gate # that exists only in prose gets satisfied by assertion. # # verify-parity.sh renders each environment through BOTH backends and diffs # the KEY=value sets after a last-wins collapse. It prints key NAMES and # counts only, never values, so these logs are safe to read and to paste. # # A mismatch FAILS the job, deliberately and loudly. The failure modes on # the other side of a bad cutover are all silent: a missing # THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start # working intermittently; a half-rendered VAPID pair makes push.py generate a # new keypair and delete every subscription row as "gone". Red here is the # only warning that arrives before those. # # Runs under sudo because the age key is 0400 root and the AppRole files are # 0400 agent/root -- the SSH user cannot read them directly. This grants CI # no vault access of its own: the HOST renders, CI only asks it to. # # The run history in Forgejo IS the record of consecutive green days. Do not # count a day this job was skipped or the runner was down as green. concurrency: group: ops-secrets-parity cancel-in-progress: false steps: - name: prod + beta (vps2) uses: https://github.com/appleboy/ssh-action@v1.2.0 with: host: ${{ secrets.VPS2_SSH_HOST }} username: ${{ secrets.VPS2_SSH_USER }} key: ${{ secrets.VPS2_SSH_KEY }} port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail # Each environment is checked from its OWN checkout. The trees are # identical, but running beta's from beta's directory also proves # that checkout is current rather than assuming it. sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta - name: dev (vps1) uses: https://github.com/appleboy/ssh-action@v1.2.0 with: host: ${{ secrets.VPS1_SSH_HOST }} username: ${{ secrets.VPS1_SSH_USER }} key: ${{ secrets.VPS1_SSH_KEY }} port: ${{ secrets.VPS1_SSH_PORT }} script: | set -euo pipefail # dev renders dev.yaml ALONE -- verify-parity.sh takes that from # env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the # comparison matches what deploy-dev.sh actually does. vps1 must # never see common.yaml: it runs Forgejo, its CI, and dev's # unreviewed branch. sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev indexnow: name: IndexNow ping runs-on: docker # Same overlap guard as the backup job above (schedule vs. manual # dispatch); --if-changed already makes a second ping a cheap no-op, but # queueing avoids two SSH sessions racing on the host regardless. concurrency: group: ops-indexnow cancel-in-progress: false steps: - name: Ping IndexNow if the URL set changed uses: https://github.com/appleboy/ssh-action@v1.2.0 with: host: ${{ secrets.VPS2_SSH_HOST }} username: ${{ secrets.VPS2_SSH_USER }} key: ${{ secrets.VPS2_SSH_KEY }} port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph/infra set -a; . /etc/thermograph.env 2>/dev/null || true; set +a bec=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_web" | head -1) [ -z "$bec" ] && bec=$(cd /opt/thermograph/infra && docker compose ps -q backend 2>/dev/null | head -1) [ -n "$bec" ] || { echo "!! no backend/web container found"; exit 1; } docker exec "$bec" python indexnow.py --if-changed \ "${THERMOGRAPH_BASE_URL:-https://thermograph.org}"