thermograph/.forgejo/workflows/ops-cron.yml

name: Ops cron (backup + IndexNow)

# Scheduled operational jobs that don't belong in the app's own worker-tier
# scheduler (notifications/scheduler.py, Track A chunk 5) because they're
# infra/ops concerns rather than app-domain background work -- see the job
# classification table in
# thermograph-docs/architecture/repo-topology-and-infrastructure.md §7.
#
# The jobs SSH into a host and run inside the already-running stack (docker
# exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new
# network exposure, no separate dependency install. Runs on the `docker` label,
# served by the vps2 runner (infra/deploy/forgejo/runner-vps2/) and, when it is
# up, the desktop.
#
# THIS WORKFLOW IS THE ESTATE'S ONLY BACKUP, so its dependency on a runner is
# load-bearing. On 2026-07-31 the desktop was the only registered runner and
# went offline for 21 hours; this schedule did not fire at 03:00Z. Forgejo held
# the run in `waiting` and it completed on reconnect, so nothing was lost that
# time. The vps2 runner exists so the next outage is not a coin flip.
#
# WHICH BOX EACH JOB TALKS TO, and why the secret names say so:
#
#   backup, indexnow  -> VPS2_SSH_*  (vps2 = 169.58.46.181: prod AND beta, the
#                        shared TimescaleDB, Postfix, the backups)
#   forgejo-backup    -> VPS1_SSH_*  (vps1 = 75.119.132.91: Forgejo, Grafana,
#                        Loki, and the dev environment)
#
# The secrets are keyed by HOST rather than by environment because an
# environment no longer implies a machine: vps2 runs two of them. The previous
# names encoded the opposite assumption and had already caused one incident --
# an early revision used SSH_* here, which meant "beta", so the job labelled
# "prod backup" was silently dumping beta while prod had no backup at all. The
# same conflation is why Forgejo's backup used to be described as running "on
# beta": beta and Forgejo happened to share a box, so one secret served both
# meanings. It does not any more.
#
# BOTH APPLICATION DATABASES ARE BACKED UP. prod and beta are separate databases
# (`thermograph` and `thermograph_beta`) on one shared instance. Dumping only
# the prod database would recreate the original failure in a new shape: a job
# that looks like "the backup" while one environment's data is silently
# uncovered. The loop below dumps each, to its own off-box prefix.

# Monorepo port: the compose file now lives under infra/ of the host's
# /opt/thermograph monorepo checkout, so every docker-compose invocation cd's
# into /opt/thermograph/infra (compose also pins `name: thermograph` so the
# project name no longer depends on the directory). Everything else unchanged.

on:
  schedule:
    # 03:00 UTC daily -- a low-traffic window for both jobs.
    - cron: '0 3 * * *'
  workflow_dispatch: {}

jobs:
  backup:
    name: pg_dump backup
    runs-on: docker
    # Guards against the schedule and a manual workflow_dispatch landing
    # close together (or two manual triggers): queue rather than cancel, so
    # a workflow_dispatch never aborts a pg_dump mid-write and leaves a
    # truncated .dump file -- same cancel-in-progress:false rationale as
    # deploy.yml/deploy-dev.yml's own SSH-script jobs.
    concurrency:
      group: ops-backup
      cancel-in-progress: false
    steps:
      - name: Dump both application databases over SSH
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed
        # into the remote script via `envs:` (a host env file isn't a reliable
        # source across boxes -- on vps1 it isn't readable by the deploy user --
        # so the CI secret is the uniform home, like the VPS*_SSH_* pairs).
        env:
          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
        with:
          host: ${{ secrets.VPS2_SSH_HOST }}
          username: ${{ secrets.VPS2_SSH_USER }}
          key: ${{ secrets.VPS2_SSH_KEY }}
          port: ${{ secrets.VPS2_SSH_PORT }}
          envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY
          script: |
            set -euo pipefail
            cd /opt/thermograph/infra
            # Source the env so `docker compose` can interpolate POSTGRES_PASSWORD;
            # without it compose fails to parse docker-compose.yml, the redirect
            # still creates the target, and the job leaves a 0-byte .dump and exits
            # non-zero (exactly how this backup was silently failing). Mirrors the
            # IndexNow job below, which already sources it.
            set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
            backup_dir="$HOME/thermograph-backups"
            mkdir -p "$backup_dir"
            stamp="$(date -u +%Y%m%dT%H%M%SZ)"
            # The db may run under plain compose OR as a Swarm stack task;
            # resolve the container either way so the backup survives a
            # deploy-mode switch. One instance now serves both environments.
            dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1)
            [ -z "$dbc" ] && dbc=$(cd /opt/thermograph/infra && docker compose ps -q db 2>/dev/null | head -1)
            [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; }
            # env:database pairs. Both are dumped every night: they are separate
            # databases on one instance, and backing up only one would leave the
            # other silently uncovered.
            for pair in prod:thermograph beta:thermograph_beta; do
              envname="${pair%%:*}"; dbname="${pair##*:}"
              # A missing database is a HARD failure, never a quiet skip -- a
              # backup job that shrugs off an absent database is exactly how an
              # environment ends up with no backups and nobody noticing.
              if ! docker exec "$dbc" psql -U thermograph -d postgres -tAc \
                   "select 1 from pg_database where datname='$dbname'" | grep -q 1; then
                echo "!! database '$dbname' ($envname) does not exist on this instance."
                if [ "$envname" = prod ]; then
                  echo "!! That is prod's own database on prod's own instance — this is not a"
                  echo "!! provisioning gap, something is badly wrong. Do not 'fix' it by creating"
                  echo "!! an empty database; find out where the real one went."
                else
                  echo "!! provision it first:  sudo bash /opt/thermograph-beta/infra/deploy/db/provision-env-db.sh $envname"
                fi
                exit 1
              fi
              out="$backup_dir/$dbname-$stamp.dump"
              # Write to a .partial and rename on success so a mid-dump failure can
              # never leave a truncated file that looks like a good backup.
              docker exec "$dbc" pg_dump -U thermograph -d "$dbname" \
                --format=custom > "$out.partial"
              mv "$out.partial" "$out"
              echo "wrote $out ($(du -h "$out" | cut -f1))"
              dumps="${dumps:-} $envname:$out"
            done
            # The dumps are the disaster-recovery copy, not a versioned
            # archive -- keep the last 14 days and let the rest age out.
            find "$backup_dir" -name '*.dump' -mtime +14 -delete
            find "$backup_dir" -name '*.dump.partial' -mtime +1 -delete
            # --- off-box copy to Contabo Object Storage (age-encrypted) ---
            # Streams the just-written dump through age (to the vault's age
            # recipient -- the same key each host renders secrets with, so
            # /etc/thermograph/age.key decrypts it for restore) straight to S3. No
            # local intermediate, no plaintext leaves the box. Skips cleanly if the
            # S3 secrets aren't set. Keeps 30 days of encrypted dumps off-box.
            if [ -n "${S3_ACCESS_KEY:-}" ]; then
              command -v rclone >/dev/null 2>&1 || sudo apt-get install -y -qq rclone
              command -v age    >/dev/null 2>&1 || sudo apt-get install -y -qq age
              export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \
                RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \
                RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
                RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
                RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true
              recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
              # One prefix per environment, so a restore never has to guess which
              # database a dump came from: backups/db/prod/ and backups/db/beta/.
              for entry in $dumps; do
                envname="${entry%%:*}"; f="${entry##*:}"
                s3base="$S3_BUCKET/backups/db/$envname"
                age -r "$recip" < "$f" | rclone rcat "archive:$s3base/$(basename "$f").age"
                echo "off-box: uploaded $(basename "$f").age to $s3base"
                rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true
              done
            else
              echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)"
            fi            

  forgejo-backup:
    name: Forgejo backup (vps1) -> S3
    runs-on: docker
    # Forgejo (the git host + all CI history) lives on vps1 and had NO backup at
    # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS,
    # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as
    # vps1's SSH user (VPS1_SSH_*), which is in the docker group -- so docker
    # exec/run need no host sudo, and the data tar runs inside a throwaway
    # container (the volume dir is root-owned on the host). rclone + age must be
    # present on vps1.
    #
    # This job follows FORGEJO, not beta. It used to use the same secret as the
    # beta deploy purely because Forgejo and beta shared a box; beta has since
    # moved to vps2 and Forgejo has not moved at all.
    concurrency:
      group: ops-forgejo-backup
      cancel-in-progress: false
    steps:
      - name: Back up Forgejo (db + data volume) over SSH
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        env:
          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
        with:
          host: ${{ secrets.VPS1_SSH_HOST }}
          username: ${{ secrets.VPS1_SSH_USER }}
          key: ${{ secrets.VPS1_SSH_KEY }}
          port: ${{ secrets.VPS1_SSH_PORT }}
          envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY
          script: |
            set -euo pipefail
            [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; }
            command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on vps1"; exit 1; }
            command -v age    >/dev/null 2>&1 || { echo "!! age missing on vps1"; exit 1; }
            export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \
              RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \
              RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
              RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
              RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true
            recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
            stamp="$(date -u +%Y%m%dT%H%M%SZ)"
            base="$S3_BUCKET/backups/forgejo"
            fdb=$(docker ps -qf name=forgejo_db | head -1)
            [ -n "$fdb" ] || { echo "!! no forgejo_db container"; exit 1; }
            # 1) Forgejo Postgres (consistent dump); creds read inside the container
            docker exec "$fdb" sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \
              | age -r "$recip" | rclone rcat "archive:$base/db/forgejo-$stamp.dump.age"
            # 2) Forgejo data volume via a throwaway container (no host sudo needed)
            docker run --rm -v forgejo_forgejo_data:/data:ro busybox tar -C /data -cf - . \
              | age -r "$recip" | rclone rcat "archive:$base/data/forgejo-data-$stamp.tar.age"
            echo "forgejo backup uploaded ($stamp)"
            rclone delete --min-age 30d "archive:$base/db/"   2>/dev/null || true
            rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true            

  secrets-parity:
    name: OpenBao/SOPS parity gate
    runs-on: docker
    # The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive
    # green parity runs" -- and which, until now, nothing implemented. A gate
    # that exists only in prose gets satisfied by assertion.
    #
    # verify-parity.sh renders each environment through BOTH backends and diffs
    # the KEY=value sets after a last-wins collapse. It prints key NAMES and
    # counts only, never values, so these logs are safe to read and to paste.
    #
    # A mismatch FAILS the job, deliberately and loudly. The failure modes on
    # the other side of a bad cutover are all silent: a missing
    # THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start
    # working intermittently; a half-rendered VAPID pair makes push.py generate a
    # new keypair and delete every subscription row as "gone". Red here is the
    # only warning that arrives before those.
    #
    # Runs under sudo because the age key is 0400 root and the AppRole files are
    # 0400 agent/root -- the SSH user cannot read them directly. This grants CI
    # no vault access of its own: the HOST renders, CI only asks it to.
    #
    # The run history in Forgejo IS the record of consecutive green days. Do not
    # count a day this job was skipped or the runner was down as green.
    concurrency:
      group: ops-secrets-parity
      cancel-in-progress: false
    steps:
      - name: prod + beta (vps2)
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.VPS2_SSH_HOST }}
          username: ${{ secrets.VPS2_SSH_USER }}
          key: ${{ secrets.VPS2_SSH_KEY }}
          port: ${{ secrets.VPS2_SSH_PORT }}
          script: |
            set -euo pipefail
            # Each environment is checked from its OWN checkout. The trees are
            # identical, but running beta's from beta's directory also proves
            # that checkout is current rather than assuming it.
            sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod
            sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta            

      - name: dev (vps1)
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.VPS1_SSH_HOST }}
          username: ${{ secrets.VPS1_SSH_USER }}
          key: ${{ secrets.VPS1_SSH_KEY }}
          port: ${{ secrets.VPS1_SSH_PORT }}
          script: |
            set -euo pipefail
            # dev renders dev.yaml ALONE -- verify-parity.sh takes that from
            # env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the
            # comparison matches what deploy-dev.sh actually does. vps1 must
            # never see common.yaml: it runs Forgejo, its CI, and dev's
            # unreviewed branch.
            sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev            

  indexnow:
    name: IndexNow ping
    runs-on: docker
    # Same overlap guard as the backup job above (schedule vs. manual
    # dispatch); --if-changed already makes a second ping a cheap no-op, but
    # queueing avoids two SSH sessions racing on the host regardless.
    concurrency:
      group: ops-indexnow
      cancel-in-progress: false
    steps:
      - name: Ping IndexNow if the URL set changed
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.VPS2_SSH_HOST }}
          username: ${{ secrets.VPS2_SSH_USER }}
          key: ${{ secrets.VPS2_SSH_KEY }}
          port: ${{ secrets.VPS2_SSH_PORT }}
          script: |
            set -euo pipefail
            cd /opt/thermograph/infra
            set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
            bec=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_web" | head -1)
            [ -z "$bec" ] && bec=$(cd /opt/thermograph/infra && docker compose ps -q backend 2>/dev/null | head -1)
            [ -n "$bec" ] || { echo "!! no backend/web container found"; exit 1; }
            docker exec "$bec" python indexnow.py --if-changed \
              "${THERMOGRAPH_BASE_URL:-https://thermograph.org}"