thermograph/.forgejo/workflows/ops-cron.yml

name: Ops cron (backup + IndexNow)

# Scheduled operational jobs that don't belong in the app's own worker-tier
# scheduler (notifications/scheduler.py, Track A chunk 5) because they're
# infra/ops concerns rather than app-domain background work -- see the job
# classification table in
# thermograph-docs/architecture/repo-topology-and-infrastructure.md §7.
#
# Both jobs SSH into the prod host and run inside the already-running compose
# stack (docker compose exec), the same way deploy.sh already runs its own
# post-deploy IndexNow ping -- no new network exposure, no separate dependency
# install. Runs on the `docker` label (the always-on Swarm-hosted runner
# deploy/forgejo/ stood up).
#
# Targets PROD via the PROD_SSH_* secrets (prod = 169.58.46.181, `agent` user,
# in the docker group so no sudo needed; /etc/thermograph.env is agent-readable).
# These are the same secrets deploy-prod.yml uses for the release->prod deploy --
# NOT the SSH_* secrets, which point at BETA (deploy.yml's `main`->beta path). An
# earlier revision reused SSH_* here, so the "prod" backup was silently dumping
# beta; prod itself had no backup at all. The prod database is the one that must
# be backed up, so both jobs use PROD_SSH_*.

# Monorepo port: the compose file now lives under infra/ of the host's
# /opt/thermograph monorepo checkout, so every docker-compose invocation cd's
# into /opt/thermograph/infra (compose also pins `name: thermograph` so the
# project name no longer depends on the directory). Everything else unchanged.

on:
  schedule:
    # 03:00 UTC daily -- a low-traffic window for both jobs.
    - cron: '0 3 * * *'
  workflow_dispatch: {}

jobs:
  backup:
    name: pg_dump backup
    runs-on: docker
    # Guards against the schedule and a manual workflow_dispatch landing
    # close together (or two manual triggers): queue rather than cancel, so
    # a workflow_dispatch never aborts a pg_dump mid-write and leaves a
    # truncated .dump file -- same cancel-in-progress:false rationale as
    # deploy.yml/deploy-dev.yml's own SSH-script jobs.
    concurrency:
      group: ops-backup
      cancel-in-progress: false
    steps:
      - name: Dump the prod database over SSH
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed
        # into the remote script via `envs:` (the host env file isn't a reliable
        # source across boxes -- beta's /etc/thermograph.env isn't readable by the
        # deploy user -- so the CI secret is the uniform home, like PROD_SSH_*).
        env:
          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
        with:
          host: ${{ secrets.PROD_SSH_HOST }}
          username: ${{ secrets.PROD_SSH_USER }}
          key: ${{ secrets.PROD_SSH_KEY }}
          port: ${{ secrets.PROD_SSH_PORT }}
          envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY
          script: |
            set -euo pipefail
            cd /opt/thermograph/infra
            # Source the env so `docker compose` can interpolate POSTGRES_PASSWORD;
            # without it compose fails to parse docker-compose.yml, the redirect
            # still creates the target, and the job leaves a 0-byte .dump and exits
            # non-zero (exactly how this backup was silently failing). Mirrors the
            # IndexNow job below, which already sources it.
            set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
            backup_dir="$HOME/thermograph-backups"
            mkdir -p "$backup_dir"
            stamp="$(date -u +%Y%m%dT%H%M%SZ)"
            out="$backup_dir/thermograph-$stamp.dump"
            # The db may run under plain compose OR as a Swarm stack task
            # (prod post-cutover); resolve the container either way so the
            # backup survives the deploy-mode switch.
            dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1)
            [ -z "$dbc" ] && dbc=$(cd /opt/thermograph/infra && docker compose ps -q db 2>/dev/null | head -1)
            [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; }
            # Write to a .partial and rename on success so a mid-dump failure can
            # never leave a truncated file that looks like a good backup.
            docker exec "$dbc" pg_dump -U thermograph -d thermograph \
              --format=custom > "$out.partial"
            mv "$out.partial" "$out"
            echo "wrote $out ($(du -h "$out" | cut -f1))"
            # The dumps are the disaster-recovery copy, not a versioned
            # archive -- keep the last 14 days and let the rest age out.
            find "$backup_dir" -name 'thermograph-*.dump' -mtime +14 -delete
            find "$backup_dir" -name 'thermograph-*.dump.partial' -mtime +1 -delete
            # --- off-box copy to Contabo Object Storage (age-encrypted) ---
            # Streams the just-written dump through age (to the vault's age
            # recipient -- the same key each host renders secrets with, so
            # /etc/thermograph/age.key decrypts it for restore) straight to S3. No
            # local intermediate, no plaintext leaves the box. Skips cleanly if the
            # S3 secrets aren't set. Keeps 30 days of encrypted dumps off-box.
            if [ -n "${S3_ACCESS_KEY:-}" ]; then
              command -v rclone >/dev/null 2>&1 || sudo apt-get install -y -qq rclone
              command -v age    >/dev/null 2>&1 || sudo apt-get install -y -qq age
              export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \
                RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \
                RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
                RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
                RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true
              recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
              s3base="$S3_BUCKET/backups/db/prod"
              age -r "$recip" < "$out" | rclone rcat "archive:$s3base/$(basename "$out").age"
              echo "off-box: uploaded $(basename "$out").age to $s3base"
              rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true
            else
              echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)"
            fi

  forgejo-backup:
    name: Forgejo backup (beta) -> S3
    runs-on: docker
    # Forgejo (the git host + all CI history) lives on beta and had NO backup at
    # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS,
    # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as
    # the beta SSH user (SSH_*), which is in the docker group -- so docker exec/run
    # need no host sudo, and the data tar runs inside a throwaway container (the
    # volume dir is root-owned on the host). rclone + age must be present on beta.
    concurrency:
      group: ops-forgejo-backup
      cancel-in-progress: false
    steps:
      - name: Back up Forgejo (db + data volume) over SSH
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        env:
          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
          S3_BUCKET: ${{ secrets.S3_BUCKET }}
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
        with:
          host: ${{ secrets.SSH_HOST }}
          username: ${{ secrets.SSH_USER }}
          key: ${{ secrets.SSH_KEY }}
          port: ${{ secrets.SSH_PORT }}
          envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY
          script: |
            set -euo pipefail
            [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; }
            command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on beta"; exit 1; }
            command -v age    >/dev/null 2>&1 || { echo "!! age missing on beta"; exit 1; }
            export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \
              RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \
              RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
              RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
              RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true
            recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
            stamp="$(date -u +%Y%m%dT%H%M%SZ)"
            base="$S3_BUCKET/backups/forgejo"
            fdb=$(docker ps -qf name=forgejo_db | head -1)
            [ -n "$fdb" ] || { echo "!! no forgejo_db container"; exit 1; }
            # 1) Forgejo Postgres (consistent dump); creds read inside the container
            docker exec "$fdb" sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \
              | age -r "$recip" | rclone rcat "archive:$base/db/forgejo-$stamp.dump.age"
            # 2) Forgejo data volume via a throwaway container (no host sudo needed)
            docker run --rm -v forgejo_forgejo_data:/data:ro busybox tar -C /data -cf - . \
              | age -r "$recip" | rclone rcat "archive:$base/data/forgejo-data-$stamp.tar.age"
            echo "forgejo backup uploaded ($stamp)"
            rclone delete --min-age 30d "archive:$base/db/"   2>/dev/null || true
            rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true

  indexnow:
    name: IndexNow ping
    runs-on: docker
    # Same overlap guard as the backup job above (schedule vs. manual
    # dispatch); --if-changed already makes a second ping a cheap no-op, but
    # queueing avoids two SSH sessions racing on the host regardless.
    concurrency:
      group: ops-indexnow
      cancel-in-progress: false
    steps:
      - name: Ping IndexNow if the URL set changed
        uses: https://github.com/appleboy/ssh-action@v1.2.0
        with:
          host: ${{ secrets.PROD_SSH_HOST }}
          username: ${{ secrets.PROD_SSH_USER }}
          key: ${{ secrets.PROD_SSH_KEY }}
          port: ${{ secrets.PROD_SSH_PORT }}
          script: |
            set -euo pipefail
            cd /opt/thermograph/infra
            set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
            bec=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_web" | head -1)
            [ -z "$bec" ] && bec=$(cd /opt/thermograph/infra && docker compose ps -q backend 2>/dev/null | head -1)
            [ -n "$bec" ] || { echo "!! no backend/web container found"; exit 1; }
            docker exec "$bec" python indexnow.py --if-changed \
              "${THERMOGRAPH_BASE_URL:-https://thermograph.org}"