2026-07-24 20:58:15 +00:00
4 changed files with 161 additions and 4 deletions
--- a/.forgejo/workflows/ops-cron.yml
+++ b/.forgejo/workflows/ops-cron.yml
@ -46,11 +46,21 @@ jobs:
    steps:
      - name: Dump the prod database over SSH
        uses: https://github.com/appleboy/ssh-action@v1.2.0
+        # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed
+        # into the remote script via `envs:` (the host env file isn't a reliable
+        # source across boxes -- beta's /etc/thermograph.env isn't readable by the
+        # deploy user -- so the CI secret is the uniform home, like PROD_SSH_*).
+        env:
+          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
+          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
        with:
          host: ${{ secrets.PROD_SSH_HOST }}
          username: ${{ secrets.PROD_SSH_USER }}
          key: ${{ secrets.PROD_SSH_KEY }}
          port: ${{ secrets.PROD_SSH_PORT }}
+          envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY
          script: |
            set -euo pipefail
            cd /opt/thermograph/infra
@ -80,6 +90,79 @@ jobs:
            # archive -- keep the last 14 days and let the rest age out.
            find "$backup_dir" -name 'thermograph-*.dump' -mtime +14 -delete
            find "$backup_dir" -name 'thermograph-*.dump.partial' -mtime +1 -delete
+            # --- off-box copy to Contabo Object Storage (age-encrypted) ---
+            # Streams the just-written dump through age (to the vault's age
+            # recipient -- the same key each host renders secrets with, so
+            # /etc/thermograph/age.key decrypts it for restore) straight to S3. No
+            # local intermediate, no plaintext leaves the box. Skips cleanly if the
+            # S3 secrets aren't set. Keeps 30 days of encrypted dumps off-box.
+            if [ -n "${S3_ACCESS_KEY:-}" ]; then
+              command -v rclone >/dev/null 2>&1 || sudo apt-get install -y -qq rclone
+              command -v age    >/dev/null 2>&1 || sudo apt-get install -y -qq age
+              export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \
+                RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \
+                RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
+                RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
+                RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true
+              recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
+              s3base="$S3_BUCKET/backups/db/prod"
+              age -r "$recip" < "$out" | rclone rcat "archive:$s3base/$(basename "$out").age"
+              echo "off-box: uploaded $(basename "$out").age to $s3base"
+              rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true
+            else
+              echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)"
+            fi
+
+  forgejo-backup:
+    name: Forgejo backup (beta) -> S3
+    runs-on: docker
+    # Forgejo (the git host + all CI history) lives on beta and had NO backup at
+    # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS,
+    # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as
+    # the beta SSH user (SSH_*), which is in the docker group -- so docker exec/run
+    # need no host sudo, and the data tar runs inside a throwaway container (the
+    # volume dir is root-owned on the host). rclone + age must be present on beta.
+    concurrency:
+      group: ops-forgejo-backup
+      cancel-in-progress: false
+    steps:
+      - name: Back up Forgejo (db + data volume) over SSH
+        uses: https://github.com/appleboy/ssh-action@v1.2.0
+        env:
+          S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
+          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USER }}
+          key: ${{ secrets.SSH_KEY }}
+          port: ${{ secrets.SSH_PORT }}
+          envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY
+          script: |
+            set -euo pipefail
+            [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; }
+            command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on beta"; exit 1; }
+            command -v age    >/dev/null 2>&1 || { echo "!! age missing on beta"; exit 1; }
+            export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \
+              RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \
+              RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \
+              RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \
+              RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true
+            recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
+            stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+            base="$S3_BUCKET/backups/forgejo"
+            fdb=$(docker ps -qf name=forgejo_db | head -1)
+            [ -n "$fdb" ] || { echo "!! no forgejo_db container"; exit 1; }
+            # 1) Forgejo Postgres (consistent dump); creds read inside the container
+            docker exec "$fdb" sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \
+              | age -r "$recip" | rclone rcat "archive:$base/db/forgejo-$stamp.dump.age"
+            # 2) Forgejo data volume via a throwaway container (no host sudo needed)
+            docker run --rm -v forgejo_forgejo_data:/data:ro busybox tar -C /data -cf - . \
+              | age -r "$recip" | rclone rcat "archive:$base/data/forgejo-data-$stamp.tar.age"
+            echo "forgejo backup uploaded ($stamp)"
+            rclone delete --min-age 30d "archive:$base/db/"   2>/dev/null || true
+            rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true

  indexnow:
    name: IndexNow ping
--- a/infra/deploy/backup/README.md
+++ b/infra/deploy/backup/README.md
@ -0,0 +1,66 @@
+# Off-box backups → Contabo Object Storage
+
+Durable, encrypted, off-box copies of the state that would otherwise die with a
+single VPS. Driven by `.forgejo/workflows/ops-cron.yml` (03:00 UTC daily +
+`workflow_dispatch`).
+
+## What is backed up
+
+| Prefix in bucket `era5-thermograph` | Source | Job | Retention off-box |
+|---|---|---|---|
+| `backups/db/prod/` | prod Postgres/TimescaleDB (`pg_dump --format=custom`) | `backup` (prod, `PROD_SSH_*`) | 30 days |
+| `backups/forgejo/db/` | Forgejo Postgres (`forgejo_db`) | `forgejo-backup` (beta, `SSH_*`) | 30 days |
+| `backups/forgejo/data/` | Forgejo data volume (repos/LFS/config) | `forgejo-backup` | 30 days |
+
+Everything is streamed through **`age`** encryption before upload — encrypted to the
+vault recipient `age1xx4dz…`, so the private key each host already has at
+`/etc/thermograph/age.key` (and the operator's `~/.config/sops/age/keys.txt`)
+decrypts it. Nothing plaintext leaves the box.
+
+## Access layer
+- S3 endpoint/bucket/keys are **Forgejo Actions secrets** (`S3_ENDPOINT`,
+  `S3_BUCKET`, `S3_ACCESS_KEY`, `S3_SECRET_KEY`) — the CI job's credential home
+  (also mirrored into the SOPS vault `deploy/secrets/{prod,beta}.yaml` for host-side
+  use). Contabo needs **path-style** addressing (`force_path_style=true`,
+  `provider=Other`, `region=default`).
+- `rclone` + `age` must be installed on prod and beta (one-time: `apt-get install -y
+  rclone age`). The prod job apt-installs them if missing; the beta user has no sudo,
+  so they are pre-installed there.
+
+## Restore (DR runbook)
+
+Prereq on the restoring host: `rclone` + `age` + the age key at
+`/etc/thermograph/age.key` (or the operator key), and rclone configured for the
+`archive:` remote (env vars `RCLONE_CONFIG_ARCHIVE_*` or `/etc/rclone/rclone.conf`).
+
+### Prod database
+```sh
+# list available encrypted dumps
+rclone ls archive:era5-thermograph/backups/db/prod/
+# download newest, decrypt, restore into a fresh db
+OBJ=archive:era5-thermograph/backups/db/prod/<name>.dump.age
+rclone cat "$OBJ" | sudo age -d -i /etc/thermograph/age.key \
+  | docker exec -i <db-container> pg_restore -U thermograph -d thermograph --clean --if-exists
+```
+
+### Forgejo
+```sh
+# database
+rclone cat archive:era5-thermograph/backups/forgejo/db/<name>.dump.age \
+  | sudo age -d -i /etc/thermograph/age.key \
+  | docker exec -i forgejo_db sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists'
+# data volume (repos/lfs/config) — restore into the volume with Forgejo stopped
+rclone cat archive:era5-thermograph/backups/forgejo/data/<name>.tar.age \
+  | sudo age -d -i /etc/thermograph/age.key \
+  | docker run --rm -i -v forgejo_forgejo_data:/data busybox tar -C /data -xf -
+```
+
+## Verify a backup is restorable (quick)
+```sh
+rclone cat archive:era5-thermograph/backups/db/prod/<name>.dump.age \
+  | sudo age -d -i /etc/thermograph/age.key | head -c 5   # -> PGDMP = valid custom-format dump
+```
+
+## Manual run
+`workflow_dispatch` the "Ops cron (backup + IndexNow)" workflow, or run the same
+`age | rclone rcat` pipeline by hand (see the workflow script).
--- a/infra/deploy/secrets/beta.yaml
+++ b/infra/deploy/secrets/beta.yaml
@ -16,6 +16,10 @@ THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:sTtkAbMVUtwvRssCZJa2PWUjlfnYS
 THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:SnFw+pwNgZlxMEhMUH/1daMZlRB2QFNJHtL+4C3S1eFcJJdT2uNGkY7Ym9FPfx1V7cwt/3urHg1oVgdDzYaIal2wye4l5HhKBP5PjtQkDW+Pck24kxul,iv:Ku4MhsrAfyA2Y3CqhGIsYKrjhKQ4cqAYjCqjFeEvBWE=,tag:hw42iS+79D7gNf+BPTYM0g==,type:str]
 THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:ox+hHiFMTPZqYwICDSwPwcT13ZDSKiGcAVHfVVMNYQ==,iv:DssV/olPXERpdmXNMmzjVYvhn/0zmuSp2XlTs42pIi0=,tag:4YHXI1BFbGZ6JsdXqJuf3w==,type:str]
 REGISTRY_TOKEN: ENC[AES256_GCM,data:pC1Zzs56la8qjzWcbkuKyeVHwS5WfIe+mrLm5Sdi/LKYziAvRU5cOA==,iv:57PlJEBdgwNJsfFdNOGqAnKk8l6zpmMSWYFEKS6pn+4=,tag:pPftNHUVU6+CY2nWMaXO6w==,type:str]
+THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:QSMp9yUEf9qMdVrSkKB56WzcW39/2QLPgHS5pRY/,iv:0YeVVf196LFl/cKhCUezVFlteA0fpC7BYDFuSH4Ihh4=,tag:HfHvrlkjT39bCKrM2ZN8WQ==,type:str]
+THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:H3/dfY5HpRPnz6BvokHVZg==,iv:1ExIBSVFSuGXThwG1d5MymPLQYUXkm0iUjZx0A8+o6c=,tag:e+pu/axaS7gKoptHBPDzhA==,type:str]
+THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:62UeBWrSsF/sl8zwUjOoSu0Tx2Q6JBPTI6bhJ5JewFM=,iv:IMnYZyTcevRZD/EXPNymKLVEUMSKHDcWhdXEBItpg9E=,tag:UC5pmSuoJ6WJxaMfOVe4aw==,type:str]
+THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:9J+L4O4mZyWBr7tc2CgV8w7Jbr3lFsxMFO6Zg8+813c=,iv:t4m1BXMnSzk0K0TxuKC3UxpYZy34RWNNnGFVyLdi23s=,tag:Kwetz4YxLWsqDA0To7z8qg==,type:str]
 sops:
    age:
        - enc: |
@ -27,7 +31,7 @@ sops:
            0+PQdtDxom72BqrAoj5lhepxW8YWPqQIRXsVS/XYQEQfPJuuuiRMKw==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
-    lastmodified: "2026-07-22T03:17:28Z"
-    mac: ENC[AES256_GCM,data:npF+ngF4Khj3NJ2PNFcQjQYIrZVpdPM0drk43pAI6HfFaynX8JOqRFZRsH0sCfMoPkXB/ffCJxla1Tt3Mu3zgKTNSWcn+mjzl+dYvlruY1VylHLS+GnTO9zKeQcwbrYzTfZNWzQhm6/mw8KCL3791U/xNHwJc6g5VxRUWl6TBdo=,iv:JLzY8rm33NLG8CqyAlU6vBhlODUiRipKu9daoRvgyzg=,tag:yIsxVZAnw8FD7Bt/x55wMQ==,type:str]
+    lastmodified: "2026-07-23T20:22:14Z"
+    mac: ENC[AES256_GCM,data:i8gfHIslHMWZe0wxGlHL+5doqXR+48lr1gV5Doo5ynChLQaPWFE1n98JNEOBqm3wDbpjhARgxm/kCMkeXBJvnMwIBCWKhuwYACDR1mY1v63lgW1GJsIYWRsmR0uQ8Rd/dYwjP8qywf54pDJdjYAJjnvhmMaDty1WTsin3OABmWs=,iv:9rUO42Esh/Msec9lAgcxHFOYqau6VCTUWDMxF9WXQ9A=,tag:a0e6Y0kYdySOCy29Yxjf6w==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
--- a/infra/deploy/secrets/prod.yaml
+++ b/infra/deploy/secrets/prod.yaml
@ -24,6 +24,10 @@ THERMOGRAPH_DISCORD_BOT_TOKEN: ENC[AES256_GCM,data:zfVJLH8ddct2Jvstl/olZLQ9SM15c
 REGISTRY_TOKEN: ENC[AES256_GCM,data:3VETZR026YXvWA7DLly4/mz+hqEK7vkEMwKlDKU6CQZeZiFpbQXWig==,iv:ixZJi2VWAZak6dFPvVyErVRanp9bzqXyV3MVa4QsBuU=,tag:Pc2Mdw58S0tSdkgJ5yuVNw==,type:str]
 THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Kg==,iv:CPYN8aCu79PHaz3Zj6OgqyJhXdHLt0tLodgML2NuwQM=,tag:1B3IhlfjCwSlPq7FnKGeRg==,type:str]
 THERMOGRAPH_DISCORD_WEATHER_CHANNEL: ENC[AES256_GCM,data:WIxYcoOnBDdDcNj8uJ5eUqXjVQ==,iv:l6kU5PGVcW2Nv7EqBUGCRxBzaJOJ3AW1nEEH33aAbmI=,tag:/KgdCKmrlQioIWdz21PcTg==,type:str]
+THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:0OwwhGiJB3dXGZaD0ObV84ZJg4XLIByIrzaCYZMQ,iv:AqerRn2lD6u90zfGneFT/cq3LAnHqBZH3ICkAuvlztk=,tag:FJAYgILUGwJFLdYtqPKWnw==,type:str]
+THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:sernOo7VeKi22X1G+WqGcg==,iv:uLiChXwP+gqlJv07hwZmKUraxdp2YwFbQgYHQNTH7C0=,tag:EtUgNayYrMx3wYprhEE4Hw==,type:str]
+THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:RDD4MsWSf0HBEHBSibjNSZFbOm867XdPS6TdACN61Is=,iv:IJSbX/B94Y2OLqaskEp5IOZh4snNzTwPEPkl+u/KCxc=,tag:Xka/7TSPuBD97l04CR/D0A==,type:str]
+THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:QD4JYKTbshcm0yx3YTHr2VUDq23ABjwk3wvMUSIDs0s=,iv:/JyMzcj1LWcxE8ioqCmkjZIFnmTBNEvaYXrPCFztETM=,tag:EnnkTQjsotXaxOd60mlOTw==,type:str]
 sops:
    age:
        - enc: |
@ -35,7 +39,7 @@ sops:
            PDXCB8EqKwdC3BnKlpD9ptYcuPHVUSgve1wIZYJ7iHIEhW9Kc6RXOw==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
-    lastmodified: "2026-07-23T04:33:06Z"
-    mac: ENC[AES256_GCM,data:BjNMEudsS9ZViR55OjSXu5qrOJ9yu2kCGx61qu4vE1ZZJIZtnnyZ3CfpsPygvLLhiFRj76xcqFnuSqTdr17BCEHz3UzsHFv0vBi17lZ5ksN5Oz+9wr+3Pg3TvxC0VF2VHJVP84VH61CmB5+G4WOms8WqLtmIGSLYcTVtcIFoUhk=,iv:jTEB8Mxg8qIyNfhpSgJtUvULVx4+XFrPLyQKhNcbkTg=,tag:SIyvNltdGCMfDGvi4kD3MA==,type:str]
+    lastmodified: "2026-07-23T20:22:14Z"
+    mac: ENC[AES256_GCM,data:flh82V4mBV7BM5lgfzJfLona87YE02/LcPEcbmWPMmlql/ZgipIPl7uCdM8MNDLyaMgXPfxmEIb20Ly9yT5b4zp2QGqHBlfc58M1wRmTOVJFuSV4SUEBJTMOv0zqpnr0IGRpJw30QAu4xq0op7lEZo9WXMIvOxyrA591viwlX08=,iv:C2TnbWL5OvIPQFMghOSTQaoh2hI3hcRNG61+V9OIl5A=,tag:hF5K+xzvdfN4d/ZIjKJEpA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2