From 6086b40fe616e4b187df314ef66cbf90e186f627 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 23 Jul 2026 13:42:09 -0700 Subject: [PATCH] Add off-box encrypted backups to Contabo Object Storage Prod DB dumps and Forgejo (db + data volume) are now copied off-box to the era5-thermograph S3 bucket, age-encrypted to the vault recipient so the host age key at /etc/thermograph/age.key decrypts them for restore. Nothing plaintext leaves the box; no local intermediate (streamed via age | rclone rcat). ops-cron: the prod pg_dump job gains an off-box push; a new forgejo-backup job dumps Forgejo's Postgres + tars its data volume on beta (the git host had NO backup at all). 30-day off-box retention on both. S3 creds are Forgejo Actions secrets (S3_ENDPOINT/BUCKET/ACCESS_KEY/SECRET_KEY), mirrored into the SOPS vault (prod.yaml/beta.yaml) for host-side use. Contabo needs path-style addressing. Adds deploy/backup/README.md with the DR restore runbook. --- .forgejo/workflows/ops-cron.yml | 83 +++++++++++++++++++++++++++++++++ infra/deploy/backup/README.md | 66 ++++++++++++++++++++++++++ infra/deploy/secrets/beta.yaml | 8 +++- infra/deploy/secrets/prod.yaml | 8 +++- 4 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 infra/deploy/backup/README.md diff --git a/.forgejo/workflows/ops-cron.yml b/.forgejo/workflows/ops-cron.yml index 3bed373..7ee5f20 100644 --- a/.forgejo/workflows/ops-cron.yml +++ b/.forgejo/workflows/ops-cron.yml @@ -46,11 +46,21 @@ jobs: steps: - name: Dump the prod database over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 + # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed + # into the remote script via `envs:` (the host env file isn't a reliable + # source across boxes -- beta's /etc/thermograph.env isn't readable by the + # deploy user -- so the CI secret is the uniform home, like PROD_SSH_*). + env: + S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: host: ${{ secrets.PROD_SSH_HOST }} username: ${{ secrets.PROD_SSH_USER }} key: ${{ secrets.PROD_SSH_KEY }} port: ${{ secrets.PROD_SSH_PORT }} + envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail cd /opt/thermograph/infra @@ -80,6 +90,79 @@ jobs: # archive -- keep the last 14 days and let the rest age out. find "$backup_dir" -name 'thermograph-*.dump' -mtime +14 -delete find "$backup_dir" -name 'thermograph-*.dump.partial' -mtime +1 -delete + # --- off-box copy to Contabo Object Storage (age-encrypted) --- + # Streams the just-written dump through age (to the vault's age + # recipient -- the same key each host renders secrets with, so + # /etc/thermograph/age.key decrypts it for restore) straight to S3. No + # local intermediate, no plaintext leaves the box. Skips cleanly if the + # S3 secrets aren't set. Keeps 30 days of encrypted dumps off-box. + if [ -n "${S3_ACCESS_KEY:-}" ]; then + command -v rclone >/dev/null 2>&1 || sudo apt-get install -y -qq rclone + command -v age >/dev/null 2>&1 || sudo apt-get install -y -qq age + export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \ + RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \ + RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ + RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ + RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true + recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 + s3base="$S3_BUCKET/backups/db/prod" + age -r "$recip" < "$out" | rclone rcat "archive:$s3base/$(basename "$out").age" + echo "off-box: uploaded $(basename "$out").age to $s3base" + rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true + else + echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)" + fi + + forgejo-backup: + name: Forgejo backup (beta) -> S3 + runs-on: docker + # Forgejo (the git host + all CI history) lives on beta and had NO backup at + # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS, + # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as + # the beta SSH user (SSH_*), which is in the docker group -- so docker exec/run + # need no host sudo, and the data tar runs inside a throwaway container (the + # volume dir is root-owned on the host). rclone + age must be present on beta. + concurrency: + group: ops-forgejo-backup + cancel-in-progress: false + steps: + - name: Back up Forgejo (db + data volume) over SSH + uses: https://github.com/appleboy/ssh-action@v1.2.0 + env: + S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} + S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USER }} + key: ${{ secrets.SSH_KEY }} + port: ${{ secrets.SSH_PORT }} + envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY + script: | + set -euo pipefail + [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; } + command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on beta"; exit 1; } + command -v age >/dev/null 2>&1 || { echo "!! age missing on beta"; exit 1; } + export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \ + RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \ + RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ + RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ + RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true + recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + base="$S3_BUCKET/backups/forgejo" + fdb=$(docker ps -qf name=forgejo_db | head -1) + [ -n "$fdb" ] || { echo "!! no forgejo_db container"; exit 1; } + # 1) Forgejo Postgres (consistent dump); creds read inside the container + docker exec "$fdb" sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dump -U $POSTGRES_USER -d $POSTGRES_DB --format=custom' \ + | age -r "$recip" | rclone rcat "archive:$base/db/forgejo-$stamp.dump.age" + # 2) Forgejo data volume via a throwaway container (no host sudo needed) + docker run --rm -v forgejo_forgejo_data:/data:ro busybox tar -C /data -cf - . \ + | age -r "$recip" | rclone rcat "archive:$base/data/forgejo-data-$stamp.tar.age" + echo "forgejo backup uploaded ($stamp)" + rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true + rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true indexnow: name: IndexNow ping diff --git a/infra/deploy/backup/README.md b/infra/deploy/backup/README.md new file mode 100644 index 0000000..5bbc23e --- /dev/null +++ b/infra/deploy/backup/README.md @@ -0,0 +1,66 @@ +# Off-box backups → Contabo Object Storage + +Durable, encrypted, off-box copies of the state that would otherwise die with a +single VPS. Driven by `.forgejo/workflows/ops-cron.yml` (03:00 UTC daily + +`workflow_dispatch`). + +## What is backed up + +| Prefix in bucket `era5-thermograph` | Source | Job | Retention off-box | +|---|---|---|---| +| `backups/db/prod/` | prod Postgres/TimescaleDB (`pg_dump --format=custom`) | `backup` (prod, `PROD_SSH_*`) | 30 days | +| `backups/forgejo/db/` | Forgejo Postgres (`forgejo_db`) | `forgejo-backup` (beta, `SSH_*`) | 30 days | +| `backups/forgejo/data/` | Forgejo data volume (repos/LFS/config) | `forgejo-backup` | 30 days | + +Everything is streamed through **`age`** encryption before upload — encrypted to the +vault recipient `age1xx4dz…`, so the private key each host already has at +`/etc/thermograph/age.key` (and the operator's `~/.config/sops/age/keys.txt`) +decrypts it. Nothing plaintext leaves the box. + +## Access layer +- S3 endpoint/bucket/keys are **Forgejo Actions secrets** (`S3_ENDPOINT`, + `S3_BUCKET`, `S3_ACCESS_KEY`, `S3_SECRET_KEY`) — the CI job's credential home + (also mirrored into the SOPS vault `deploy/secrets/{prod,beta}.yaml` for host-side + use). Contabo needs **path-style** addressing (`force_path_style=true`, + `provider=Other`, `region=default`). +- `rclone` + `age` must be installed on prod and beta (one-time: `apt-get install -y + rclone age`). The prod job apt-installs them if missing; the beta user has no sudo, + so they are pre-installed there. + +## Restore (DR runbook) + +Prereq on the restoring host: `rclone` + `age` + the age key at +`/etc/thermograph/age.key` (or the operator key), and rclone configured for the +`archive:` remote (env vars `RCLONE_CONFIG_ARCHIVE_*` or `/etc/rclone/rclone.conf`). + +### Prod database +```sh +# list available encrypted dumps +rclone ls archive:era5-thermograph/backups/db/prod/ +# download newest, decrypt, restore into a fresh db +OBJ=archive:era5-thermograph/backups/db/prod/.dump.age +rclone cat "$OBJ" | sudo age -d -i /etc/thermograph/age.key \ + | docker exec -i pg_restore -U thermograph -d thermograph --clean --if-exists +``` + +### Forgejo +```sh +# database +rclone cat archive:era5-thermograph/backups/forgejo/db/.dump.age \ + | sudo age -d -i /etc/thermograph/age.key \ + | docker exec -i forgejo_db sh -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_restore -U $POSTGRES_USER -d $POSTGRES_DB --clean --if-exists' +# data volume (repos/lfs/config) — restore into the volume with Forgejo stopped +rclone cat archive:era5-thermograph/backups/forgejo/data/.tar.age \ + | sudo age -d -i /etc/thermograph/age.key \ + | docker run --rm -i -v forgejo_forgejo_data:/data busybox tar -C /data -xf - +``` + +## Verify a backup is restorable (quick) +```sh +rclone cat archive:era5-thermograph/backups/db/prod/.dump.age \ + | sudo age -d -i /etc/thermograph/age.key | head -c 5 # -> PGDMP = valid custom-format dump +``` + +## Manual run +`workflow_dispatch` the "Ops cron (backup + IndexNow)" workflow, or run the same +`age | rclone rcat` pipeline by hand (see the workflow script). diff --git a/infra/deploy/secrets/beta.yaml b/infra/deploy/secrets/beta.yaml index d7a74c3..d5a63be 100644 --- a/infra/deploy/secrets/beta.yaml +++ b/infra/deploy/secrets/beta.yaml @@ -16,6 +16,10 @@ THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:sTtkAbMVUtwvRssCZJa2PWUjlfnYS THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:SnFw+pwNgZlxMEhMUH/1daMZlRB2QFNJHtL+4C3S1eFcJJdT2uNGkY7Ym9FPfx1V7cwt/3urHg1oVgdDzYaIal2wye4l5HhKBP5PjtQkDW+Pck24kxul,iv:Ku4MhsrAfyA2Y3CqhGIsYKrjhKQ4cqAYjCqjFeEvBWE=,tag:hw42iS+79D7gNf+BPTYM0g==,type:str] THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:ox+hHiFMTPZqYwICDSwPwcT13ZDSKiGcAVHfVVMNYQ==,iv:DssV/olPXERpdmXNMmzjVYvhn/0zmuSp2XlTs42pIi0=,tag:4YHXI1BFbGZ6JsdXqJuf3w==,type:str] REGISTRY_TOKEN: ENC[AES256_GCM,data:pC1Zzs56la8qjzWcbkuKyeVHwS5WfIe+mrLm5Sdi/LKYziAvRU5cOA==,iv:57PlJEBdgwNJsfFdNOGqAnKk8l6zpmMSWYFEKS6pn+4=,tag:pPftNHUVU6+CY2nWMaXO6w==,type:str] +THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:QSMp9yUEf9qMdVrSkKB56WzcW39/2QLPgHS5pRY/,iv:0YeVVf196LFl/cKhCUezVFlteA0fpC7BYDFuSH4Ihh4=,tag:HfHvrlkjT39bCKrM2ZN8WQ==,type:str] +THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:H3/dfY5HpRPnz6BvokHVZg==,iv:1ExIBSVFSuGXThwG1d5MymPLQYUXkm0iUjZx0A8+o6c=,tag:e+pu/axaS7gKoptHBPDzhA==,type:str] +THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:62UeBWrSsF/sl8zwUjOoSu0Tx2Q6JBPTI6bhJ5JewFM=,iv:IMnYZyTcevRZD/EXPNymKLVEUMSKHDcWhdXEBItpg9E=,tag:UC5pmSuoJ6WJxaMfOVe4aw==,type:str] +THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:9J+L4O4mZyWBr7tc2CgV8w7Jbr3lFsxMFO6Zg8+813c=,iv:t4m1BXMnSzk0K0TxuKC3UxpYZy34RWNNnGFVyLdi23s=,tag:Kwetz4YxLWsqDA0To7z8qg==,type:str] sops: age: - enc: | @@ -27,7 +31,7 @@ sops: 0+PQdtDxom72BqrAoj5lhepxW8YWPqQIRXsVS/XYQEQfPJuuuiRMKw== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-22T03:17:28Z" - mac: ENC[AES256_GCM,data:npF+ngF4Khj3NJ2PNFcQjQYIrZVpdPM0drk43pAI6HfFaynX8JOqRFZRsH0sCfMoPkXB/ffCJxla1Tt3Mu3zgKTNSWcn+mjzl+dYvlruY1VylHLS+GnTO9zKeQcwbrYzTfZNWzQhm6/mw8KCL3791U/xNHwJc6g5VxRUWl6TBdo=,iv:JLzY8rm33NLG8CqyAlU6vBhlODUiRipKu9daoRvgyzg=,tag:yIsxVZAnw8FD7Bt/x55wMQ==,type:str] + lastmodified: "2026-07-23T20:22:14Z" + mac: ENC[AES256_GCM,data:i8gfHIslHMWZe0wxGlHL+5doqXR+48lr1gV5Doo5ynChLQaPWFE1n98JNEOBqm3wDbpjhARgxm/kCMkeXBJvnMwIBCWKhuwYACDR1mY1v63lgW1GJsIYWRsmR0uQ8Rd/dYwjP8qywf54pDJdjYAJjnvhmMaDty1WTsin3OABmWs=,iv:9rUO42Esh/Msec9lAgcxHFOYqau6VCTUWDMxF9WXQ9A=,tag:a0e6Y0kYdySOCy29Yxjf6w==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/secrets/prod.yaml b/infra/deploy/secrets/prod.yaml index af4aa27..34ba72c 100644 --- a/infra/deploy/secrets/prod.yaml +++ b/infra/deploy/secrets/prod.yaml @@ -24,6 +24,10 @@ THERMOGRAPH_DISCORD_BOT_TOKEN: ENC[AES256_GCM,data:zfVJLH8ddct2Jvstl/olZLQ9SM15c REGISTRY_TOKEN: ENC[AES256_GCM,data:3VETZR026YXvWA7DLly4/mz+hqEK7vkEMwKlDKU6CQZeZiFpbQXWig==,iv:ixZJi2VWAZak6dFPvVyErVRanp9bzqXyV3MVa4QsBuU=,tag:Pc2Mdw58S0tSdkgJ5yuVNw==,type:str] THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Kg==,iv:CPYN8aCu79PHaz3Zj6OgqyJhXdHLt0tLodgML2NuwQM=,tag:1B3IhlfjCwSlPq7FnKGeRg==,type:str] THERMOGRAPH_DISCORD_WEATHER_CHANNEL: ENC[AES256_GCM,data:WIxYcoOnBDdDcNj8uJ5eUqXjVQ==,iv:l6kU5PGVcW2Nv7EqBUGCRxBzaJOJ3AW1nEEH33aAbmI=,tag:/KgdCKmrlQioIWdz21PcTg==,type:str] +THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:0OwwhGiJB3dXGZaD0ObV84ZJg4XLIByIrzaCYZMQ,iv:AqerRn2lD6u90zfGneFT/cq3LAnHqBZH3ICkAuvlztk=,tag:FJAYgILUGwJFLdYtqPKWnw==,type:str] +THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:sernOo7VeKi22X1G+WqGcg==,iv:uLiChXwP+gqlJv07hwZmKUraxdp2YwFbQgYHQNTH7C0=,tag:EtUgNayYrMx3wYprhEE4Hw==,type:str] +THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:RDD4MsWSf0HBEHBSibjNSZFbOm867XdPS6TdACN61Is=,iv:IJSbX/B94Y2OLqaskEp5IOZh4snNzTwPEPkl+u/KCxc=,tag:Xka/7TSPuBD97l04CR/D0A==,type:str] +THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:QD4JYKTbshcm0yx3YTHr2VUDq23ABjwk3wvMUSIDs0s=,iv:/JyMzcj1LWcxE8ioqCmkjZIFnmTBNEvaYXrPCFztETM=,tag:EnnkTQjsotXaxOd60mlOTw==,type:str] sops: age: - enc: | @@ -35,7 +39,7 @@ sops: PDXCB8EqKwdC3BnKlpD9ptYcuPHVUSgve1wIZYJ7iHIEhW9Kc6RXOw== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-23T04:33:06Z" - mac: ENC[AES256_GCM,data:BjNMEudsS9ZViR55OjSXu5qrOJ9yu2kCGx61qu4vE1ZZJIZtnnyZ3CfpsPygvLLhiFRj76xcqFnuSqTdr17BCEHz3UzsHFv0vBi17lZ5ksN5Oz+9wr+3Pg3TvxC0VF2VHJVP84VH61CmB5+G4WOms8WqLtmIGSLYcTVtcIFoUhk=,iv:jTEB8Mxg8qIyNfhpSgJtUvULVx4+XFrPLyQKhNcbkTg=,tag:SIyvNltdGCMfDGvi4kD3MA==,type:str] + lastmodified: "2026-07-23T20:22:14Z" + mac: ENC[AES256_GCM,data:flh82V4mBV7BM5lgfzJfLona87YE02/LcPEcbmWPMmlql/ZgipIPl7uCdM8MNDLyaMgXPfxmEIb20Ly9yT5b4zp2QGqHBlfc58M1wRmTOVJFuSV4SUEBJTMOv0zqpnr0IGRpJw30QAu4xq0op7lEZo9WXMIvOxyrA591viwlX08=,iv:C2TnbWL5OvIPQFMghOSTQaoh2hI3hcRNG61+V9OIl5A=,tag:hF5K+xzvdfN4d/ZIjKJEpA==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2