From d138f00a2077a5e9c7e0fab9d74b437fd43e6101 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 06:56:38 +0000 Subject: [PATCH 1/2] =?UTF-8?q?infra:=20split=20the=20estate=20into=20vps1?= =?UTF-8?q?/vps2=20=E2=80=94=20beta=20joins=20prod,=20dev=20gets=20a=20hom?= =?UTF-8?q?e=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/hooks/README.md | 16 +- .claude/hooks/prod-guard.sh | 17 +- .forgejo/workflows/deploy.yml | 100 +++-- .forgejo/workflows/infra-sync.yml | 97 +++- .forgejo/workflows/observability-validate.yml | 3 +- .forgejo/workflows/ops-cron.yml | 147 +++--- CLAUDE.md | 60 ++- CUTOVER-NOTES.md | 35 ++ README.md | 7 +- docs/onboarding/01-orientation.md | 51 ++- docs/onboarding/02-setup.md | 13 +- docs/onboarding/03-repo-map.md | 46 +- docs/onboarding/04-backend.md | 7 +- docs/onboarding/05-frontend.md | 2 +- docs/onboarding/07-ci-and-release.md | 85 ++-- docs/onboarding/08-infra-secrets.md | 209 ++++++--- docs/onboarding/09-observability.md | 71 +-- docs/onboarding/10-recipes.md | 19 +- docs/onboarding/11-traps.md | 61 ++- docs/onboarding/README.md | 11 +- infra/.claude/skills/key-gaps/SKILL.md | 25 +- infra/.env.example | 17 +- infra/ACCESS.md | 158 ++++--- infra/CLAUDE.md | 96 +++- infra/DEPLOY-DEV.md | 233 ++++++---- infra/DEPLOY.md | 188 ++++---- infra/Makefile | 10 +- infra/README.md | 87 ++-- infra/deploy/Caddyfile | 120 ----- infra/deploy/Caddyfile.vps1 | 61 +++ infra/deploy/Caddyfile.vps2 | 145 ++++++ infra/deploy/RUNBOOK-vps1-vps2-cutover.md | 420 ++++++++++++++++++ infra/deploy/backup/README.md | 41 +- infra/deploy/db/init/20-tuning.sh | 7 +- infra/deploy/db/provision-env-db.sh | 161 +++++++ infra/deploy/deploy-dev.sh | 99 +++-- infra/deploy/deploy.sh | 96 +++- infra/deploy/env-topology.sh | 237 ++++++++++ infra/deploy/forgejo/README.md | 96 ++-- infra/deploy/forgejo/caddy-git.conf | 11 +- infra/deploy/forgejo/docker-stack.yml | 31 +- infra/deploy/forgejo/register-lan-runner.sh | 27 +- infra/deploy/openmeteo/README.md | 10 +- infra/deploy/provision-dev-lan.sh | 30 -- infra/deploy/provision-dev.sh | 79 ++++ infra/deploy/provision-mail.sh | 62 ++- infra/deploy/provision-secrets.sh | 46 +- infra/deploy/render-secrets.sh | 27 +- infra/deploy/secrets/README.md | 142 ++++-- infra/deploy/secrets/beta.yaml | 30 +- infra/deploy/secrets/dev.yaml | 38 +- infra/deploy/secrets/seed-encrypt-on-host.sh | 22 +- infra/deploy/secrets/seed-from-live.sh | 22 +- infra/deploy/stack/deploy-stack.sh | 146 ++++-- infra/deploy/stack/lb/Caddyfile.beta | 34 ++ infra/deploy/stack/thermograph-beta-stack.yml | 278 ++++++++++++ infra/deploy/swarm/README.md | 71 +-- infra/deploy/swarm/firewall-swarm.sh | 14 +- infra/deploy/swarm/init-swarm.sh | 11 +- infra/deploy/swarm/join-swarm.sh | 4 +- infra/deploy/swarm/label-forge-node.sh | 4 +- infra/deploy/swarm/setup-wireguard.sh | 6 +- infra/deploy/thermograph.env.example | 18 +- infra/docker-compose.dev.yml | 41 +- infra/ops/ICEBERG-HANDOFF.md | 32 ++ infra/ops/README.md | 61 ++- infra/ops/dbq.sh | 69 ++- infra/ops/iceberg.sh | 23 +- infra/terraform/README.md | 75 +++- infra/terraform/main.tf | 36 +- .../modules/thermograph-host/variables.tf | 2 +- infra/terraform/outputs.tf | 2 +- infra/terraform/terraform.tfvars.example | 168 ++++--- infra/terraform/variables.tf | 146 +++--- observability/.env.example | 11 +- observability/CLAUDE.md | 35 +- observability/README.md | 150 +++++-- observability/alloy/config.alloy | 159 ++++++- .../alloy/docker-compose.agent.beta.yml | 24 + observability/alloy/docker-compose.agent.yml | 53 ++- observability/caddy-grafana.conf | 13 +- observability/docker-compose.yml | 21 +- .../provisioning/alerting/contact-points.yml | 9 +- .../provisioning/alerting/rules-prod.yml | 42 +- observability/loki/config.yml | 11 +- 85 files changed, 4218 insertions(+), 1482 deletions(-) delete mode 100644 infra/deploy/Caddyfile create mode 100644 infra/deploy/Caddyfile.vps1 create mode 100644 infra/deploy/Caddyfile.vps2 create mode 100644 infra/deploy/RUNBOOK-vps1-vps2-cutover.md create mode 100755 infra/deploy/db/provision-env-db.sh create mode 100644 infra/deploy/env-topology.sh delete mode 100755 infra/deploy/provision-dev-lan.sh create mode 100755 infra/deploy/provision-dev.sh create mode 100644 infra/deploy/stack/lb/Caddyfile.beta create mode 100644 infra/deploy/stack/thermograph-beta-stack.yml create mode 100644 observability/alloy/docker-compose.agent.beta.yml diff --git a/.claude/hooks/README.md b/.claude/hooks/README.md index 0075e86..afec666 100644 --- a/.claude/hooks/README.md +++ b/.claude/hooks/README.md @@ -13,7 +13,7 @@ They are wired up in `.claude/settings.json`. ## Why prod-guard exists -`agent` has passwordless sudo on prod and beta, and the operator's global settings +`agent` has passwordless sudo on vps1 and vps2, and the operator's global settings allow `Bash(ssh prod:*)` outright. Nothing about `ssh prod 'docker service rm …'` goes through a pull request, so CI cannot see it — before this hook, any session in any directory could delete production with no prompt. @@ -24,9 +24,17 @@ is wrong by construction — the first destructive verb nobody thought of sails straight through. Being wrong in the "ask" direction costs a keystroke; being wrong the other way costs production. -Beta is guarded as strictly as prod: it serves beta.thermograph.org *and* hosts -Forgejo, so a destructive command there takes out git, CI and the registry at once. -The LAN dev box is deliberately unguarded. +Beta is guarded as strictly as prod — more strictly than the names alone suggest, +now: vps2 (`169.58.46.181`) runs **both** prod and beta as separate Swarm stacks on +the same manager, sharing one TimescaleDB instance, so a "beta" command there has +prod's blast radius (see `infra/deploy/env-topology.sh`). vps1 (`75.119.132.91`) is +the other guarded host: Forgejo (git + CI + registry) and Grafana/Loki live there, +plus the `dev` environment, so a destructive command there takes out git, CI and the +registry at once. Mesh IPs did not move in the vps1/vps2 split (vps1 is still +`10.10.0.2`, vps2 is still `10.10.0.1`); what moved is which environment runs where +— `HOST_TOKEN_RE` in `prod-guard.sh` recognises `vps1`/`vps2` alongside the +environment names (`prod`/`beta`) and the raw IPs, so a command naming the host +either way still gets caught. The LAN dev box is deliberately unguarded. ## Changing the classifier diff --git a/.claude/hooks/prod-guard.sh b/.claude/hooks/prod-guard.sh index 5d81dda..319eaf3 100755 --- a/.claude/hooks/prod-guard.sh +++ b/.claude/hooks/prod-guard.sh @@ -19,9 +19,15 @@ # That is also what happens if this script errors or jq is missing, so a bug here # fails toward the prompt rather than toward silent execution. # -# Beta is guarded as strictly as prod: it serves beta.thermograph.org AND hosts -# Forgejo, so a destructive command there can take out the git server, CI and the -# container registry at once. +# Beta is guarded as strictly as prod -- more strictly than the names alone +# suggest, now: vps2 (169.58.46.181) runs prod AND beta as two Swarm stacks on +# the SAME manager, sharing the same TimescaleDB instance, so a "beta" command +# there has PROD's blast radius (shared Docker host, shared Postgres server -- +# see infra/deploy/env-topology.sh). vps1 (75.119.132.91) is the other guarded +# host: Forgejo (git + CI + registry), Grafana/Loki, and the dev environment -- +# a destructive command there can take out the git server, CI and the container +# registry at once. Mesh IPs did NOT move in the vps1/vps2 split (vps1 is still +# 10.10.0.2, vps2 is still 10.10.0.1); what moved is which environment runs where. set -uo pipefail payload=$(cat) @@ -32,7 +38,10 @@ tool=$(printf '%s' "$payload" | jq -r '.tool_name // ""') # An ssh target naming a live host, matched as a whole token after any `user@`. # Backslashes are doubled because awk -v processes escape sequences in the value # before the regex ever sees it; a single \. arrives as a bare dot (and warns). -readonly HOST_TOKEN_RE='^(prod|beta|169\\.58\\.46\\.181|75\\.119\\.132\\.91|10\\.10\\.0\\.[12])$' +# vps2 (169.58.46.181) carries BOTH prod and beta; vps1 (75.119.132.91) carries +# Forgejo/Grafana/dev -- "vps1"/"vps2" are recognised alongside the environment +# names and raw IPs so a command naming the host either way still gets caught. +readonly HOST_TOKEN_RE='^(prod|beta|vps1|vps2|169\\.58\\.46\\.181|75\\.119\\.132\\.91|10\\.10\\.0\\.[12])$' emit() { # $1=allow|ask|deny $2=reason jq -n --arg d "$1" --arg r "$2" \ diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index ceae65e..f28931f 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -10,16 +10,32 @@ name: Deploy # parameterised, so the duplication bought nothing and cost six files to keep in # step. # -# The two *-deploy-dev.yml workflows are NOT ported. They were documented as -# inert: they call the monorepo layout at ~/thermograph-dev on the LAN box, -# which is still a split-era thermograph-infra checkout, so that path does not -# exist there. LAN dev is a local `make dev-up` concern, not a CI environment. +# DEV IS A REAL DEPLOY TARGET AGAIN. It previously was not: dev meant a +# sudo-free compose stack on the operator's desktop, and the two +# *-deploy-dev.yml workflows that drove it were inert. Dev now lives on vps1 at +# /opt/thermograph-dev — a normal fleet host, reached over SSH exactly like beta +# and prod — so it gets a leg here. +# +# SECRETS ARE KEYED BY HOST, NOT BY ENVIRONMENT. `SSH_*` used to mean "beta" and +# `PROD_SSH_*` "prod", which worked only while each environment owned a box. It +# stopped being true in two directions at once: vps2 now runs beta AND prod, and +# the box `SSH_*` pointed at is now vps1, which runs dev, Forgejo and Grafana. +# The old names would have made "the beta secret" and "the Forgejo box secret" +# the same value by accident — the ops-cron file already records one incident +# caused by exactly that conflation. So: VPS1_SSH_* and VPS2_SSH_*, named for +# the machine, and the environment is passed separately as THERMOGRAPH_ENV. +# +# BETA AND PROD DEPLOYING TO ONE BOX DO NOT RACE. They have separate checkouts +# (/opt/thermograph-beta and /opt/thermograph), so the `git reset --hard` in one +# cannot pull the tree out from under the other, and deploy.sh's flock is per +# checkout. The concurrency groups below stay keyed by ref+service, which keeps +# two pushes to the SAME branch serialised — the property that actually matters. # # DELIBERATELY BORING EXPRESSIONS. No dynamic matrix (fromJSON), no # `cond && secrets.A || secrets.B` ternary. Those are GitHub idioms that a # Forgejo/act runner may evaluate differently, and the failure mode here is # "production does not deploy" or, worse, "deploys with an empty SSH host". The -# two environments therefore get two explicit, mutually exclusive steps. +# three environments therefore get three explicit, mutually exclusive steps. # # What is preserved from the originals, all of it load-bearing: # - fetch-depth: 0, because the image tag is keyed to the LAST COMMIT THAT @@ -29,14 +45,18 @@ name: Deploy # - The 12-hex truncation, matching build-push exactly. # - Per-service, per-environment concurrency with cancel-in-progress: false -- # a half-finished deploy must never be cancelled by a newer one. -# - Separate PROD_SSH_* credentials, so a beta credential leak cannot reach -# prod. +# - Separate credentials per host. Note what this does and does not buy now: +# it keeps vps1 (Forgejo, its CI, and whatever unreviewed branch dev is +# running) away from vps2 entirely. It no longer puts a host boundary +# between beta and prod, because they share vps2 by design — that boundary +# is now at the database (separate roles and databases) and the filesystem +# (separate checkouts and rendered env files). # - appleboy/ssh-action by full URL; it is not mirrored in Forgejo's default # action registry. on: push: - branches: [main, release] + branches: [dev, main, release] paths: - 'backend/**' - 'frontend/**' @@ -71,10 +91,15 @@ jobs: run: | set -euo pipefail - # Branch selects the environment. main -> beta, release -> prod. + # Branch selects the environment; the environment selects the host and + # the script path. dev -> dev (vps1), main -> beta (vps2), + # release -> prod (vps2). The paths differ because each environment + # has its own checkout — two environments on vps2 must never share + # one — and dev has its own entry point for its secrets policy. case "${{ github.ref_name }}" in - main) environment=beta ;; - release) environment=prod ;; + dev) environment=dev ; script=/opt/thermograph-dev/infra/deploy/deploy-dev.sh ;; + main) environment=beta ; script=/opt/thermograph-beta/infra/deploy/deploy.sh ;; + release) environment=prod ; script=/opt/thermograph/infra/deploy/deploy.sh ;; *) echo "::error::Deploy triggered on unexpected ref '${{ github.ref_name }}'"; exit 1 ;; esac @@ -105,6 +130,7 @@ jobs: { echo "environment=$environment" + echo "script=$script" echo "changed=$changed" echo "tag=$tag" } >> "$GITHUB_OUTPUT" @@ -126,38 +152,58 @@ jobs: echo "==> ${{ matrix.service }} -> $environment | changed=$changed | tag=$tag" - - name: Deploy to beta + - name: Deploy to dev (vps1) + if: steps.plan.outputs.changed == 'true' && steps.plan.outputs.environment == 'dev' + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} + envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG,THERMOGRAPH_ENV + script: ${{ steps.plan.outputs.script }} + env: + SERVICE: ${{ matrix.service }} + THERMOGRAPH_ENV: dev + BACKEND_IMAGE_TAG: ${{ matrix.service == 'backend' && steps.plan.outputs.tag || '' }} + FRONTEND_IMAGE_TAG: ${{ matrix.service == 'frontend' && steps.plan.outputs.tag || '' }} + + - name: Deploy to beta (vps2) if: steps.plan.outputs.changed == 'true' && steps.plan.outputs.environment == 'beta' uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} - envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG - script: /opt/thermograph/infra/deploy/deploy.sh + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG,THERMOGRAPH_ENV + script: ${{ steps.plan.outputs.script }} env: SERVICE: ${{ matrix.service }} + # THERMOGRAPH_ENV is what makes this a BETA deploy rather than a prod + # one: same host, same credentials, same script — the environment is + # the only thing that differs, and deploy.sh refuses to run if it + # disagrees with the checkout it was invoked from. + THERMOGRAPH_ENV: beta # Only the matching one is read by deploy.sh for a single-service roll; # the other stays empty and the persisted .image-tags.env supplies the # sibling's live tag, so this roll never disturbs it. BACKEND_IMAGE_TAG: ${{ matrix.service == 'backend' && steps.plan.outputs.tag || '' }} FRONTEND_IMAGE_TAG: ${{ matrix.service == 'frontend' && steps.plan.outputs.tag || '' }} - - name: Deploy to prod + - name: Deploy to prod (vps2) if: steps.plan.outputs.changed == 'true' && steps.plan.outputs.environment == 'prod' uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - # A completely separate secret set from beta's, deliberately: a beta - # credential leak must not reach prod. - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} - envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG - script: /opt/thermograph/infra/deploy/deploy.sh + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG,THERMOGRAPH_ENV + script: ${{ steps.plan.outputs.script }} env: SERVICE: ${{ matrix.service }} + THERMOGRAPH_ENV: prod BACKEND_IMAGE_TAG: ${{ matrix.service == 'backend' && steps.plan.outputs.tag || '' }} FRONTEND_IMAGE_TAG: ${{ matrix.service == 'frontend' && steps.plan.outputs.tag || '' }} diff --git a/.forgejo/workflows/infra-sync.yml b/.forgejo/workflows/infra-sync.yml index 5c6acfa..8f86697 100644 --- a/.forgejo/workflows/infra-sync.yml +++ b/.forgejo/workflows/infra-sync.yml @@ -2,65 +2,114 @@ name: Sync infra to hosts # The infra DOMAIN's own pipeline (new with the monorepo -- the split era had # no infra deploy workflow at all; infra changes rode along lazily with the -# next app deploy's `git reset`). On a push to `main` touching infra/**, SSH -# to beta AND prod, fast-forward the host's /opt/thermograph monorepo checkout -# and re-render /etc/thermograph.env from the SOPS vault -- so a compose edit -# or a secret rotation lands push-button instead of waiting for the next app -# deploy. +# next app deploy's `git reset`). On a push touching infra/**, SSH to every +# host, fast-forward each ENVIRONMENT's monorepo checkout and re-render that +# environment's env file from the SOPS vault -- so a compose edit or a secret +# rotation lands push-button instead of waiting for the next app deploy. # # Deliberately does NOT roll any service: image tags are the app domains' -# axis, not infra's. A compose change that must recreate containers takes -# effect on the next app deploy, or a by-hand +# axis, not infra's. A compose or stack change that must recreate containers +# takes effect on the next app deploy, or a by-hand # `SERVICE=all ... infra/deploy/deploy.sh` run (see that script's header). # -# Both hosts track infra via `main` (the split-era model, unchanged): prod's -# *app images* are staged by the `release` branch, but its checkout follows -# main -- deploy.sh's own BRANCH default encodes the same thing. +# THREE CHECKOUTS, TWO HOSTS. This is the shape the vps1/vps2 split forces: +# +# vps1 /opt/thermograph-dev branch dev -> /etc/thermograph.env +# vps2 /opt/thermograph-beta branch main -> /etc/thermograph-beta.env +# vps2 /opt/thermograph branch main -> /etc/thermograph.env +# +# The two checkouts on vps2 are separate directories on purpose: a `git reset +# --hard` for beta must not be able to move prod's tree, and each environment +# renders its own env file from its own vault file. Because the paths differ, +# the beta and prod jobs below can safely run at the same time on one box. +# +# Beta and prod both track `main` (infra is not environment-staged -- only app +# IMAGES are, via tags). Dev tracks `dev`, which is why this workflow now +# triggers on both branches: a dev-branch infra change has to reach the dev +# checkout, and only that one. +# +# Each `render_thermograph_secrets` call passes the environment name and the +# output path EXPLICITLY. It used to rely on the host's +# /etc/thermograph/secrets-env marker, which cannot answer the question on vps2 +# -- one marker, two environments. Passing them here means a beta sync can +# never render prod's vault, and vice versa. on: push: - branches: [main] + branches: [dev, main] paths: ['infra/**'] workflow_dispatch: {} jobs: + sync-dev: + # Only the dev branch feeds the dev checkout. + if: github.ref_name == 'dev' + runs-on: docker + concurrency: + group: infra-sync-dev + cancel-in-progress: false + steps: + - name: Sync dev checkout on vps1 + re-render secrets + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} + script: | + set -euo pipefail + cd /opt/thermograph-dev + git fetch --prune origin dev + git reset --hard origin/dev + if [ -f infra/deploy/render-secrets.sh ]; then + . infra/deploy/render-secrets.sh + # SKIP_COMMON is dev's standing rule, not a preference: common.yaml + # is the fleet's shared production credential set, and vps1 also + # runs Forgejo and its CI. See render-secrets.sh. + THERMOGRAPH_SECRETS_SKIP_COMMON=1 \ + render_thermograph_secrets /opt/thermograph-dev/infra dev /etc/thermograph.env + fi + echo "synced to $(git log --oneline -1)" + sync-beta: + if: github.ref_name == 'main' runs-on: docker concurrency: group: infra-sync-beta cancel-in-progress: false steps: - - name: Sync beta checkout + re-render secrets + - name: Sync beta checkout on vps2 + re-render secrets uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail - cd /opt/thermograph + cd /opt/thermograph-beta git fetch --prune origin main git reset --hard origin/main if [ -f infra/deploy/render-secrets.sh ]; then . infra/deploy/render-secrets.sh - render_thermograph_secrets /opt/thermograph/infra + render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env fi echo "synced to $(git log --oneline -1)" sync-prod: + if: github.ref_name == 'main' runs-on: docker concurrency: group: infra-sync-prod cancel-in-progress: false steps: - - name: Sync prod checkout + re-render secrets + - name: Sync prod checkout on vps2 + re-render secrets uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph @@ -68,6 +117,6 @@ jobs: git reset --hard origin/main if [ -f infra/deploy/render-secrets.sh ]; then . infra/deploy/render-secrets.sh - render_thermograph_secrets /opt/thermograph/infra + render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env fi echo "synced to $(git log --oneline -1)" diff --git a/.forgejo/workflows/observability-validate.yml b/.forgejo/workflows/observability-validate.yml index 175eafb..a3d9030 100644 --- a/.forgejo/workflows/observability-validate.yml +++ b/.forgejo/workflows/observability-validate.yml @@ -1,6 +1,7 @@ name: Validate observability stack -# The observability DOMAIN deploys by hand (`docker compose up -d` on beta + +# The observability DOMAIN deploys by hand (`docker compose up -d` on vps1 — +# the monitoring host, not the beta environment, which runs on vps2 now — plus # the Alloy agent on each node) with no build step, so nothing caught a # malformed compose file, a broken dashboard JSON, or an unparseable config # until it failed live on the host. This is that missing guard: it parses diff --git a/.forgejo/workflows/ops-cron.yml b/.forgejo/workflows/ops-cron.yml index 7ee5f20..112d0be 100644 --- a/.forgejo/workflows/ops-cron.yml +++ b/.forgejo/workflows/ops-cron.yml @@ -6,19 +6,32 @@ name: Ops cron (backup + IndexNow) # classification table in # thermograph-docs/architecture/repo-topology-and-infrastructure.md §7. # -# Both jobs SSH into the prod host and run inside the already-running compose -# stack (docker compose exec), the same way deploy.sh already runs its own -# post-deploy IndexNow ping -- no new network exposure, no separate dependency -# install. Runs on the `docker` label (the always-on Swarm-hosted runner -# deploy/forgejo/ stood up). +# The jobs SSH into a host and run inside the already-running stack (docker +# exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new +# network exposure, no separate dependency install. Runs on the `docker` label +# (the always-on Swarm-hosted runner deploy/forgejo/ stood up). # -# Targets PROD via the PROD_SSH_* secrets (prod = 169.58.46.181, `agent` user, -# in the docker group so no sudo needed; /etc/thermograph.env is agent-readable). -# These are the same secrets deploy-prod.yml uses for the release->prod deploy -- -# NOT the SSH_* secrets, which point at BETA (deploy.yml's `main`->beta path). An -# earlier revision reused SSH_* here, so the "prod" backup was silently dumping -# beta; prod itself had no backup at all. The prod database is the one that must -# be backed up, so both jobs use PROD_SSH_*. +# WHICH BOX EACH JOB TALKS TO, and why the secret names say so: +# +# backup, indexnow -> VPS2_SSH_* (vps2 = 169.58.46.181: prod AND beta, the +# shared TimescaleDB, Postfix, the backups) +# forgejo-backup -> VPS1_SSH_* (vps1 = 75.119.132.91: Forgejo, Grafana, +# Loki, and the dev environment) +# +# The secrets are keyed by HOST rather than by environment because an +# environment no longer implies a machine: vps2 runs two of them. The previous +# names encoded the opposite assumption and had already caused one incident -- +# an early revision used SSH_* here, which meant "beta", so the job labelled +# "prod backup" was silently dumping beta while prod had no backup at all. The +# same conflation is why Forgejo's backup used to be described as running "on +# beta": beta and Forgejo happened to share a box, so one secret served both +# meanings. It does not any more. +# +# BOTH APPLICATION DATABASES ARE BACKED UP. prod and beta are separate databases +# (`thermograph` and `thermograph_beta`) on one shared instance. Dumping only +# the prod database would recreate the original failure in a new shape: a job +# that looks like "the backup" while one environment's data is silently +# uncovered. The loop below dumps each, to its own off-box prefix. # Monorepo port: the compose file now lives under infra/ of the host's # /opt/thermograph monorepo checkout, so every docker-compose invocation cd's @@ -44,22 +57,22 @@ jobs: group: ops-backup cancel-in-progress: false steps: - - name: Dump the prod database over SSH + - name: Dump both application databases over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed - # into the remote script via `envs:` (the host env file isn't a reliable - # source across boxes -- beta's /etc/thermograph.env isn't readable by the - # deploy user -- so the CI secret is the uniform home, like PROD_SSH_*). + # into the remote script via `envs:` (a host env file isn't a reliable + # source across boxes -- on vps1 it isn't readable by the deploy user -- + # so the CI secret is the uniform home, like the VPS*_SSH_* pairs). env: S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} S3_BUCKET: ${{ secrets.S3_BUCKET }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail @@ -73,23 +86,45 @@ jobs: backup_dir="$HOME/thermograph-backups" mkdir -p "$backup_dir" stamp="$(date -u +%Y%m%dT%H%M%SZ)" - out="$backup_dir/thermograph-$stamp.dump" - # The db may run under plain compose OR as a Swarm stack task - # (prod post-cutover); resolve the container either way so the - # backup survives the deploy-mode switch. + # The db may run under plain compose OR as a Swarm stack task; + # resolve the container either way so the backup survives a + # deploy-mode switch. One instance now serves both environments. dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1) [ -z "$dbc" ] && dbc=$(cd /opt/thermograph/infra && docker compose ps -q db 2>/dev/null | head -1) [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; } - # Write to a .partial and rename on success so a mid-dump failure can - # never leave a truncated file that looks like a good backup. - docker exec "$dbc" pg_dump -U thermograph -d thermograph \ - --format=custom > "$out.partial" - mv "$out.partial" "$out" - echo "wrote $out ($(du -h "$out" | cut -f1))" + # env:database pairs. Both are dumped every night: they are separate + # databases on one instance, and backing up only one would leave the + # other silently uncovered. + for pair in prod:thermograph beta:thermograph_beta; do + envname="${pair%%:*}"; dbname="${pair##*:}" + # A missing database is a HARD failure, never a quiet skip -- a + # backup job that shrugs off an absent database is exactly how an + # environment ends up with no backups and nobody noticing. + if ! docker exec "$dbc" psql -U thermograph -d postgres -tAc \ + "select 1 from pg_database where datname='$dbname'" | grep -q 1; then + echo "!! database '$dbname' ($envname) does not exist on this instance." + if [ "$envname" = prod ]; then + echo "!! That is prod's own database on prod's own instance — this is not a" + echo "!! provisioning gap, something is badly wrong. Do not 'fix' it by creating" + echo "!! an empty database; find out where the real one went." + else + echo "!! provision it first: sudo bash /opt/thermograph-beta/infra/deploy/db/provision-env-db.sh $envname" + fi + exit 1 + fi + out="$backup_dir/$dbname-$stamp.dump" + # Write to a .partial and rename on success so a mid-dump failure can + # never leave a truncated file that looks like a good backup. + docker exec "$dbc" pg_dump -U thermograph -d "$dbname" \ + --format=custom > "$out.partial" + mv "$out.partial" "$out" + echo "wrote $out ($(du -h "$out" | cut -f1))" + dumps="${dumps:-} $envname:$out" + done # The dumps are the disaster-recovery copy, not a versioned # archive -- keep the last 14 days and let the rest age out. - find "$backup_dir" -name 'thermograph-*.dump' -mtime +14 -delete - find "$backup_dir" -name 'thermograph-*.dump.partial' -mtime +1 -delete + find "$backup_dir" -name '*.dump' -mtime +14 -delete + find "$backup_dir" -name '*.dump.partial' -mtime +1 -delete # --- off-box copy to Contabo Object Storage (age-encrypted) --- # Streams the just-written dump through age (to the vault's age # recipient -- the same key each host renders secrets with, so @@ -105,23 +140,33 @@ jobs: RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - s3base="$S3_BUCKET/backups/db/prod" - age -r "$recip" < "$out" | rclone rcat "archive:$s3base/$(basename "$out").age" - echo "off-box: uploaded $(basename "$out").age to $s3base" - rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true + # One prefix per environment, so a restore never has to guess which + # database a dump came from: backups/db/prod/ and backups/db/beta/. + for entry in $dumps; do + envname="${entry%%:*}"; f="${entry##*:}" + s3base="$S3_BUCKET/backups/db/$envname" + age -r "$recip" < "$f" | rclone rcat "archive:$s3base/$(basename "$f").age" + echo "off-box: uploaded $(basename "$f").age to $s3base" + rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true + done else echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)" fi forgejo-backup: - name: Forgejo backup (beta) -> S3 + name: Forgejo backup (vps1) -> S3 runs-on: docker - # Forgejo (the git host + all CI history) lives on beta and had NO backup at + # Forgejo (the git host + all CI history) lives on vps1 and had NO backup at # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS, # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as - # the beta SSH user (SSH_*), which is in the docker group -- so docker exec/run - # need no host sudo, and the data tar runs inside a throwaway container (the - # volume dir is root-owned on the host). rclone + age must be present on beta. + # vps1's SSH user (VPS1_SSH_*), which is in the docker group -- so docker + # exec/run need no host sudo, and the data tar runs inside a throwaway + # container (the volume dir is root-owned on the host). rclone + age must be + # present on vps1. + # + # This job follows FORGEJO, not beta. It used to use the same secret as the + # beta deploy purely because Forgejo and beta shared a box; beta has since + # moved to vps2 and Forgejo has not moved at all. concurrency: group: ops-forgejo-backup cancel-in-progress: false @@ -134,16 +179,16 @@ jobs: S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; } - command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on beta"; exit 1; } - command -v age >/dev/null 2>&1 || { echo "!! age missing on beta"; exit 1; } + command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on vps1"; exit 1; } + command -v age >/dev/null 2>&1 || { echo "!! age missing on vps1"; exit 1; } export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \ RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \ RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ @@ -177,10 +222,10 @@ jobs: - name: Ping IndexNow if the URL set changed uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph/infra diff --git a/CLAUDE.md b/CLAUDE.md index 2e5710a..3b640aa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,24 +15,30 @@ every change, so a stale one is a correctness bug, not a documentation bug. | Branch | Deploys to | Workflow | |---|---|---| | feature branch | nothing | PR into `dev` | -| `dev` | nothing | integration branch only — see below | -| `main` | beta (beta.thermograph.org) | `deploy.yml` | -| `release` | prod (thermograph.org) | `deploy.yml` | +| `dev` | dev (vps1, mesh-only, own Postgres) | `deploy.yml` | +| `main` | beta (beta.thermograph.org, vps2) | `deploy.yml` | +| `release` | prod (thermograph.org, vps2) | `deploy.yml` | `dev`, `main` and `release` are protected: **everything is a PR**, for humans and agents alike. Promotion is one PR per hop, `dev` → `main` → `release`. -**`dev` deploys nowhere.** It is purely the integration branch that feature PRs -land on before promotion to `main`. The two LAN-dev deploy workflows were deleted -rather than kept: they called a monorepo path that does not exist on the LAN box -(`~/thermograph-dev` is still a split-era `thermograph-infra` checkout), so they -had been inert since cutover. Run LAN dev locally with `infra/`'s `make dev-up`. +**`dev` is a first-class hosted environment, not just an integration branch.** +It runs on vps1 — the same box as Forgejo and the monitoring stack — with its +own Postgres container, reachable only on the WireGuard mesh +(`10.10.0.2:8137`): no public DNS record, no Caddy site, no TLS. It is deployed +by CI like beta and prod. The desktop hosts no Thermograph environment at all; +`make dev-up` there is a laptop convenience for running the stack locally, not +a deployment target. -**One workflow deploys everything.** `deploy.yml` handles both services and both -environments: the branch selects the environment (`main` → beta, `release` → -prod), a matrix covers backend and frontend, and each leg checks whether this -push actually touched its domain before rolling. `build-push.yml` is the same -shape for images. They replaced six and two near-identical files respectively. +**One workflow deploys everything.** `deploy.yml` handles both services and all +three environments: the branch selects the environment (`dev` → vps1, `main` → +beta on vps2, `release` → prod on vps2), a matrix covers backend and frontend, +and each leg checks whether this push actually touched its domain before +rolling. `build-push.yml` is the same shape for images. `THERMOGRAPH_ENV` is the +input that tells a leg which environment it's deploying — load-bearing on vps2, +which runs beta and prod side by side and has no other way to tell them apart. +See `infra/deploy/env-topology.sh` for the single source of truth on where each +environment's checkout, branch, stack and ports live. ## The deploy contract @@ -44,15 +50,25 @@ SERVICE=backend|frontend|all BACKEND_IMAGE_TAG=sha-<12hex> FRONTEND_IMAGE_TAG= ``` `deploy.sh` resets the host checkout, renders secrets from the SOPS vault, then -either rolls compose services or — if `/etc/thermograph/deploy-mode` contains -`stack` — execs `infra/deploy/stack/deploy-stack.sh`. +either rolls compose services or — if `infra/deploy/env-topology.sh` says the +environment's deploy mode is `stack` — execs `infra/deploy/stack/deploy-stack.sh`. +The old host-wide `/etc/thermograph/deploy-mode` marker survives only as a +fallback for a by-hand run with no environment resolvable any other way; it +cannot describe vps2 alone, since vps2 runs beta and prod side by side. -- **prod runs Swarm.** Its stack is `infra/deploy/stack/thermograph-stack.yml` - (db, web, worker, lake, daemon, frontend, autoscaler, autoscaler-lake). - `deploy-stack.sh` also offers `STACK_TEST=1`: a full parallel rehearsal on - throwaway volumes and ports that cannot touch live data. -- **beta and LAN dev run compose**, from `infra/docker-compose.yml` - (db, backend, lake, daemon, frontend). +- **prod and beta both run Swarm, as two separate stacks co-resident on vps2.** + Prod's is `infra/deploy/stack/thermograph-stack.yml` (db, web, worker, lake, + daemon, frontend, autoscaler, autoscaler-lake). Beta's is the separate + `infra/deploy/stack/thermograph-beta-stack.yml` — its services are prefixed + (`beta-web`, `beta-worker`, …) because Swarm registers a service's short name + as a DNS alias on every network it joins, and beta shares prod's `data` + network to reach the database. Beta has no `db` service of its own: one + TimescaleDB instance serves both, on separate databases and separate + NOSUPERUSER roles. `deploy-stack.sh` also offers `STACK_TEST=1`: a full + parallel rehearsal on throwaway volumes and ports that cannot touch live data. +- **dev runs compose**, from `infra/docker-compose.yml` (db, backend, lake, + daemon, frontend), on its own host (vps1) with its own Postgres container — + the one environment not sharing a database with anything else. - Each service's live tag is persisted host-side, so a single-service roll never disturbs the sibling's running tag. @@ -76,7 +92,7 @@ reunification. - Anything named "prefetch" must never spend the Open-Meteo quota. Nominatim ≤ 1 req/s. - The compose project name is pinned (`name: thermograph` in - `infra/docker-compose.yml`); LAN dev overrides with + `infra/docker-compose.yml`); dev (and a local `make dev-up`) overrides with `COMPOSE_PROJECT_NAME=thermograph-dev`. Don't remove either half — the pinned name is what makes the Swarm stack's external volume names line up. - `CUTOVER-NOTES.md` is the source of truth for what is and isn't live yet. diff --git a/CUTOVER-NOTES.md b/CUTOVER-NOTES.md index 18743d5..cc6ba93 100644 --- a/CUTOVER-NOTES.md +++ b/CUTOVER-NOTES.md @@ -153,3 +153,38 @@ Checked every split-repo branch by dry-run merge; migrated everything real: historical. - Infra `Makefile` local-image targets still assume sibling checkouts — update when first needed locally. + +## vps1/vps2 host-topology re-architecture (landed 2026-07-25, NOT YET EXECUTED) + +**This section documents a second re-architecture, layered on top of the +monorepo cutover above. It is landed IN THE REPO — code, stack files, deploy +scripts, docs — but has NOT been run against the live estate.** The boxes +described elsewhere in this file (prod at `169.58.46.181`, beta at +`75.119.132.91`, a desktop LAN dev box) are still what is actually running +today. Do not treat anything below as live until the cutover runbook has been +executed and this note updated. + +What changes, in one line: hosts are renamed by **role** instead of by +environment, and beta moves onto the same box as prod. + +- **vps1** (mesh `10.10.0.2`, public `75.119.132.91`) — unchanged box, new job. + Keeps Forgejo, Grafana/Loki/Alloy and the `emigriffith.dev` portfolio; gains + the **dev** environment (its own Postgres, plain compose, mesh-only — + `10.10.0.2:8137`, no DNS record, no Caddy site, no TLS). This is the box that + used to be called "beta". +- **vps2** (mesh `10.10.0.1`, public `169.58.46.181`) — unchanged box, new job. + Keeps prod; gains **beta** as a second, co-resident Docker Swarm stack + (prefixed services, own checkout, own env file, own loopback LB ports). One + TimescaleDB instance now serves both prod and beta, on separate databases and + separate roles. This is the box that used to be called "prod". +- **desktop** (mesh `10.10.0.3`) — loses the LAN dev stack and the Forgejo + Actions runner entirely; becomes AI-model hosting plus flex Swarm capacity. + `make dev-up` still works there as a laptop convenience, but it is no longer + an environment. + +The mesh IPs did not move — only which environment lives on which box did. See +`infra/deploy/env-topology.sh` for the single source of truth on per-environment +checkout paths, branches, stack names, ports and DB roles, and +**[`infra/deploy/RUNBOOK-vps1-vps2-cutover.md`](infra/deploy/RUNBOOK-vps1-vps2-cutover.md)** +for the execution runbook. Until that runbook has been run, `CLAUDE.md`, +`README.md` and `docs/onboarding/` describe the target state, not today's. diff --git a/README.md b/README.md index a999ec5..b4fa580 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ for: **per-domain images, per-domain deploys, and an async FE/BE contract**. |---|---|---| | `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `build-push` → image `emi/thermograph/backend`; `deploy` | | `frontend/` | Public client: static JS/CSS + SSR pages | same `build-push` / `deploy` workflows, matrixed by domain; image `emi/thermograph/frontend` | -| `infra/` | Compose (beta, LAN dev) + the Swarm stack (prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | +| `infra/` | Compose (dev, on vps1) + two Swarm stacks co-resident on vps2 (beta, prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | | `observability/` | Loki + Grafana + Alloy stack | `observability-validate` | `thermograph-docs` deliberately **stays its own repo** (ADRs + runbooks, no @@ -36,6 +36,9 @@ The one intentionally *coupled* piece is `pr-build.yml`: a single always-running path-filtered required check would deadlock auto-merge). Branch model (unchanged from the split era): PRs → `dev`, `main` → beta, -`release` → prod; infra tracked via `main` on all hosts. +`release` → prod. Infra isn't environment-staged the same way app images are: +beta's and prod's checkouts (both on vps2) track `main`; dev's checkout (on +vps1) tracks `dev` itself, since it's the one environment that isn't a +rehearsal for something downstream. **Before pointing anything live at this repo, read `CUTOVER-NOTES.md`.** diff --git a/docs/onboarding/01-orientation.md b/docs/onboarding/01-orientation.md index 0194611..b53ad9a 100644 --- a/docs/onboarding/01-orientation.md +++ b/docs/onboarding/01-orientation.md @@ -55,40 +55,59 @@ and `PAYLOAD_VER` is the lever that moves it.** See ## The estate -Four machines on a WireGuard mesh (`10.10.0.0/24`): +Hosts are named by **role**, not by environment, plus the operator's desktop — +all on a WireGuard mesh (`10.10.0.0/24`): | Host | Mesh IP | Public | Runs | |---|---|---|---| -| **prod** | `10.10.0.1` | `169.58.46.181`, thermograph.org | Docker **Swarm** app stack, TimescaleDB, Postfix, backups, Centralis | -| **beta** | `10.10.0.2` | `75.119.132.91`, beta.thermograph.org | **compose** app stack, Forgejo (git + CI + registry), Grafana + Loki | -| **desktop** | `10.10.0.3` | — | LAN dev stack, the Forgejo Actions runner, your editor | +| **vps1** | `10.10.0.2` | `75.119.132.91`, git.thermograph.org, dashboard.thermograph.org | Forgejo (git + CI + registry), Grafana + Loki + Alloy, the `emigriffith.dev` portfolio, and **dev** — its own Postgres, plain compose, **mesh-only** (no public DNS, no Caddy site, no TLS) | +| **vps2** | `10.10.0.1` | `169.58.46.181`, thermograph.org, beta.thermograph.org | **prod** and **beta** as two separate Docker **Swarm** stacks, Centralis, Postfix, backups | +| **desktop** | `10.10.0.3` | — | AI-model hosting (voice-to-text, an upcoming-feature LLM) + flex Swarm capacity. Hosts **no** Thermograph environment — `make dev-up` there is a laptop convenience only | | phone | — | — | alerts | +vps2 runs prod and beta **co-resident on one box**: one TimescaleDB instance +serves both, on separate databases and separate roles (`thermograph` / +`thermograph_beta`, the latter `NOSUPERUSER`/`NOCREATEDB`, `CONNECT` revoked +from `PUBLIC`); separate checkouts, env files and loopback LB ports; beta's +Swarm service names are prefixed (`beta-web`, not `web`) because Swarm +registers a service's short name as a DNS alias on every network it joins, and +beta shares prod's network to reach the database. Beta rehearses prod's actual +orchestrator this way — that's the point of putting it next to prod rather +than next to dev. + +The mesh IPs did not move when this topology landed — vps1 is the box that used +to be called "beta", vps2 the one that used to be called "prod". What moved is +which environment lives where. `infra/deploy/env-topology.sh` is the single +source of truth for every per-environment path, port and role; `THERMOGRAPH_ENV` +(`dev`/`beta`/`prod`) is the input a deploy leg passes, since vps2 can no longer +tell beta and prod apart by which host it's running on. + Two consequences you will hit within the first week: -- **Forgejo is mesh-only.** `git.thermograph.org` resolves publicly to beta's - IP, but beta's Caddy rejects `/v2/*` (the registry API) from outside the mesh. - Any host that pulls images needs `10.10.0.2 git.thermograph.org` in +- **Forgejo is mesh-only.** `git.thermograph.org` resolves publicly to vps1's + IP, but vps1's Caddy rejects `/v2/*` (the registry API) from outside the + mesh. Any host that pulls images needs `10.10.0.2 git.thermograph.org` in `/etc/hosts`. -- **Prod and beta run different orchestrators.** Prod is Swarm - (`infra/deploy/stack/`), beta and LAN dev are compose - (`infra/docker-compose.yml`). `deploy.sh` routes between them by reading - `/etc/thermograph/deploy-mode` on the box. Most of the time you don't care — - but when you're reading logs or naming containers, you do - (see [observability](09-observability.md)). +- **Beta and prod run the same orchestrator now — Swarm, as two stacks on one + box.** Dev is the only environment on compose (`infra/docker-compose.yml`), + and it lives alone on vps1. Most of the time you don't care which stack + you're reading — but when you're reading logs or naming containers, you do + (see [observability](09-observability.md)), and on vps2 you additionally have + to say *which* environment: `deploy.sh` there routes on the explicit + `THERMOGRAPH_ENV` a caller passes, not on which host it's running on. ## How a change travels ``` -PR ──(required check: `gate`)──▶ dev ──▶ LAN dev box +PR ──(required check: `gate`)──▶ dev ──▶ dev (vps1, mesh-only) │ promotion PR ▼ - main ──▶ beta.thermograph.org + main ──▶ beta.thermograph.org (vps2) │ promotion PR ← the owner's call ▼ - release ──▶ thermograph.org + release ──▶ thermograph.org (vps2) ``` `dev`, `main` and `release` are **protected** — no direct pushes, for humans or diff --git a/docs/onboarding/02-setup.md b/docs/onboarding/02-setup.md index 52aa099..3ea84ee 100644 --- a/docs/onboarding/02-setup.md +++ b/docs/onboarding/02-setup.md @@ -172,11 +172,18 @@ ends fail closed. With Discord unconfigured it logs once and runs cron-only. ```bash cd infra make dev-up # docker-compose.yml + docker-compose.dev.yml overlay: - # uncapped CPU, backend published on 0.0.0.0:8137 for the LAN + # uncapped CPU, backend published on 0.0.0.0:8137 for your LAN make dev-down ``` -The dev overlay exports `COMPOSE_PROJECT_NAME=thermograph-dev` so the LAN stack +This is a **laptop convenience**, not the hosted dev environment: the actual +`dev` environment now runs on vps1 with its own Postgres, deployed by CI, and +bound only to the WireGuard mesh (`10.10.0.2:8137`) — never `0.0.0.0`, since +vps1 is a public box. See [Infra and secrets](08-infra-secrets.md). A local +`make dev-up` is fine on `0.0.0.0` because it's your own machine's LAN, not the +public internet. + +The dev overlay exports `COMPOSE_PROJECT_NAME=thermograph-dev` so this stack keeps volumes separate from anything else. **Do not remove either half of the project-name pinning** — `infra/docker-compose.yml` pins `name: thermograph`, and without it running compose from `infra/` derives the project name `infra`, @@ -216,7 +223,7 @@ it will be fresher than this page. | Hook | When | What | |---|---|---| -| `prod-guard.sh` | before Bash / live-host MCP calls | Classifies by **allowlist**: only positively-recognised read-only commands pass; everything else asks. Beta is guarded as strictly as prod — it hosts Forgejo, Grafana *and* beta.thermograph.org. | +| `prod-guard.sh` | before Bash / live-host MCP calls | Classifies by **allowlist**: only positively-recognised read-only commands pass; everything else asks. vps1 is guarded as strictly as vps2 — it hosts Forgejo, Grafana *and* the mesh-only `dev` environment, so a destructive command there takes out git, CI and the registry at once, not just a dev sandbox. | | `secrets-guard.sh` | before Write/Edit | Denies any direct write to `infra/deploy/secrets/*.yaml`. Use `sops edit`. | | `lint-after-edit.sh` | after Write/Edit | shellchecks an edited `*.sh` and feeds findings straight back. Exits quietly if shellcheck is missing. | diff --git a/docs/onboarding/03-repo-map.md b/docs/onboarding/03-repo-map.md index 65cebc4..60a3952 100644 --- a/docs/onboarding/03-repo-map.md +++ b/docs/onboarding/03-repo-map.md @@ -114,24 +114,36 @@ Deep dive: [05-frontend.md](05-frontend.md). ## `infra/` — how and where images run ``` -docker-compose.yml the compose stack (beta + LAN dev): - db, backend, lake, daemon, frontend. +docker-compose.yml the compose stack — dev's, on vps1 (and a local + `make dev-up`): db, backend, lake, daemon, frontend. `name: thermograph` is PINNED — see below. -docker-compose.dev.yml LAN overlay: uncapped, backend on 0.0.0.0:8137 +docker-compose.dev.yml dev overlay: uncapped, backend bound to + `${DEV_BIND_ADDR:-127.0.0.1}` — vps1's mesh IP + (10.10.0.2) at deploy time, never 0.0.0.0 docker-compose.openmeteo.yml self-hosted Open-Meteo overlay (prod-only) Makefile compose orchestration only deploy/ - deploy.sh ★ the single entry point for beta and prod - render-secrets.sh renders /etc/thermograph.env from the SOPS vault + env-topology.sh ★ single source of truth: env → host, checkout, + branch, deploy mode, stack name, env file, LB + ports, DB role/name, service prefix + deploy.sh ★ the single entry point for dev, beta and prod + deploy-dev.sh thin wrapper: routes deploy.sh at dev's compose + overlay and secrets policy + render-secrets.sh renders the env file for a given environment from + the SOPS vault secrets/ the vault: common.yaml, prod/beta/dev.yaml, centralis.prod.yaml, example.yaml - stack/ the SWARM path (live on prod): - thermograph-stack.yml, deploy-stack.sh, - autoscale.sh, the LB + stack/ the SWARM path — both live on vps2: + thermograph-stack.yml (prod), + thermograph-beta-stack.yml (beta, prefixed + services, no `db` of its own), deploy-stack.sh, + autoscale.sh, lb/ (per-environment Caddyfiles) + db/provision-env-db.sh creates each environment's database + role on the + shared TimescaleDB instance + provision-dev.sh one-time bootstrap of the dev environment on vps1 swarm/, forgejo/ mesh + Forgejo/CI-runner provisioning Caddyfile the reverse proxy (path-splits FE vs BE) - provision-*.sh host bootstrap: agent access, dev LAN, mail, secrets - db/init/ TimescaleDB init + tuning + provision-*.sh host bootstrap: agent access, mail, secrets migrations/ hand-run SQL migrations ops/ dbq.sh read-only psql into any environment's db container @@ -141,26 +153,26 @@ ACCESS.md, DEPLOY.md, DEPLOY-DEV.md (partly stale — see traps) ``` **Why the project name is pinned.** Compose creates volumes -`thermograph_pgdata` / `_appdata` / `_applogs`, and the Swarm stack declares +`thermograph_pgdata` / `_appdata` / `_applogs`, and the Swarm stacks declare those exact names as `external: true` at the same mount paths. Running compose from `infra/` without `name: thermograph` derives project `infra`, which makes a whole new stack with empty volumes next to the running one. `deploy-dev.sh` exports `COMPOSE_PROJECT_NAME=thermograph-dev` (env wins over the file key) to -keep LAN dev separate on purpose. Keep both halves. +keep dev separate on purpose. Keep both halves. Deep dive: [08-infra-secrets.md](08-infra-secrets.md). ## `observability/` — the logging stack ``` -docker-compose.yml Loki + Grafana (runs on beta) +docker-compose.yml Loki + Grafana (runs on vps1) loki/config.yml mesh-only, filesystem storage, 30-day retention grafana/provisioning/ datasource + dashboard provider (auto-loaded) grafana/provisioning/alerting/ rules, the Discord contact point, the policy grafana/dashboards/*.json the fleet-logs dashboard alloy/config.alloy the per-node shipper alloy/docker-compose.agent.yml runs Alloy on a node (ALLOY_NODE per host) -caddy-grafana.conf beta's Grafana vhost, kept for reference +caddy-grafana.conf vps1's Grafana vhost, kept for reference ``` No build, no deploy automation — it ships by hand. **The repo is the only @@ -177,10 +189,10 @@ Deep dive: [09-observability.md](09-observability.md). | `pr-build.yml` | PR → `dev`/`main` | The `gate` required check. Diffs the PR, builds only touched domains. Deliberately **not** path-filtered. | | `build.yml` | `workflow_call` | Build one domain's image; run the backend suite **inside** the built image. | | `build-push.yml` | push to `dev`/`main`/`release` touching an app domain, or a `v*.*.*` tag | Builds + pushes `sha-<12hex>` images. A version tag builds **both**. | -| `deploy.yml` | push to `main`/`release` touching an app domain | Branch selects environment; matrix covers services; SSHes and runs `deploy.sh`. | -| `infra-sync.yml` | push to `main` touching `infra/**` | Fast-forwards each host's checkout and re-renders secrets. Rolls **no** service. | +| `deploy.yml` | push to `dev`/`main`/`release` touching an app domain | Branch selects environment — three legs: `dev`→vps1, `main`→beta on vps2, `release`→prod on vps2; matrix covers services; SSHes and runs `deploy.sh` with `THERMOGRAPH_ENV` set. | +| `infra-sync.yml` | push to `main` touching `infra/**` | Fast-forwards beta's and prod's checkouts (both on vps2) and re-renders their secrets. Rolls **no** service. | | `observability-validate.yml` | push touching `observability/**`, or `workflow_call` | Parses every artifact; validates Alloy config with the pinned binary; strict alerting checks. | -| `ops-cron.yml` | daily 03:00 UTC | **THE prod backup** (pg_dump) + IndexNow. Uses `PROD_SSH_*`. | +| `ops-cron.yml` | daily 03:00 UTC | **THE prod backup** (pg_dump) + IndexNow, against prod on vps2. Uses `VPS2_SSH_*`. | | `secrets-guard.yml` | every PR and push | Fails if any `infra/deploy/secrets/*.yaml` isn't SOPS-encrypted. | | `shell-lint.yml` | every PR and push | shellcheck (pinned v0.11.0 + sha256) over every `*.sh`. | diff --git a/docs/onboarding/04-backend.md b/docs/onboarding/04-backend.md index b6e59b4..06371c1 100644 --- a/docs/onboarding/04-backend.md +++ b/docs/onboarding/04-backend.md @@ -17,7 +17,7 @@ them is impossible. | **web** | `THERMOGRAPH_ROLE=web` | Serves requests. Never starts the notifier, even if it would win the election — so the web tier scales to N replicas without also scaling background sweeps. | | **worker** | `THERMOGRAPH_ROLE=worker` | Owns the subscription notifier. Still serves requests today. | | **all** | default | Both — the single-process dev default. | -| **lake** | `THERMOGRAPH_ROLE=lake` | `deploy/entrypoint.sh` execs `uvicorn lake_app:app` on port 8141 instead. No database, no migrations. Prod only. | +| **lake** | `THERMOGRAPH_ROLE=lake` | `deploy/entrypoint.sh` execs `uvicorn lake_app:app` on port 8141 instead. No database, no migrations. Runs in every environment — its own Swarm service on prod and beta (`lake` / `beta-lake`), a compose service on dev. | | **daemon** | compose/Swarm sets the command to `/usr/local/bin/thermograph-daemon` | The Go binary: Discord gateway + cron timers. | `deploy/entrypoint.sh` runs `alembic upgrade head` (retried — a fresh Postgres @@ -169,7 +169,7 @@ warming, also warm-only. **The catch-all proxy** — the frontend owns every page and asset. In prod and beta, Caddy path-splits directly, so this proxy is never exercised. It exists as -the fallback for environments with no Caddy in front (LAN dev, bare-metal). It +the fallback for environments with no Caddy in front (dev, bare-metal). It forwards `X-Forwarded-Host`/`-Proto` so the frontend can build correct absolute URLs instead of resolving to the internal hop's own address. Because it's registered dead last, every real backend route wins first — @@ -236,7 +236,8 @@ never overlaps a job with itself, and drops ticks that fire mid-run. ## The lake (`lake_app.py` + `data/era5lake.py`) -Prod-only Swarm service, same image, `THERMOGRAPH_ROLE=lake`. +Runs as a Swarm service on both prod and beta (`lake` / `beta-lake`, same +image, `THERMOGRAPH_ROLE=lake`), and as a compose service on dev. The lake is parquet in an S3-compatible bucket (Contabo, `era5-thermograph`), extracted once from the public Earthmover ERA5 Icechunk archive by diff --git a/docs/onboarding/05-frontend.md b/docs/onboarding/05-frontend.md index c1dd17e..840a3a3 100644 --- a/docs/onboarding/05-frontend.md +++ b/docs/onboarding/05-frontend.md @@ -80,7 +80,7 @@ The static mount is registered last so the explicit routes win. The shell HTML files in `static/` carry an `__ORIGIN__` placeholder for the link-preview/Open Graph tags — preview crawlers need absolute URLs and the host -differs between LAN and prod. The substituted HTML and its ETag are **memoized +differs between dev and prod. The substituted HTML and its ETag are **memoized per origin**, not recomputed per request. Origin resolution prefers `X-Forwarded-Host` over `Host`. That matters when the diff --git a/docs/onboarding/07-ci-and-release.md b/docs/onboarding/07-ci-and-release.md index 5576594..27db954 100644 --- a/docs/onboarding/07-ci-and-release.md +++ b/docs/onboarding/07-ci-and-release.md @@ -3,15 +3,15 @@ ## The branch model ``` -PR ──(required check: `gate`)──▶ dev ──▶ LAN dev box +PR ──(required check: `gate`)──▶ dev ──▶ dev (vps1, mesh-only) │ promotion PR ▼ - main ──▶ beta.thermograph.org + main ──▶ beta.thermograph.org (vps2) │ promotion PR ← the owner's call, always ▼ - release ──▶ thermograph.org + release ──▶ thermograph.org (vps2) ``` `dev`, `main` and `release` are protected — direct pushes are blocked for @@ -40,9 +40,10 @@ for the separate hotfix path. **Commit counts are not deliverable content.** Before promoting, compare the two tips' *trees*. Identical trees mean the same changes are already on the target under different SHAs — promoting there merges nothing while still firing the -target's deploy workflows, including `infra-sync` re-rendering -`/etc/thermograph.env` on prod and beta. The ahead/behind numbers will look like -real work. `promote` refuses this outright. +target's deploy workflows, including `infra-sync` re-rendering prod's and +beta's env files (`/etc/thermograph.env` and `/etc/thermograph-beta.env`, both +on vps2). The ahead/behind numbers will look like real work. `promote` refuses +this outright. **Expect `dev`, `main` and `release` to be mutually divergent, and expect `--ff-only` to fail.** Every promotion merges the source into the target, @@ -111,16 +112,17 @@ Triggers on pushes to `dev`/`main`/`release` touching `backend/**` or - The registry is mesh-only, so the runner host needs `10.10.0.2 git.thermograph.org` in `/etc/hosts`. -### `deploy.yml` — one file, two environments, two services +### `deploy.yml` — one file, three environments, two services Formerly six near-identical files. Branch selects the environment -(`main → beta`, `release → prod`); a matrix covers the services. +(`dev → dev`, `main → beta`, `release → prod`) — **three legs now, not two** — +and a matrix covers the services. The style is **deliberately boring**: no dynamic `fromJSON` matrix, no `cond && secrets.A || secrets.B` ternary. Those are GitHub idioms a Forgejo/act runner may evaluate differently, and the failure mode here is "production does -not deploy" or, worse, "deploys with an empty SSH host". So the two environments -get two explicit, mutually exclusive steps and the tag is computed in shell. +not deploy" or, worse, "deploys with an empty SSH host". So each environment +gets its own explicit, mutually exclusive step and the tag is computed in shell. Preserved from the originals, all load-bearing: @@ -129,11 +131,34 @@ Preserved from the originals, all load-bearing: - 12-hex truncation matching `build-push.yml` exactly. - Per-service, per-environment concurrency with **`cancel-in-progress: false`** — a half-finished deploy must never be cancelled by a newer one. -- **Separate `PROD_SSH_*` credentials**, so a beta credential leak cannot reach - prod. - `appleboy/ssh-action` referenced by full URL; it isn't mirrored in Forgejo's default action registry. +**SSH credentials are keyed by HOST, not by environment**: `VPS1_SSH_*` and +`VPS2_SSH_*`, replacing the old `SSH_*` (which actually meant beta) and +`PROD_SSH_*` (which actually meant prod). That old naming used to read as a +security boundary — "separate `PROD_SSH_*` credentials, so a beta credential +leak cannot reach prod" — and on the old topology it happened to be true, +because beta and prod were also on separate hosts. They no longer are: beta and +prod are now two co-resident Swarm stacks on vps2, so a `VPS2_SSH_*` credential +reaches both by construction, and there is no host-level SSH boundary between +them to preserve. The boundary that replaces it lives at two other levels +instead: the **database** (beta connects as its own `NOSUPERUSER` role to its +own database, `CONNECT` revoked from `PUBLIC` on prod's) and the **filesystem** +(separate checkouts, separate rendered env files, separate deploy locks — +`env-topology.sh` derives all of it so nothing is shared by accident). + +The credential boundary that *is* still real, and still matters, is +**vps1-vs-vps2**: vps1 runs Forgejo, its own CI, and whatever's currently on +`dev` — unreviewed by definition, since `dev` is exactly the branch PRs land on +before review gates them into `main`. That is precisely why the `dev` +environment renders `dev.yaml` alone and never layers `common.yaml` (the +fleet's shared production credentials, including a read-write S3 keypair and +the VAPID push-signing key) — see [Infra and secrets](08-infra-secrets.md). A +`VPS1_SSH_*` leak should never be able to reach a production credential, which +is a property `common.yaml`'s dev exclusion buys independently of which SSH key +was used to get there. + There's also a per-leg refinement: the workflow-level `paths` filter only says *backend or frontend moved*, so each leg re-checks whether **this** push touched **this** domain. Without it a backend-only push would also roll the frontend, @@ -144,17 +169,20 @@ skipping silently strands a change. ### `infra-sync.yml` — infra's own pipeline -Push to `main` touching `infra/**` → SSH to beta **and** prod, fast-forward -`/opt/thermograph`, re-render `/etc/thermograph.env` from the vault. +Push to `main` touching `infra/**` → SSH to vps2, fast-forward **both** +checkouts there (`/opt/thermograph` for prod, `/opt/thermograph-beta` for +beta), re-render each one's own env file from the vault. **Rolls no service.** Image tags are the app domains' axis, not infra's. A -compose change that must recreate containers takes effect on the next app -deploy, or on a by-hand `SERVICE=all … infra/deploy/deploy.sh`. +compose or stack change that must recreate containers takes effect on the next +app deploy, or on a by-hand `SERVICE=all … infra/deploy/deploy.sh` (or +`deploy-dev.sh` for dev). Note the asymmetry: app code *is* environment-staged (`dev`→`main`→`release` -maps to LAN→beta→prod via image tags); **infra is not** — both hosts track infra -via `main`. Prod's *app images* are staged by `release`, but its checkout -follows `main`. +maps to dev(vps1)→beta→prod(vps2) via image tags); **infra is not** — beta's and +prod's checkouts both track infra via `main`, while dev's checkout tracks `dev` +itself. Prod's *app images* are staged by `release`, but its checkout follows +`main`. ### `observability-validate.yml` @@ -171,13 +199,17 @@ silent no-op is exactly what this check exists to prevent. ### `ops-cron.yml` — the prod backup -Daily at 03:00 UTC: `pg_dump` on prod, plus an IndexNow ping. Both SSH into prod -and run inside the already-running stack. +Daily at 03:00 UTC: `pg_dump` on prod, plus an IndexNow ping. Both SSH into +vps2 and run inside prod's already-running stack — specifically prod's, not +beta's, even though both now live on the same box. -**It uses `PROD_SSH_*`, not `SSH_*`.** An earlier revision reused `SSH_*` — so -the "prod" backup was silently dumping *beta*, and prod had no backup at all. If -you touch this file, verify a dump actually lands in -`agent@prod:~/thermograph-backups/`. +**It uses `VPS2_SSH_*`, not a beta-flavoured secret.** An earlier revision (on +the old topology) reused the credential meant for beta's host — so the "prod" +backup was silently dumping *beta*, and prod had no backup at all. On vps2's +co-residency, the host-keyed secret alone doesn't disambiguate prod from beta +the way it used to; the script also needs to target prod's checkout and `db` +specifically. If you touch this file, verify a dump actually lands in +`agent@vps2:~/thermograph-backups/` and that it's prod's data, not beta's. ### `secrets-guard.yml` and `shell-lint.yml` — the backstops @@ -199,7 +231,8 @@ guard. ## Runner facts -- Jobs run on the `docker` label — the LAN box's Forgejo Actions runner. +- Jobs run on the `docker` label — vps1's Forgejo Actions runner (vps1 is where + Forgejo itself lives; the desktop no longer runs a runner). - Job containers get the **host's docker.sock** automounted (Docker-outside-of-Docker, no privileged mode). The job image (`node:20-bookworm`) ships no docker CLI, which is why several workflows diff --git a/docs/onboarding/08-infra-secrets.md b/docs/onboarding/08-infra-secrets.md index ed8eec5..7f0c513 100644 --- a/docs/onboarding/08-infra-secrets.md +++ b/docs/onboarding/08-infra-secrets.md @@ -7,23 +7,36 @@ Read [`infra/CLAUDE.md`](../../infra/CLAUDE.md) alongside this. ## The machines -| Host | Public | Mesh | Orchestrator | Login | +Two VPS boxes named by **role**, not by environment, plus the operator's +desktop: + +| Host | Public | Mesh | Runs | Login | |---|---|---|---|---| -| **prod** | `169.58.46.181` / thermograph.org | `10.10.0.1` | Docker **Swarm** | `agent`, passwordless sudo | -| **beta** | `75.119.132.91` / beta.thermograph.org | `10.10.0.2` | **compose** | `agent`, passwordless sudo | -| **desktop** | — | `10.10.0.3` | compose (LAN dev) | it's your box | +| **vps1** | `75.119.132.91` / git.thermograph.org, dashboard.thermograph.org | `10.10.0.2` | Forgejo, Grafana + Loki + Alloy, `emigriffith.dev`, and **dev** (own Postgres, compose, mesh-only) | `agent`, passwordless sudo | +| **vps2** | `169.58.46.181` / thermograph.org, beta.thermograph.org | `10.10.0.1` | **prod** and **beta**, as two co-resident Docker **Swarm** stacks; Centralis; Postfix; backups | `agent`, passwordless sudo | +| **desktop** | — | `10.10.0.3` | AI-model hosting + flex Swarm capacity. No Thermograph environment; `make dev-up` is a laptop convenience there | it's your box | ```bash -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # prod -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # beta +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # vps2 (prod + beta) +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # vps1 (Forgejo + Grafana + dev) ``` -Beta also hosts **Forgejo** (git + CI + registry) and **Grafana + Loki**, which -is why beta is guarded as strictly as prod: a destructive command there takes -out git, CI and the registry at once. The LAN dev box is deliberately -unguarded. +The mesh IPs did not move when this topology landed — vps1 is the box that used +to be called "beta" (still mesh `10.10.0.2`), vps2 the one that used to be +called "prod" (still mesh `10.10.0.1`). What moved is which environment runs on +which box: beta moved from vps1 onto vps2 to sit next to prod, and dev moved +onto vps1. -Every root-effective command on the VPSes is logged by `auditd` +**vps1 is guarded as strictly as vps2**, not more loosely because it "used to +be beta" or because dev now lives there. It hosts Forgejo, its CI runner and +the registry, plus Grafana/Loki — a destructive command there takes out git, CI +and the registry at once. Dev being mesh-only and a public VPS's tenant, rather +than someone's desktop, is exactly why it gets the same guard as everything +else on that box: it shares a kernel with the estate's source of truth for code +and the monitoring stack, sudo there is passwordless, and it runs whatever +branch is currently in flight. + +Every root-effective command on both VPSes is logged by `auditd` (`ausearch -k agentcmd`). That's a feature — fixes are attributable. Prefer Centralis for routine work (`run_on_host`, `fleet_status`, @@ -31,20 +44,36 @@ Prefer Centralis for routine work (`run_on_host`, `fleet_status`, ## The two orchestrators -Which path a host takes is decided by **`/etc/thermograph/deploy-mode`**: the -string `stack` makes `deploy.sh` exec `deploy/stack/deploy-stack.sh`; anything -else is compose. The workflows never need to know which mode a host runs. +Which path an environment takes is decided by **`infra/deploy/env-topology.sh`** +(`thermograph_topology `, keyed on `THERMOGRAPH_ENV=dev|beta|prod`): it +derives the checkout, branch, deploy mode, stack/compose name, env file, LB +ports and DB role for that environment, and `deploy.sh` execs +`deploy/stack/deploy-stack.sh` when the mode is `stack`, or rolls compose +otherwise. The old host-wide `/etc/thermograph/deploy-mode` marker still exists +as a **fallback** for a by-hand run on a box that predates this file, but it +cannot describe vps2 on its own — vps2 runs two environments, so "which mode +does this host use" is no longer a well-formed question there. -| | compose (beta, LAN dev) | Swarm (prod) | +| | compose (dev) | Swarm (prod, beta) | |---|---|---| -| File | `infra/docker-compose.yml` | `infra/deploy/stack/thermograph-stack.yml` | -| Services | `db`, `backend`, `lake`, `daemon`, `frontend` | `db`, `web`, `worker`, `lake`, `daemon`, `frontend`, `autoscaler`, `autoscaler-lake` | +| Host | vps1 | vps2, both stacks co-resident | +| File | `infra/docker-compose.yml` | `infra/deploy/stack/thermograph-stack.yml` (prod), `thermograph-beta-stack.yml` (beta) | +| Services | `db`, `backend`, `lake`, `daemon`, `frontend` | prod: `db`, `web`, `worker`, `lake`, `daemon`, `frontend`, `autoscaler`, `autoscaler-lake`. beta: `beta-web`, `beta-worker`, `beta-lake`, `beta-daemon`, `beta-frontend` — prefixed, no `db`, no autoscalers | | Rolling | `up -d --no-deps ` | start-first, health-gated, auto-rollback | | Tag file | `deploy/.image-tags.env` | `deploy/.stack-image-tags.env` | Note prod splits `backend` into **`web`** and **`worker`** (the -`THERMOGRAPH_ROLE` split from [04](04-backend.md)); compose runs one `backend` -service in role `all`. +`THERMOGRAPH_ROLE` split from [04](04-backend.md)); dev's compose stack runs one +`backend` service in role `all`. Beta's Swarm services are prefixed +(`beta-web`, not `web`) because Swarm registers a service's short name as a DNS +alias on every network it joins — two stacks both naming a service `web` on +the shared `data` network would make `web` ambiguous, and beta joins that +network (declared `external` in its stack file) purely to reach prod's `db`. +Beta keeps its own `internal` overlay for beta-to-beta traffic and has no `db` +service of its own: **one TimescaleDB instance serves both environments**, on +separate databases (`thermograph` / `thermograph_beta`) and separate roles +(beta's is `NOSUPERUSER`/`NOCREATEDB`/`NOCREATEROLE`, `CONNECT` revoked from +`PUBLIC` on prod's database). `STACK_TEST=1` rehearses the entire Swarm deploy under stack name `thermograph-test` on throwaway volumes and ports `18137`/`18080`. Leftover @@ -58,11 +87,15 @@ coupling is why `name: thermograph` is pinned in the compose file — running compose from `infra/` without it derives project `infra`, silently creating a whole new stack with empty volumes beside the running one. `deploy-dev.sh` exports `COMPOSE_PROJECT_NAME=thermograph-dev` (env wins over the file key) to -keep LAN dev separate on purpose. **Keep both halves.** +keep dev separate on purpose. **Keep both halves.** ## `deploy/deploy.sh` — read this before you touch a deploy -Single entry point for beta and prod. Contract: +Single entry point for dev, beta and prod — the same script for all three; +`infra/deploy/env-topology.sh` is what tells it which checkout, branch, stack +and ports apply. Contract, run from that environment's own checkout +(`/opt/thermograph` for prod, `/opt/thermograph-beta` for beta, both on vps2; +`/opt/thermograph-dev` on vps1 via the `deploy-dev.sh` wrapper): ```bash SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh @@ -76,13 +109,17 @@ What it does, in order, and why each step is the way it is: the whole run. Backend and frontend deploys can fire for the same push seconds apart and both SSH into one checkout; concurrent runs race the git reset, the compose project and the tag file. `-w 600` bounds the wait. -2. **Render secrets** from the SOPS vault into `/etc/thermograph.env`, then - source it, so a by-hand run interpolates the same as the systemd unit does. - Guarded on the helper's existence so the very deploy that *introduces* - `render-secrets.sh` is safe. +2. **Render secrets** from the SOPS vault into that environment's own env file + (`/etc/thermograph.env` for prod, `/etc/thermograph-beta.env` for beta, + likewise `/etc/thermograph.env` on dev's separate host), then source it, so + a by-hand run interpolates the same as CI's does. Guarded on the helper's + existence so the very deploy that *introduces* `render-secrets.sh` is safe. 3. **`git reset --hard origin/$BRANCH`** on the checkout root (`BRANCH` defaults - to `main`). ⚠️ **Uncommitted edits in `/opt/thermograph` evaporate here.** - The one exception is `deploy/.image-tags.env`, untracked on purpose. + to `main` for beta/prod, `dev` for dev). ⚠️ **Uncommitted edits in that + checkout evaporate here** — on vps2 that's `/opt/thermograph` for a prod + deploy or `/opt/thermograph-beta` for a beta one; each reset only ever + touches its own checkout, never the sibling's. The one exception is + `deploy/.image-tags.env`, untracked on purpose. 4. **Read `.image-tags.env`** so a single-service roll re-renders compose with *both* services' real tags and never accidentally recreates or downgrades the sibling. @@ -115,12 +152,15 @@ What it does, in order, and why each step is the way it is: Rollback is redeploying the previous image tag. But **do not treat image retention as a rollback guarantee**: both `deploy.sh` and `deploy-stack.sh` -*end* by deleting images outside the running pair. On beta that succeeds, so -beta typically holds **no local rollback target at all**; on prod it usually -fails only because Swarm's stopped task containers still hold references, which -is why prod keeps a handful. Verify the target tag is actually present before -promising a rollback — Centralis's `rollback_to(dry_run: true)` checks both host -and registry. +*end* by deleting images outside the running pair. Beta now runs Swarm exactly +like prod (it used to be compose, on the old topology, where this pruning +reliably left no rollback target at all); on both Swarm stacks it usually +fails only because Swarm's stopped task containers still hold references, +which is why prod and beta both tend to keep a handful. Dev, the one remaining +compose environment, is the one where this prune still reliably succeeds and +leaves nothing to roll back to. Verify the target tag is actually present +before promising a rollback — Centralis's `rollback_to(dry_run: true)` checks +both host and registry. `docker system prune -a` on a live box is forbidden for the same reason. `docker image prune -f` (dangling only) is the safe form. @@ -129,55 +169,81 @@ and registry. **The single source of truth is `infra/deploy/secrets/*.yaml`**, committed encrypted (values only — keys stay readable so diffs mean something) and -rendered into `/etc/thermograph.env` at deploy time by -`deploy/render-secrets.sh`. +rendered at deploy time by `deploy/render-secrets.sh` into each environment's +own env file (`/etc/thermograph.env` for prod, `/etc/thermograph-beta.env` for +beta, `/etc/thermograph.env` again on dev's separate host). **Cycling a key is: edit → commit → deploy.** No SSH, no hand-edited root file, no per-host duplication. | File | Holds | |---|---| -| `common.yaml` | the 16 values identical on prod **and** beta — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config | +| `common.yaml` | the 16 values identical on prod **and** beta — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config. **Never layered under `dev`** — see below | | `prod.yaml` | prod's own — the three held-back credentials, sizing, `THERMOGRAPH_BASE_URL`, Discord + mail credentials that exist nowhere else | | `beta.yaml` | beta's own — the three held-back credentials, sizing, base URL | -| `dev.yaml` | the LAN box's own, **self-contained**, 12 values, no production credential | +| `dev.yaml` | dev's own, **self-contained**, 12 values, no production credential | | `centralis.prod.yaml` | Centralis's nine variables → `/etc/centralis.env`, its own renderer | | `example.yaml` | format reference / CI fixture (fake values) | | `../../.sops.yaml` | which age recipients files encrypt to (plaintext config) | The renderer concatenates `common.yaml` then `.yaml`, so a **host value -wins** (last occurrence of a duplicate key). `` comes from -`/etc/thermograph/secrets-env` on the box. +wins** (last occurrence of a duplicate key). `` is passed explicitly as +`THERMOGRAPH_ENV` by the deploy caller — required now that vps2 alone hosts two +environments and a host-wide marker can't tell them apart; the marker +(`/etc/thermograph/secrets-env`) survives only as the fallback for a by-hand +run on a single-environment box. -### Two design decisions worth understanding +### Two design decisions worth understanding — re-derived for vps1/vps2 **Three credentials are deliberately *not* in `common.yaml`** — `POSTGRES_PASSWORD`, `THERMOGRAPH_AUTH_SECRET`, `THERMOGRAPH_DATABASE_URL`. -They hold identical values on prod and beta today, so by the mechanical rule -they belong there. They're kept per-host anyway because these are the -credentials that let one environment *act as* another: with them, a foothold on -beta (the more exposed box — public Forgejo and Grafana) is a foothold on prod's -database and prod's session signing. They match because beta was seeded from -prod, not because the two are meant to be one system. Keeping them per-host -costs one extra line and buys the ability to diverge. +They hold identical *values* on prod and beta today (beta was seeded from +prod), so by the mechanical rule they'd belong in `common.yaml`. They're kept +per-host anyway, and the reason used to be phrased as a host-isolation +argument — "a foothold on beta is a foothold on prod's database, and beta is +the more exposed box." **That framing no longer holds**: beta and prod are now +two Swarm stacks co-resident on vps2, so a foothold on the host reaches both +regardless of which file a credential lives in. What these three credentials +still buy, correctly stated, is a **database-level** and **file-level** +boundary, not a host-level one: beta's `THERMOGRAPH_DATABASE_URL` names its own +`NOSUPERUSER` role (`thermograph_beta`) against its own database on the shared +TimescaleDB instance, with `CONNECT` revoked from `PUBLIC` on prod's database, +and each environment's `POSTGRES_PASSWORD`/`THERMOGRAPH_AUTH_SECRET` live in a +separate file so rotating prod's stops implying "and beta's too." Keeping them +per-host costs one extra line and buys the ability to diverge — that part is +unchanged. What changed is what "diverge" is defending against: not a beta +compromise reaching a separate prod host, but a beta compromise reaching +prod's role and session-signing key on the host they now already share. -**`dev` renders `dev.yaml` alone** — `deploy-dev.sh` exports -`THERMOGRAPH_SECRETS_SKIP_COMMON=1`. Eleven of `common.yaml`'s sixteen values -are live production credentials, including a **read-write** object-storage -keypair on the bucket that also holds prod's database backups, and the VAPID -private key that signs push to real subscribers. The render is plaintext -concatenation, so layering it would put all of those into every container in the -dev stack — on the operator's desktop, which is also the CI runner whose -`docker`-labelled jobs get the host socket, running unreviewed `dev` code. An +**The credential boundary that actually maps to hosts now is vps1-vs-vps2, not +beta-vs-prod** — which is exactly why `dev` renders `dev.yaml` **alone**, +never layering `common.yaml` (the fleet's shared production credentials): +`deploy-dev.sh` exports `THERMOGRAPH_SECRETS_SKIP_COMMON=1`. Eleven of +`common.yaml`'s sixteen values are live production credentials, including a +**read-write** object-storage keypair on the bucket that also holds prod's +database backups, and the VAPID private key that signs push to real +subscribers. The render is plaintext concatenation, so layering it would put +all of those into every container in the dev stack — on vps1, the box that +also runs Forgejo and its CI runner, whose `docker`-labelled jobs get the host +socket, running whatever branch is currently `dev` (unreviewed by definition: +that's the branch PRs land on before the gate promotes them to `main`). An override in `dev.yaml` wouldn't help: last-wins governs *consumers*, but the -production value is still physically a line in the file. +production value is still physically a line in the file. This is the load- +bearing boundary in the vault today, not a per-environment credential split +that co-residency has already dissolved at the host level. -Dev works fine without them: no S3 keys means the `lake` service answers 503 and -history falls through to the Open-Meteo archive (an accelerator, never a -dependency); VAPID and IndexNow both self-generate and persist to the `appdata` -volume; with no metrics token `/api/v2/metrics` is direct-loopback-only, which -is right on a LAN box. One `common.yaml` value is simply *wrong* for dev — -`THERMOGRAPH_COOKIE_SECURE=1` silently breaks login over plain HTTP. +Dev works fine without `common.yaml`'s values: no S3 keys means the `lake` +service answers 503 and history falls through to the Open-Meteo archive (an +accelerator, never a dependency); VAPID and IndexNow both self-generate and +persist to the `appdata` volume; with no metrics token `/api/v2/metrics` is +direct-loopback-only, which is right for a mesh-only box. One `common.yaml` +value is simply *wrong* for dev — `THERMOGRAPH_COOKIE_SECURE=1` silently breaks +login over dev's plain-HTTP mesh URL. + +None of this makes dev unimportant or unguarded. It's a public VPS's tenant +now, sharing a box with Forgejo and the monitoring stack — not somebody's +desktop — which is precisely why it gets its own exclusion from `common.yaml` +rather than being treated as beneath the vault's concern. ### Working with the vault @@ -223,12 +289,14 @@ infra/ops/iceberg.sh prod -c "select count(*) from era5_daily" ``` Both exec **into the container**. No database is exposed over TCP — each listens -only on its private docker network, and prod's is a Swarm **overlay the prod -host itself cannot route to**, so `ssh -L` works for beta and is *impossible* -for prod. Exec works identically everywhere with no ports, no tunnels, no infra -changes. The prod container name is a Swarm task name that changes on every -redeploy, so it's resolved at call time via `docker ps --filter name=`, never -hardcoded. +only on its private docker network, and prod's and beta's are both Swarm +**overlays the vps2 host itself cannot route to**, so `ssh -L` is *impossible* +for either. (Dev's, on vps1, is plain compose, so `ssh -L` would actually work +there — but exec is used uniformly across environments anyway, so it doesn't +matter which one happens to allow the shortcut.) Exec works identically +everywhere with no ports, no tunnels, no infra changes. The prod and beta +container names are Swarm task names that change on every redeploy, so they're +resolved at call time via `docker ps --filter name=`, never hardcoded. Queries connect as **`thermograph_ro`** — `NOSUPERUSER`, granted only `pg_read_all_data`. Read-only is enforced by Postgres, not by convention: @@ -244,8 +312,9 @@ confirmation — on prod, know what you are doing. ## Backups -The nightly `ops-cron.yml` `pg_dump` into `agent@prod:~/thermograph-backups/` -is it. Known gaps, stated plainly: **a single copy on the same box as the +The nightly `ops-cron.yml` `pg_dump` into `agent@vps2:~/thermograph-backups/` +(prod's data specifically — see [CI and release](07-ci-and-release.md)) is it. +Known gaps, stated plainly: **a single copy on the same box as the database, no offsite yet**, and restores must handle TimescaleDB's `continuous_agg` circular-FK warning (`--disable-triggers`). The `backups/` prefix in the object-storage bucket belongs to those jobs — don't write outside diff --git a/docs/onboarding/09-observability.md b/docs/onboarding/09-observability.md index 8605fba..5ad6fab 100644 --- a/docs/onboarding/09-observability.md +++ b/docs/onboarding/09-observability.md @@ -1,26 +1,34 @@ # 9. Observability -Fleet-wide log aggregation: **Loki + Grafana on beta**, fed by a **Grafana -Alloy** agent on every node, all over the WireGuard mesh. +Fleet-wide log aggregation: **Loki + Grafana on vps1**, fed by a **Grafana +Alloy** agent on every node, all over the WireGuard mesh. vps2 runs two +environments (beta, prod) behind that one Alloy agent; vps1 runs Alloy +alongside the things it's shipping logs *from* (Forgejo, Grafana itself, dev). ``` - prod (10.10.0.1) beta (10.10.0.2) desktop / LAN dev (10.10.0.3) + vps2 (10.10.0.1) vps1 (10.10.0.2) desktop (10.10.0.3) + prod + beta Loki, Grafana, dev ┌────────────┐ ┌──────────────────┐ ┌────────────┐ │ Alloy agent│──wg0──┐ │ Loki ◀── Alloy │ ┌───│ Alloy agent│ └────────────┘ └──▶│ Grafana (Caddy) │◀─┘ └────────────┘ - docker+caddy+app └──────────────────┘ docker+caddy+app + docker+caddy+app └──────────────────┘ docker (AI models) ``` - **Loki** — filesystem storage, **30-day retention**, listening on the mesh IP `10.10.0.2:3100`. Never public. -- **Grafana** — the UI, fronted by beta's Caddy at +- **Grafana** — the UI, fronted by vps1's Caddy at **`dashboard.thermograph.org`** (Google SSO, pre-provisioned accounts only, `allow_sign_up=false`, plus a break-glass local admin). Use that hostname everywhere; **never** `grafana.thermograph.org`. -- **Alloy** — on each node, shipping three sources: every Docker container's - stdout/stderr, Caddy's host access logs, and the app's structured JSON logs - (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase` become - labels). Every line is tagged `host = prod|beta|dev`. +- **Alloy** — one agent per *host*, not per environment: vps2's single agent + ships logs for both beta and prod, vps1's ships Forgejo/Grafana/dev, and the + desktop's ships whatever runs there now. Each ships three sources: every + Docker container's stdout/stderr, Caddy's host access logs, and the app's + structured JSON logs (`errors`/`access`/`audit` `*.jsonl`, parsed so + `level`/`tag`/`phase` become labels). Every app log line is tagged + `host = prod|beta|dev` — that label is the *environment*, not the box, which + matters on vps2 where one Alloy agent ships lines carrying two different + `host` values. There is **no build and no deploy automation** for this domain — it ships by hand. But `observability-validate.yml` parses every artifact that ships to a @@ -42,20 +50,24 @@ Start wide, then narrow: 2. `logs_overview since="1h"` — where the volume is, and what has gone quiet. 3. `logs_query service=… host=…` — the actual lines. -### ⚠ The prod trap +### ⚠ The prod (and now beta) trap -Prod runs a Swarm stack, and its `job="docker"` streams carry **no `service` -label**. A raw `{host="prod", service="backend"}` query matches nothing and -reads as *"no logs"* when it actually means *"wrong selector"*. +Prod and beta both run Swarm stacks — co-resident on vps2 — and their +`job="docker"` streams carry **no `service` label**. A raw +`{host="prod", service="backend"}` query matches nothing and reads as *"no +logs"* when it actually means *"wrong selector"*. This used to be a prod-only +trap when beta ran compose; it now applies to both environments on vps2, and +dev (compose, on vps1) is the one environment this does *not* affect. Use Centralis's `logs_query` `service` argument, which rewrites it to a container name-regex with the churning task suffix wildcarded. Prod service names are `thermograph_web`, `_worker`, `_frontend`, `_db`, `_autoscaler`, -`_lake`. +`_lake`; beta's are the same names prefixed `beta-` (`beta-web`, `beta-worker`, +…) with no `_db` or autoscaler equivalents. -Related: **prod container names change on every redeploy** (the Swarm task -suffix). Never hardcode one; resolve via `docker ps --filter name=` at call -time. +Related: **prod and beta container names both change on every redeploy** (the +Swarm task suffix). Never hardcode one; resolve via `docker ps --filter name=` +at call time. ## The dashboard @@ -81,16 +93,16 @@ Three reasons, all learned the hard way: - Grafana's factory default pointed at the literal string `` — a paging path that had never delivered a message to anyone. -- Beta's Grafana relays SMTP through **prod's** Postfix. Email alerts therefore - travel through the box most likely to be on fire, and vanish exactly when they - matter. +- vps1's Grafana relays SMTP through **vps2's** Postfix (prod's mail service). + Email alerts therefore travel through the box most likely to be on fire, and + vanish exactly when they matter. - Discord is off-estate, works when prod is dead, and reaches a phone. `#ops-alerts` is dedicated. `#dev` / `#uat` / `#prod` are the *product's* alert subscription output — real deliveries to real subscribers — so routing ops noise there would corrupt the evidence. -The webhook URL is a secret and lives only in beta's gitignored `.env` as +The webhook URL is a secret and lives only in vps1's gitignored `.env` as `DISCORD_ALERT_WEBHOOK_URL`. Grafana refuses to start if it's unset. A literal webhook URL committed to the repo is a **hard CI failure**. @@ -135,7 +147,7 @@ pages nobody. **The only proof is a message actually arriving in ## Deploying an observability change > ⚠️ **Merging is not deploying — and this one has already bitten.** -> `/opt/observability` on beta and prod is a checkout of the **archived** +> `/opt/observability` on vps1 and vps2 is a checkout of the **archived** > `emi/thermograph-observability` repo. It can never `git pull` again; the > content now lives here under `observability/`. A previous observability PR > merged and had **zero live effect** for exactly this reason. @@ -146,13 +158,14 @@ hand: back up the live file with a UTC-timestamped suffix, `scp` the new one up, Two gotchas in that sequence: -- Use `sudo docker restart ` on **prod** — not `docker compose`, +- Use `sudo docker restart ` on **vps2** — not `docker compose`, which there demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to - interpolate from. Beta's stack *does* have an `.env`, so compose works there. + interpolate from. vps1's stack *does* have an `.env`, so compose works there. - **`docker restart` will not pick up a new environment variable.** Any change - that introduces one needs `docker compose up -d grafana` on beta. + that introduces one needs `docker compose up -d grafana` on vps1. -Alerting lives on **beta only** — prod and dev run Alloy agents, not Grafana. +Alerting lives on **vps1 only** — vps2 (prod and beta alike) and dev run Alloy +agents, not Grafana. Full step-by-step, including rollback, is in [`observability/README.md`](../../observability/README.md). @@ -177,9 +190,9 @@ Full step-by-step, including rollback, is in - **Grafana cannot alert on its own death.** `AlertingWatchdog` is a *manual* dead-man's switch — it relies on somebody noticing the daily message stopped. An external uptime pinger from off-estate would close this properly. -- **Beta runs a `docker-compose.override.yml` that is not in this repo**, wiring - Grafana's SMTP to prod's Postfix. It should be mirrored into the tracked - compose file or deleted; alerting no longer depends on it. +- **vps1 runs a `docker-compose.override.yml` that is not in this repo**, wiring + Grafana's SMTP to vps2's Postfix (prod's). It should be mirrored into the + tracked compose file or deleted; alerting no longer depends on it. ## When something breaks diff --git a/docs/onboarding/10-recipes.md b/docs/onboarding/10-recipes.md index 58766e7..8538514 100644 --- a/docs/onboarding/10-recipes.md +++ b/docs/onboarding/10-recipes.md @@ -79,14 +79,17 @@ git commit && push && open a PR ``` Then it reaches the hosts by either route: an `infra/**` push to `main` fires -`infra-sync.yml` (re-renders `/etc/thermograph.env` on beta and prod, rolls -nothing), or the next app deploy renders it as step 2 of `deploy.sh`. +`infra-sync.yml` (re-renders prod's and beta's env files — `/etc/thermograph.env` +and `/etc/thermograph-beta.env`, both on vps2 — rolls nothing), or the next app +deploy renders it as step 2 of `deploy.sh`. Dev's vault (`dev.yaml`) is never +part of that sync — it's rendered only by dev's own deploy on vps1, and it +never layers `common.yaml` regardless. Decide **which file** first: - identical on prod *and* beta, and not a "lets one environment act as another" - credential → `common.yaml`; -- per-host, or one of `POSTGRES_PASSWORD` / `THERMOGRAPH_AUTH_SECRET` / + credential → `common.yaml` (never reaches dev, on vps1, at all); +- per-environment, or one of `POSTGRES_PASSWORD` / `THERMOGRAPH_AUTH_SECRET` / `THERMOGRAPH_DATABASE_URL` → `prod.yaml` / `beta.yaml`; - dev → `dev.yaml`, and give it **its own** value. Never a copy of prod's. @@ -209,9 +212,11 @@ SERVICE=backend BACKEND_IMAGE_TAG=sha- \ ``` **Check the tag actually exists first.** Both deploy scripts *end* by deleting -images outside the running pair, so beta typically holds no local rollback -target at all. `rollback_to(dry_run: true)` checks host and registry and will -tell you when it's in neither. +images outside the running pair. Beta now runs Swarm exactly like prod +(co-resident on vps2), so both usually keep a handful of stopped-task +references; dev, the one remaining compose environment (on vps1), is the one +that typically holds no local rollback target at all. `rollback_to(dry_run: +true)` checks host and registry and will tell you when it's in neither. --- diff --git a/docs/onboarding/11-traps.md b/docs/onboarding/11-traps.md index 4604b5d..86bff0a 100644 --- a/docs/onboarding/11-traps.md +++ b/docs/onboarding/11-traps.md @@ -48,19 +48,31 @@ walkthrough as superseded — but the drift goes further than that note admits. The **key-management** sections (Part 1) and the `/etc/hosts` registry note are still accurate and useful. -### `infra/DEPLOY-DEV.md` and `infra/ACCESS.md` reference deleted workflows +### `infra/DEPLOY-DEV.md` and `infra/ACCESS.md` reference deleted workflows — and predate dev becoming a real environment -Both talk about `.forgejo/workflows/deploy-dev.yml`. It does not exist. The two -LAN-dev deploy workflows were deleted rather than ported at the CI -consolidation — they were documented as inert (they targeted a monorepo layout -at `~/thermograph-dev` on a box still holding a split-era checkout). **LAN dev -is a local `make dev-up` concern, not a CI environment.** +Both talk about `.forgejo/workflows/deploy-dev.yml`. It does not exist under +that name. The two original LAN-dev deploy workflows were deleted rather than +ported at the CI consolidation — at the time they were genuinely inert (they +targeted a monorepo layout at `~/thermograph-dev` on a box still holding a +split-era checkout), which is where the old claim **"LAN dev is a local `make +dev-up` concern, not a CI environment"** came from. + +**That claim is now false, and worth flagging precisely because it used to be +true.** `dev` is a first-class hosted environment on vps1 — its own Postgres +container, mesh-only exposure, deployed by CI like beta and prod, as the third +leg of `deploy.yml` (see [07](07-ci-and-release.md) and +[08](08-infra-secrets.md)). The desktop's `make dev-up` is still a real, +useful thing — a laptop-local rehearsal — but it is no longer the *only* way +`dev` runs, and neither doc has been updated to say so. `ACCESS.md` §3c also lists `{build,pr-build,deploy-dev,deploy}.yml` as the workflow set; there are now nine files and the deploy/build ones were collapsed. +It also still describes the host table from before the vps1/vps2 rename — see +`CUTOVER-NOTES.md`'s vps1/vps2 section for the current one. -Everything else in `ACCESS.md` — the host table, the agent-access model, the key -rotation procedure, the Swarm/Forgejo tracks — is current and worth reading. +Everything else in `ACCESS.md` — the agent-access model, the key rotation +procedure, the Swarm/Forgejo tracks — is current and worth reading; just +mentally translate host names. ### `backend/README.md` is written for the split-repo era @@ -93,7 +105,7 @@ it — that's the standing instruction in the root `CLAUDE.md`. ### `observability/README.md`: merging is not deploying -`/opt/observability` on beta and prod is a checkout of the **archived** +`/opt/observability` on vps1 and vps2 is a checkout of the **archived** `emi/thermograph-observability` repo. It can never `git pull` again. A previous observability PR merged and had **zero live effect** for exactly this reason. The README documents the hand-ship procedure; until the checkouts are re-pointed @@ -153,7 +165,7 @@ test run. `infra/docker-compose.yml` pins `name: thermograph`. Without it, running compose from `infra/` derives project `infra` — a silently **new** stack with empty volumes beside the running one. `deploy-dev.sh` exports -`COMPOSE_PROJECT_NAME=thermograph-dev` to keep LAN dev separate on purpose. +`COMPOSE_PROJECT_NAME=thermograph-dev` to keep dev separate on purpose. Keep both halves. ### `deploy.sh` hard-resets the host checkout @@ -167,11 +179,13 @@ roll the sibling onto `local`. ### The deploy scripts eat your rollback image Both `deploy.sh` and `deploy-stack.sh` **end** by deleting images outside the -running pair. On beta that succeeds, so beta typically holds **no local rollback -target at all**; on prod it usually fails only because Swarm's stopped task -containers still hold references. `docker system prune -a` is forbidden on a -live box for the same reason — but the deploy script is the thing that actually -eats the image, not prune. Verify a tag exists before promising a rollback. +running pair. Beta now runs Swarm co-resident with prod on vps2, so both +usually fail to fully prune only because Swarm's stopped task containers still +hold references — dev, the one remaining compose environment (on vps1), is the +one that typically holds **no local rollback target at all**. `docker system +prune -a` is forbidden on a live box for the same reason — but the deploy +script is the thing that actually eats the image, not prune. Verify a tag +exists before promising a rollback. ### `daemon` and `lake` are never deploy targets on their own @@ -189,13 +203,14 @@ line, in file order. A `| head -1` therefore grabbed `db`'s image, and and dropped the daemon from *every* backend deploy. The script now builds the image reference from the same variables compose interpolates. -### Prod's Loki streams carry no `service` label +### Prod's (and beta's) Loki streams carry no `service` label -Prod runs Swarm; its `job="docker"` streams have no `service` label. A raw -`{host="prod", service="backend"}` query matches nothing and reads as "no logs" -when it means "wrong selector". Use `logs_query`'s `service` argument. Prod -container names also change on every redeploy (the task suffix) — resolve via -`docker ps --filter name=` at call time, never hardcode. +Prod and beta both run Swarm, co-resident on vps2; their `job="docker"` streams +have no `service` label. A raw `{host="prod", service="backend"}` query matches +nothing and reads as "no logs" when it means "wrong selector". Use +`logs_query`'s `service` argument. Prod and beta container names also change on +every redeploy (the task suffix) — resolve via `docker ps --filter name=` at +call time, never hardcode. Dev, on compose, doesn't have this problem. ### A Grafana alert rule can provision cleanly and never fire @@ -212,9 +227,9 @@ It's executable documentation until state is bootstrapped. ### The registry is mesh-only -`git.thermograph.org` resolves publicly to beta's IP, but beta's Caddy rejects +`git.thermograph.org` resolves publicly to vps1's IP, but vps1's Caddy rejects `/v2/*` from outside the WireGuard mesh. Any host that pulls needs -`10.10.0.2 git.thermograph.org` in `/etc/hosts`. Beta itself doesn't — it *is* +`10.10.0.2 git.thermograph.org` in `/etc/hosts`. vps1 itself doesn't — it *is* the box. ### Forgejo runners only have the labels they registered with diff --git a/docs/onboarding/README.md b/docs/onboarding/README.md index 9b177df..9150399 100644 --- a/docs/onboarding/README.md +++ b/docs/onboarding/README.md @@ -42,12 +42,13 @@ readings. the superseded original — see [traps](11-traps.md).) - **`infra/`** — compose and Swarm files, deploy scripts, the SOPS secrets vault, Terraform, ops query tooling. -- **`observability/`** — Loki + Grafana on beta, an Alloy agent per node. +- **`observability/`** — Loki + Grafana on vps1, an Alloy agent per node. -Branches stage environments: PR → `dev` (LAN dev) → `main` (beta) → `release` -(prod). The two app domains build and deploy **independently** — that -independence is the whole reason the split-then-reunify history exists, and -[contracts](06-contracts.md) is the list of things that keep it safe. +Branches stage environments: PR → `dev` (vps1, mesh-only) → `main` (beta, vps2) +→ `release` (prod, vps2). The two app domains build and deploy +**independently** — that independence is the whole reason the split-then-reunify +history exists, and [contracts](06-contracts.md) is the list of things that +keep it safe. ## Your first day diff --git a/infra/.claude/skills/key-gaps/SKILL.md b/infra/.claude/skills/key-gaps/SKILL.md index f243540..d353ebf 100644 --- a/infra/.claude/skills/key-gaps/SKILL.md +++ b/infra/.claude/skills/key-gaps/SKILL.md @@ -28,12 +28,24 @@ python3 .claude/skills/key-gaps/key_gaps.py \ ## Audit the LIVE boxes (read-only, key names only) -Gather the key names over SSH (never the values), then audit. Hosts/keys per INFRA.md: +Gather the key names over SSH (never the values), then audit. Hosts per the +root `CLAUDE.md` topology: + +- **vps2** (`169.58.46.181`) hosts BOTH prod and beta as separate Swarm + stacks — TWO live env files on the same box: `/etc/thermograph.env` (prod) + and `/etc/thermograph-beta.env` (beta). "The live env file per environment" + is no longer one path per host; it's one path per environment, and both + environments' files live on this one host. Get both in the same SSH round + trip, or two separate commands against the same host — never assume one + host means one file here. +- **vps1** (`75.119.132.91`) hosts dev, at the same `/etc/thermograph.env` + path (it's a different host, so no collision with prod's file of the same + name). ```sh K=~/.ssh/thermograph_agent_ed25519 -# sudo: /etc/thermograph.env is root-owned (0640). On prod `agent` can read it -# directly too, but sudo works uniformly on both boxes. +# sudo: the env files are root-owned (0640). On these boxes `agent` can often +# read them directly too, but sudo works uniformly regardless. # # The character class must allow DIGITS. This was `^[A-Z_]+=` until 2026-07-24, # which silently skipped every key whose name contains a digit — in this estate @@ -44,9 +56,10 @@ K=~/.ssh/thermograph_agent_ed25519 # no audit: the invented finding sends someone to provision a credential that # already exists, and it was briefly recorded as a root cause of a real bug. # The match still stops at the `=`, so no value is ever read. -ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/prod.keys # prod -ssh -i $K agent@75.119.132.91 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/beta.keys # beta -python3 .claude/skills/key-gaps/key_gaps.py prod=/tmp/prod.keys beta=/tmp/beta.keys +ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/prod.keys # vps2: prod +ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph-beta.env' > /tmp/beta.keys # vps2: beta +ssh -i $K agent@75.119.132.91 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/dev.keys # vps1: dev +python3 .claude/skills/key-gaps/key_gaps.py prod=/tmp/prod.keys beta=/tmp/beta.keys dev=/tmp/dev.keys ``` ## Verify a SOPS cutover matches live diff --git a/infra/.env.example b/infra/.env.example index 3253125..3da1d93 100644 --- a/infra/.env.example +++ b/infra/.env.example @@ -1,9 +1,11 @@ # Local docker-compose overrides. Copy to .env (gitignored) for `docker compose -# up` / `make up` on a dev box: cp .env.example .env +# up` / `make up` on a laptop: cp .env.example .env # # Compose auto-reads a repo-root .env for ${VAR} interpolation in -# docker-compose.yml. In production these live in /etc/thermograph.env instead -# (loaded by the systemd unit), so this file is only for local runs. +# docker-compose.yml. On the fleet (dev/beta/prod), these values live in each +# environment's own rendered env file instead (/etc/thermograph.env, +# /etc/thermograph-beta.env — see deploy/secrets/README.md), so this file is +# only for a local, unmanaged run. # Database password. Compose uses it to initialize the postgres container AND to # build the app's THERMOGRAPH_DATABASE_URL. Change it before first `up`. @@ -33,8 +35,13 @@ THERMOGRAPH_INTERNAL_TOKEN= # replicate with another -- see docker-compose.yml's db service comment. # TIMESCALEDB_TAG=latest-pg18 -# Postgres sizing. Terraform sets these per host in prod/beta (prod DB_MEMORY -# 16g); local/beta default to 8g / 2 CPUs. +# Postgres sizing. Only meaningful for an environment that runs its OWN db +# service — dev's compose stack (this file) and prod's Swarm stack, which +# sizes the ONE shared TimescaleDB instance on vps2 (see +# deploy/secrets/prod.yaml, currently DB_MEMORY=16g). Beta shares that same +# instance rather than running a second one, so a DB_MEMORY/DB_CPUS value in +# beta's own vault file no longer sizes anything — don't be misled by its +# presence there. Local/dev default to 8g / 2 CPUs. # DB_MEMORY=8g # DB_CPUS=2 diff --git a/infra/ACCESS.md b/infra/ACCESS.md index 8ff1f4f..6713241 100644 --- a/infra/ACCESS.md +++ b/infra/ACCESS.md @@ -5,28 +5,55 @@ provisioning / Postgres / Terraform work described in `terraform/README.md` and (for the historical Track A/Track B plan this whole effort grew from, including the decision to split this repo out of the app monorepo) `thermograph-docs/runbooks/implementation-handoff.md` and -`thermograph-docs/architecture/repo-topology-and-infrastructure.md`. Terraform -provisions **prod** (the new 48 GB / 12-core box, `thermograph.org`) and -**beta** (the old VPS, `75.119.132.91`) — see `terraform/README.md` / -`terraform.tfvars.example`. **None of this touches the app repo** (its -backend/frontend source, `Dockerfile`, or CI) — this repo owns only how and -where the already-built app image runs; the app repo owns building it. +`thermograph-docs/architecture/repo-topology-and-infrastructure.md`. +**Note:** `terraform/README.md`'s own host table still describes the +pre-cutover shape (`prod` and `beta` as two single-purpose boxes) and has not +yet been updated to the vps1/vps2 split below — that file is out of this +pass's scope; treat this document as the current source of truth for the +physical topology in the meantime. ``` Track 1: deploy/provision-agent-access.sh — a dedicated full-root login for me -Track 2: deploy/swarm/ — Swarm cluster spanning THREE nodes +Track 2: deploy/swarm/ — Swarm mesh spanning THREE nodes Track 3: deploy/forgejo/ + .forgejo/ — Forgejo + Forgejo Actions, replacing GitHub ``` -**Three nodes, not two**: prod, beta, and **the desktop** (the LAN dev -machine — same box that already runs the pre-Forgejo GitHub self-hosted -runner). This matches the canonical Track B design in the handoff doc; an -earlier revision of this whole effort covered just prod+beta and has been -realigned. +## The estate (renamed by role, not by environment) + +The same two VPS boxes as always — **neither public IP nor WireGuard mesh +address moved** — plus the operator's desktop, now named for what they *do* +rather than which environment happens to live there: + +| Host | Public IP | Mesh IP | Role | +|------|-----------|---------|------| +| **vps1** | `75.119.132.91` | `10.10.0.2` | "Operational programs": Forgejo (git + CI + registry, `git.thermograph.org`), Grafana/Loki/Alloy (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio, and the **dev** environment (mesh-only, `10.10.0.2:8137`, no public DNS/Caddy/TLS) | +| **vps2** | `169.58.46.181` | `10.10.0.1` | "The deployed environment": **prod** and **beta**, as two separate Docker Swarm stacks, plus Centralis, Postfix, the backups, and the one shared TimescaleDB instance both stacks use | +| **desktop** | — | `10.10.0.3` | AI-model hosting (voice-to-text, an upcoming-feature LLM) plus flex Swarm-worker capacity. Hosts **no** Thermograph environment — `make dev-up` still works there, but only as a laptop-local rehearsal | + +This is a **rename**, not a re-provisioning: vps1 is the box that used to be +called "beta" (it already ran Forgejo and Grafana before this split), and vps2 +is the box that used to be called "prod". What moved is the **beta +environment** — off vps1 and onto vps2, so a beta green light is evidence +about prod (same orchestrator, same Postgres build, same Caddy, same mesh +position) — and the **dev environment**, off the desktop and onto vps1, so the +desktop can retire from the Thermograph estate entirely. See +`deploy/env-topology.sh`'s header comment for the full rationale. + +**Three nodes on the Swarm mesh, not two**: vps2 (manager), vps1 (worker), and +the desktop (worker, flex capacity + AI model hosting). One manager, not more: +Raft needs 3 nodes for real quorum-based HA, and this cluster only has 3 nodes +total, so a second manager would still fall short of real HA while adding +split-brain risk. If the manager (vps2) goes down, the workers keep running +whatever was already scheduled on them (Forgejo, pinned to vps1) but the +cluster can't reschedule anything until vps2 is back — acceptable for a small +cluster whose only Swarm-scheduled workload today is Forgejo (prod and beta's +app stacks are separate Swarm stacks that also happen to run on vps2, the +manager, because their volumes are local to that node — see +`deploy/stack/thermograph-stack.yml`'s header). ## Track 1 — Agent access -Run `sudo bash deploy/provision-agent-access.sh` on **prod and beta** (not +Run `sudo bash deploy/provision-agent-access.sh` on **vps1 and vps2** (not the desktop — that's wherever you're already working from, no separate access-provisioning step needed there). Creates a dedicated `agent` user (not raw root login — a distinct name gives a clean audit trail) with passwordless @@ -42,20 +69,27 @@ Both VPS boxes are provisioned and reachable as of this writing: | Host | Role | Public IP | Login | |------|------|-----------|-------| -| prod | Swarm manager; Thermograph's live prod home (`release`, thermograph.org) | `169.58.46.181` | `agent` | -| beta | Swarm worker; `main`/beta.thermograph.org + hosts Forgejo | `75.119.132.91` | `agent` | -| desktop | Swarm worker, LAN dev machine + Forgejo Actions runner | (local) | (already have access) | +| vps2 | Swarm manager; prod AND beta live here (`thermograph.org`, `beta.thermograph.org`) | `169.58.46.181` | `agent` | +| vps1 | Swarm worker; Forgejo + Grafana/Loki + dev (`git.thermograph.org`, `dashboard.thermograph.org`) | `75.119.132.91` | `agent` | +| desktop | Swarm worker, AI-model hosting | (local) | (already have access) | ``` -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # prod -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # beta +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # vps2 (prod + beta) +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # vps1 (dev + Forgejo + Grafana) ``` +**On vps2, always say which environment you mean.** A shell on vps2 can act on +either `/opt/thermograph` (prod) or `/opt/thermograph-beta` (beta) — there is +no "the app" on that box any more. `deploy.sh` refuses to run if +`THERMOGRAPH_ENV` disagrees with the checkout it was invoked from (see +`deploy/env-topology.sh`), which catches the one genuinely dangerous mistake +here: running prod's checkout with `THERMOGRAPH_ENV=beta`, or the reverse. + The private half of the agent's dedicated keypair lives at `~/.ssh/thermograph_agent_ed25519` on the operator's own machine (never in this repo, never in a CI secret) — same convention `DEPLOY.md` already uses -for the separate CI deploy key (`~/.ssh/thermograph_ci`). If you're setting -this up fresh elsewhere, generate a new pair the same way +for the separate CI deploy keys. If you're setting this up fresh elsewhere, +generate a new pair the same way (`ssh-keygen -t ed25519 -a 100 -C "claude-agent@thermograph-infra" -f ~/.ssh/thermograph_agent_ed25519 -N ""`), embed the new public half in `AGENT_PUBKEY` at the top of `provision-agent-access.sh`, and re-run it on @@ -67,31 +101,21 @@ directory that gets cleaned up, not anywhere-in-repo. Losing it just means regenerating and re-running the provisioning script; it does not lock either box, since your own login is untouched. -**Live as of 2026-07-21** (was: "no Docker / Terraform not applied / Swarm -inactive" — all now done): Docker is installed on all three nodes; the -WireGuard mesh and Docker Swarm are up with all three nodes `Ready` (prod -manager, beta + desktop workers); Terraform has been applied to both VPS -boxes (prod stood up fresh on `release`, beta rebuilt on `main`); Forgejo is -serving at `git.thermograph.org`; and the desktop runs the Forgejo Actions -runner. See the verification checklist at the end. - ## Track 2 — Swarm See `deploy/swarm/README.md` for the exact order of operations across all **three** nodes (WireGuard mesh first, then swarm init/join, then lock the -Swarm ports down to the tunnel interface, then label beta for Forgejo -placement). This cluster's only job is hosting Forgejo — it does not -orchestrate the Terraform-managed app deploys. +Swarm ports down to the tunnel interface, then label vps1 for Forgejo +placement). This cluster's Swarm-scheduled workload is Forgejo; prod and +beta's app stacks are separate `docker stack deploy`s that happen to live on +the same manager node (vps2) because their volumes are local to it today. ## Track 3 — Forgejo, replacing GitHub ### 3a. Stand up Forgejo -See `deploy/forgejo/README.md` — deploys the stack (pinned to beta), then -walks through registering the Actions runner **on the desktop** as a plain -systemd service (`register-lan-runner.sh`), not as a Swarm-scheduled -container. Only one Swarm secret to mint now (`forgejo_db_password`) — the -runner token is no longer a Swarm secret, since the runner isn't a Swarm -service anymore. +See `deploy/forgejo/README.md` — deploys the stack (pinned to **vps1**), then +walks through registering the Actions runner as a plain systemd service +(`register-lan-runner.sh`), not as a Swarm-scheduled container. ### 3b. Migrate the repo (mirror first, verify, then cut over) 1. In Forgejo: **+ New Migration → GitHub**. Point it at @@ -103,13 +127,12 @@ service anymore. 3. **Don't retarget any secret or disable a GitHub workflow yet** — see 3d. ### 3c. Workflows -`.forgejo/workflows/{build,pr-build,deploy-dev,deploy}.yml` mirror +`.forgejo/workflows/{build,pr-build,deploy}.yml` (a single `Deploy` workflow +now covers all three environments — see its own header comment) mirror `.github/workflows/*.yml`, copied with the mechanical changes Forgejo needs: -- `runs-on: ubuntu-latest` → `runs-on: docker` (one of the two labels the - desktop's runner registers under — see 3a; general CI/deploy jobs get a - fresh container, the LAN-deploy job in `deploy-dev.yml` runs bare/host-native - under the `thermograph-lan` label instead, since it needs real filesystem - access). +- `runs-on: ubuntu-latest` → `runs-on: docker` (the runner's `docker` label — + containerized jobs get a fresh container via the docker-socket-automounted + runner). - `appleboy/ssh-action` referenced by full GitHub URL (not mirrored in Forgejo's default action registry, unlike `actions/checkout` / `actions/setup-python`, which resolve unchanged). @@ -119,44 +142,51 @@ service anymore. `build` status check required (Settings → Branches). Forgejo does **not** auto-merge on green by itself — a ready, green, mergeable PR is merged **explicitly** (`POST .../pulls/{n}/merge`, `{"Do": "squash"}`), per - `CLAUDE.md`. That merge is an ordinary push, so `deploy-dev.yml`'s push + `CLAUDE.md`. That merge is an ordinary push, so `deploy.yml`'s push trigger fires naturally afterward. - **Branch/deploy mapping (settled):** `.forgejo/workflows/deploy.yml` - ("Deploy to beta VPS") triggers on `main` and SSHes to **beta** - (`75.119.132.91`) running `deploy/deploy.sh`. **Prod is not deployed by a - push-triggered workflow** — it's deployed with `terraform apply` - (`terraform/README.md`) on the `release` branch. So the promotion chain is - `dev` → `main` (this workflow deploys beta) → `release` (Terraform deploys - prod). There is deliberately no `release`-triggered workflow. + triggers on `dev`, `main` and `release`, and SSHes to the environment each + branch maps to (`dev` → dev on **vps1**, `main` → beta on **vps2**, + `release` → prod on **vps2**), running `deploy/deploy-dev.sh` or + `deploy/deploy.sh` per `deploy/env-topology.sh`. Credentials are keyed by + **host** (`VPS1_SSH_*`, `VPS2_SSH_*`), not by environment, since vps2 alone + now answers to two of them — see the workflow's own header comment for why + that rename matters. A second, independent set of Forgejo workflows -(`.forgejo/workflows/{ci,build-push,ops-cron}.yml`) was added by a parallel +(`.forgejo/workflows/{build-push,ops-cron}.yml`) was added by a parallel effort implementing `thermograph-docs/runbooks/implementation-handoff.md` Track A chunk -7 — see that doc and its own reconciliation PR (which already fixed its -runner-label and ssh-action guesses to match the real infrastructure here). -`ci.yml` duplicated this PR's `build.yml` exactly and was dropped in that -reconciliation; `build-push.yml` (image → Forgejo's built-in registry) and -`ops-cron.yml` (backup + IndexNow) are novel and still present. +7 — see that doc and its own reconciliation PR. `build-push.yml` (image → +Forgejo's built-in registry) and `ops-cron.yml` (backup + IndexNow) are novel +and still present; the latter's secrets are also keyed by host now (see its +own header comment). ### 3d. Cut over — DONE (2026-07-21) The GitHub → Forgejo cutover is complete; this records what was done: -1. Deploy secrets (`SSH_HOST`, `SSH_USER`, `SSH_KEY`, `SSH_PORT`) live in - Forgejo's repo Secrets, pointing at beta with a dedicated CI deploy key. -2. The LAN dev runner was re-pointed to Forgejo via - `deploy/forgejo/register-lan-runner.sh` (systemd --user, on the desktop). +1. Deploy secrets live in Forgejo's repo Secrets, keyed by host + (`VPS1_SSH_*`, `VPS2_SSH_*`) since the vps1/vps2 split, pointing at each + box with a dedicated CI deploy key. +2. The Actions runner is registered against Forgejo via + `deploy/forgejo/register-lan-runner.sh`. 3. The repo was migrated into Forgejo; `dev`/`main`/`release` all promoted through Forgejo and deploys verified live. 4. GitHub is retired as git host and CI (PR "Retire GitHub as the git host and CI platform") — `origin` remains only as a read-only mirror of history. -## Verification checklist — all met (2026-07-21) -- [x] `ssh agent@` and `ssh agent@` both work; `sudo whoami` → `root` +## Verification checklist — all met (2026-07-21, topology as of 2026-07-25) +- [x] `ssh agent@` and `ssh agent@` both work; `sudo whoami` → `root` - [x] Password SSH auth confirmed dead on both boxes (tested with an actual failed login attempt, not just config inspection) -- [x] `docker node ls` (from prod) shows all three nodes `Ready` +- [x] `docker node ls` (from vps2, the manager) shows all three nodes `Ready` - [x] Swarm ports (2377/7946/4789) unreachable from outside the WireGuard tunnel - [x] `https://git.thermograph.org` serves Forgejo over TLS; `/v2/` registry API is 403 from off-mesh (firewalled to the WireGuard CIDR) - [x] A test PR flow verified: required `build` check runs → explicit squash - merge → LAN dev deploy lands + merge → deploy lands on the branch's mapped environment - [x] GitHub retired as git host + CI; kept only as a read-only history mirror + +TODO(cutover): re-verify the checklist above against the vps1/vps2 rename +specifically (e.g. confirm the Forgejo Swarm label now reads `vps1`'s node +name, not a leftover `beta` node name) — this document was updated to match +the new topology brief, but a fresh live check wasn't run as part of this +pass. diff --git a/infra/CLAUDE.md b/infra/CLAUDE.md index ffa2576..390e0e4 100644 --- a/infra/CLAUDE.md +++ b/infra/CLAUDE.md @@ -8,33 +8,67 @@ orchestrator. ## The machines -Per `ACCESS.md`: **dev** (operator's box — LAN dev server and CI runner), -**prod** (`169.58.46.181`, thermograph.org, `agent` user, passwordless sudo), -**beta** (`75.119.132.91`, beta.thermograph.org + Forgejo + Grafana/Loki, -`agent` user). All four are on a WireGuard mesh. Hosts' `/opt/thermograph` is a -checkout of **this monorepo**, not an infra-only repo. +Two VPS boxes plus the operator's desktop, on one WireGuard mesh, named by +ROLE rather than by environment: + +- **vps1** — `75.119.132.91`, mesh `10.10.0.2`. "Operational programs": Forgejo + (git + CI + registry, `git.thermograph.org`), Grafana/Loki/Alloy + (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio site, and the + **dev** environment (`/opt/thermograph-dev`, tracks `dev`), with its own + Postgres container. Dev is mesh-only here — published on `10.10.0.2:8137`, + no public DNS, no Caddy site, no TLS. +- **vps2** — `169.58.46.181`, mesh `10.10.0.1`. "The deployed environment" — + everything an external user can reach: **prod** (`/opt/thermograph`, tracks + `main`, Swarm stack `thermograph`) AND **beta** (`/opt/thermograph-beta`, + tracks `main`, Swarm stack `thermograph-beta`), Centralis, Postfix, and the + backups. One shared TimescaleDB instance serves both, on separate + databases/roles (`thermograph` / `thermograph_beta`). +- **desktop** — mesh `10.10.0.3`. Hosts no Thermograph environment at all: AI + model hosting plus flex Swarm-worker capacity. `make dev-up` still works + there as a laptop-local rehearsal, but that is not an environment. + +`agent` user, passwordless sudo, on vps1 and vps2. A host's `/opt/thermograph*` +is a checkout of **this monorepo**, not an infra-only repo — note vps2 carries +**two** such checkouts side by side (`/opt/thermograph` and +`/opt/thermograph-beta`), which is the one thing most deploy logic here has to +get right that a single-environment host never had to. ## Deploy paths -- **`deploy/deploy.sh`** — the single entry point for beta and prod. Resets the - host checkout to `BRANCH` (default `main`), renders secrets, then routes: - if `/etc/thermograph/deploy-mode` says `stack` it execs - `deploy/stack/deploy-stack.sh`, otherwise it rolls compose services. Per-service - tags persist in untracked `deploy/.image-tags.env` (compose) and +- **`deploy/env-topology.sh`** — sourced by every path below. `THERMOGRAPH_ENV` + (`dev`/`beta`/`prod`) is the input; host, checkout, branch, deploy mode, + stack/compose name, env file, LB ports, DB role/database and service-name + prefix are all derived from it. This exists because vps2 alone runs two + environments — a host-wide marker can no longer answer "which environment is + this", so the caller (the deploy workflow) says so explicitly. +- **`deploy/deploy.sh`** — the single entry point for dev, beta and prod. + Resets the host checkout to `BRANCH`, renders secrets, then routes: beta and + prod (`TG_DEPLOY_MODE=stack`) exec `deploy/stack/deploy-stack.sh`; dev + (`compose`) rolls compose services. The old host-wide + `/etc/thermograph/deploy-mode` marker is honoured only when + `env-topology.sh` can't resolve an environment at all (a checkout that + predates the split, or a by-hand run with nothing set). Per-service tags + persist in untracked `deploy/.image-tags.env` (compose) and `deploy/.stack-image-tags.env` (stack) so the two never mix. -- **`deploy/stack/deploy-stack.sh`** — the Swarm path, live on prod. `backend` - rolls web **and** worker; `frontend` rolls frontend; `all` runs a full - `docker stack deploy`. Start-first, health-gated, auto-rollback. - `STACK_TEST=1` rehearses the whole thing under stack name `thermograph-test` +- **`deploy/stack/deploy-stack.sh`** — the Swarm path, live on vps2 for both + prod and beta, picking the stack file/ports/DB role/service prefix out of + `env-topology.sh`. `backend` rolls web **and** worker; `frontend` rolls + frontend; `all` runs a full `docker stack deploy`. Start-first, health-gated, + auto-rollback. `STACK_TEST=1` rehearses under stack name `thermograph-test` on throwaway volumes and ports `18137`/`18080`. -- **`deploy/deploy-dev.sh`** — thin LAN-dev wrapper (dev compose overlay, - `~/thermograph-dev`). Its CI trigger is inert; see root `CLAUDE.md`. +- **`deploy/deploy-dev.sh`** — thin wrapper around `deploy.sh` for dev on vps1 + (dev compose overlay, `/opt/thermograph-dev`, `THERMOGRAPH_SECRETS_SKIP_COMMON=1`). + Deployed like beta/prod now — a push to `dev` triggers the same `Deploy` + workflow over SSH, no LAN-specific runner involved. - **`Makefile`** — compose orchestration only: `up`, `down`, `db-up`, `dev-up`, `om-up`, `om-backfill`. Volumes are the reason the compose project name is pinned: compose creates `thermograph_pgdata`/`_appdata`/`_applogs`, and the Swarm stack declares those -same names as `external: true` at the same mount paths. +same names as `external: true` at the same mount paths. Beta's stack has no +`db` of its own — it reaches prod's `db` service over prod's `thermograph_internal` +overlay (declared `external: true` in beta's stack file) and keeps a separate +`internal` overlay for its own beta-to-beta traffic. ## Rules @@ -42,13 +76,29 @@ same names as `external: true` at the same mount paths. an apply would attempt full re-provisioning of live hosts. Terraform here is executable documentation until state is bootstrapped. - **Secrets: SOPS vault only** (`deploy/secrets/*.yaml`, `sops edit` → commit → - deploy). Never hand-edit `/etc/thermograph.env` — it is a rendered artifact. - `secrets-guard` CI rejects plaintext. `deploy/secrets/seed-from-live.sh` reads - production secrets and is **not** for an agent to run. + deploy). Never hand-edit `/etc/thermograph.env` / `/etc/thermograph-beta.env` + — they are rendered artifacts. `secrets-guard` CI rejects plaintext. + `deploy/secrets/seed-from-live.sh` reads production secrets and is **not** + for an agent to run. +- **vps1 must never hold prod credentials.** It's the box that runs Forgejo, + its CI runner, and dev's unreviewed branch — the opposite of an isolation + boundary. This is why dev renders `dev.yaml` **alone** and never layers + `common.yaml` (the fleet's shared production credential set). See + `deploy/secrets/README.md`. +- **A foothold on vps2 is a foothold on both beta and prod's host** — they are + co-resident by design now. What still separates them is the database + (separate roles/databases, `CONNECT` revoked from `PUBLIC`) and the + filesystem (separate checkouts, separate rendered env files). Don't describe + beta and prod as isolated at the host or SSH-credential level; they aren't + anymore. - **The ops cron (`.forgejo/workflows/ops-cron.yml`, at the repo root) is THE - prod backup.** It uses `PROD_SSH_*`, not `SSH_*` — an earlier revision reused - `SSH_*` and silently dumped beta while prod had no backup at all. If you touch - it, verify a dump actually lands in `agent@prod:~/thermograph-backups/`. + backup for both application databases and for Forgejo.** Secrets are keyed + by host, not environment: `VPS2_SSH_*` for the `backup`/`indexnow` jobs + (prod and beta's databases, both on vps2) and `VPS1_SSH_*` for + `forgejo-backup` (Forgejo now lives on vps1, not co-located with beta). If + you touch it, verify a dump actually lands for **both** the prod and beta + databases, not just one — an earlier revision backed up only prod and + silently left beta uncovered. - Shell here runs as root over SSH against live hosts with no test suite in front of it. `shell-lint` CI (pinned shellcheck) is the only guard — keep the tree at zero findings. diff --git a/infra/DEPLOY-DEV.md b/infra/DEPLOY-DEV.md index e66796b..65e3806 100644 --- a/infra/DEPLOY-DEV.md +++ b/infra/DEPLOY-DEV.md @@ -1,33 +1,45 @@ -# Dev CI/CD → LAN server on this machine +# Dev CI/CD → the dev environment on vps1 -Parallel to the prod pipeline (`main` → VPS, see `DEPLOY.md`), the **`dev`** -branch continuously deploys to a **LAN server running on this computer**. Git -hosting and CI are self-hosted **Forgejo** (`git.thermograph.org`, reachable -at `http://10.10.0.2:3080` over the WireGuard mesh) — GitHub is retired. +Parallel to beta/prod's pipeline (`main`→beta, `release`→prod, see `DEPLOY.md`), +the **`dev`** branch continuously deploys to the **dev environment on vps1** +(`75.119.132.91`, mesh `10.10.0.2`) — `/opt/thermograph-dev`, a normal fleet +checkout reached over SSH exactly like beta and prod, **not** a stack on the +operator's desktop or LAN any more. Git hosting and CI are self-hosted +**Forgejo** (`git.thermograph.org`, also on vps1, reachable at +`http://10.10.0.2:3080` over the WireGuard mesh) — GitHub is retired. ``` -open PR ──▶ CI (build + boot/health, on the LAN Forgejo runner) +open PR ──▶ CI (build + boot/health, on the `docker`-labeled Forgejo runner) │ green ▼ merge into dev (explicit — Forgejo does not auto-merge on its own; see "Landing a PR" below) │ ▼ - LAN runner (thermograph-lan label) runs deploy/deploy-dev.sh + the Deploy workflow SSHes into vps1 (VPS1_SSH_*, THERMOGRAPH_ENV=dev) + and runs deploy/deploy-dev.sh │ ▼ - ~/thermograph-dev updated ─▶ docker compose stack (backend + frontend + - Postgres 18) brought up, backend on 0.0.0.0:8137 + /opt/thermograph-dev updated ─▶ docker compose stack (backend + frontend + + Postgres) brought up, backend published + MESH-ONLY on 10.10.0.2:8137 ``` -Reachable at `http://:8137/` from any device on your Wi-Fi. +Reachable at `http://10.10.0.2:8137/` from anything already on the WireGuard +mesh, and from nowhere else — no public DNS record, no Caddy site, no TLS. +vps1 is a public VPS running whatever branch is currently in flight (including +unreviewed code), so unlike the old LAN box it must **never** publish on +`0.0.0.0`. -The dev server runs the **same containerized stack as prod** (`docker-compose.yml`), -overlaid with `docker-compose.dev.yml`: **uncapped** (no CPU limits, unlike prod's -backend=4 / frontend=2 / db=2) and backend published on `0.0.0.0:8137` for the -LAN (prod binds loopback behind Caddy; frontend has no published port here -either way, no Caddy to reach it directly, so it's only reached through -backend's own reverse-proxy fallback). Bring it up by hand with `make dev-up`. +The dev stack runs the **same containerized stack as prod** (`docker-compose.yml`), +overlaid with `docker-compose.dev.yml`: **uncapped** (no CPU limits, unlike +prod's Swarm-managed replicas) and backend published on the WireGuard address +(`10.10.0.2:8137` — set via `DEV_BIND_ADDR` in `deploy/env-topology.sh`; prod +and beta bind loopback behind their own Caddy LBs instead). Frontend has no +published port either way — reached only through backend's own +reverse-proxy fallback. Bring dev up by hand with `make dev-up`, which is a +**laptop-local convenience** (uncapped, loopback-published) — not the dev +environment itself, which only ever runs on vps1. ## Why it's built this way @@ -39,26 +51,40 @@ via the API (`POST .../pulls/{n}/merge`, `{"Do": "squash"}`). Nothing merges a ready PR for you automatically just because checks pass, unlike GitHub's old in-workflow `ci-cd.yml` (retired along with the rest of `.github/`). +**Dev moved off the desktop and onto vps1 so the desktop could retire from the +Thermograph estate entirely.** The desktop now hosts AI models and offers flex +Swarm-worker capacity — it is no longer "the LAN dev server" and no longer the +primary CI runner story. What dev keeps that beta and prod do not: + +- It renders `dev.yaml` **alone**, never layering `common.yaml` (the fleet's + shared production credentials) — vps1 also runs Forgejo and its CI, and dev + runs whatever branch is in flight. See the long note in + `deploy/render-secrets.sh` and `deploy/secrets/README.md`. +- It is **mesh-only** — see above. A public VPS running unreviewed branches + must never be reachable from the open internet the way the old LAN box + (behind a home router, reachable only on the Wi-Fi) safely could be. + ## Moving parts | File | Purpose | |------|---------| | `.forgejo/workflows/pr-build.yml` | required status check for PRs into `dev` (calls `build.yml`) | -| `.forgejo/workflows/deploy-dev.yml` | build + deploy on pushes to `dev` (direct, or a PR merge) | +| `.forgejo/workflows/deploy.yml` | the single Deploy workflow — `dev` branch pushes SSH into vps1 and run `deploy-dev.sh` | | `.forgejo/workflows/build.yml` | shared build gate: deps, backend tests, JS syntax check, boot/health check | -| `deploy/deploy-dev.sh` | pull `dev` into `~/thermograph-dev`, `docker compose up` the uncapped LAN stack, health check | +| `deploy/env-topology.sh` | source of truth: dev = vps1, `/opt/thermograph-dev`, branch `dev`, compose mode, mesh-only bind address | +| `deploy/deploy-dev.sh` | thin wrapper around `deploy.sh`: dev's compose overlay + secrets policy, then delegates to the shared pull/roll/health-check logic | | `deploy/secrets/dev.yaml` | dev's SOPS vault — its own secrets and config, rendered to `/etc/thermograph.env` at deploy time (see "Config and secrets") | -| `docker-compose.dev.yml` | dev overlay: no CPU limits, backend published on `0.0.0.0:8137` (frontend unpublished) | -| `deploy/provision-dev-lan.sh` | one-time sudo-free bootstrap of the checkout + runner | -| `deploy/forgejo/register-lan-runner.sh` | registers the Forgejo Actions runner on this machine | +| `docker-compose.dev.yml` | dev overlay: no CPU limits, backend published on the mesh address (frontend unpublished) | +| `deploy/provision-dev.sh` | one-time bootstrap of the checkout + vault marker on vps1 | +| `deploy/forgejo/register-lan-runner.sh` | registers the Forgejo Actions runner (name predates the vps1/vps2 split — see the script's own header) | -`deploy-dev.yml`'s `deploy` job runs on the `thermograph-lan` label — **not** -`[self-hosted, thermograph-lan]` (the array form is what the original GitHub -version used; GitHub implicitly tags every self-hosted runner `self-hosted`, -but Forgejo's runner has only the labels it was explicitly registered with, -so the array form is permanently unschedulable there — a real bug found and -fixed once, worth knowing about if a "deploy" job ever silently stops -appearing in the Actions history again). +`deploy.yml`'s dev leg SSHes in with `VPS1_SSH_*` and runs +`/opt/thermograph-dev/infra/deploy/deploy-dev.sh` directly over SSH — the same +shape as the beta and prod legs, just a different host and script. There is no +longer a `thermograph-lan`-labeled runner job doing host-native +`systemctl`/`docker-compose` calls from inside a CI job; the deploy happens the +same way for all three environments now (a `docker`-labeled runner opens an SSH +session to the target host and runs that environment's deploy script there). `deploy-dev.sh`'s git auth (`GH_TOKEN`/`GITHUB_TOKEN`, Forgejo's per-job token exposed under that name for compatibility) is scoped to `REPO_URL`'s @@ -68,15 +94,25 @@ hardcoded host would make git silently skip the header (no error) and fall through to whatever ambient credentials happen to be available, not fail loudly. -## The self-hosted runner +## The Forgejo Actions runner -A Forgejo Actions **runner** (`forgejo-runner`) runs on this machine, labels +TODO(cutover): `deploy/forgejo/register-lan-runner.sh`'s own header comment +still describes "the desktop" as the canonical placement for this runner, and +its `docker`-labeled jobs are what actually execute `deploy.yml`'s SSH-based +dev/beta/prod deploy steps today. Whether the runner has actually moved to +vps1 (alongside Forgejo) or still runs on the desktop under the new topology +is not settled by anything read for this pass — that script is out of this +document's edit scope. Confirm where it actually lives before relying on the +description below. + +A Forgejo Actions **runner** (`forgejo-runner`) serves this repo, labels `docker` (containerized jobs, node:20-bookworm, with the host's docker.sock automounted in — `container.docker_host: automount` in `~/forgejo-runner/config.yaml` — so `docker build`/`push` work without -privileged Docker-in-Docker) and `thermograph-lan` (host-native jobs, for -`deploy-dev.sh`'s systemctl/docker-compose calls). Installed under -`~/forgejo-runner`, runs as the `forgejo-runner` systemd `--user` service. +privileged Docker-in-Docker) and `thermograph-lan` (a legacy label from when a +deploy job ran host-native on the LAN box; nothing in the current `deploy.yml` +schedules against it, since every environment now deploys over SSH from the +`docker`-labeled job instead). ```bash # status / logs @@ -87,62 +123,66 @@ journalctl --user -u forgejo-runner -f bash deploy/forgejo/register-lan-runner.sh ``` -`git.thermograph.org`'s public DNS resolves to beta's **public** IP, which -the registry's mesh-only ACL (see `deploy/forgejo/README.md`) rejects for -`/v2/` paths — `docker build-push.yml` pushes run against the *host's* -automounted daemon, so it's this **host's** own resolver that needs to route -git.thermograph.org over the mesh, not anything settable from a job -container. If registry pushes ever start failing with an HTTP/TLS mismatch -or a 403 on `/v2/`, check `getent hosts git.thermograph.org` here — it needs -to resolve to beta's WireGuard IP (`10.10.0.2`), typically via a `/etc/hosts` -line, not the public one. +Registry pushes (`build-push.yml`) run against the runner host's own +automounted Docker daemon, so it's that **host's** own resolver that needs to +route `git.thermograph.org` over the mesh, not anything settable from a job +container. If registry pushes ever start failing with an HTTP/TLS mismatch or +a 403 on `/v2/`, check `getent hosts git.thermograph.org` on the runner host — +it needs to resolve to vps1's WireGuard IP (`10.10.0.2`), typically via a +`/etc/hosts` line, not the public one (a non-issue if the runner is co-located +with Forgejo on vps1 itself, since `git.thermograph.org` then resolves to +itself either way). -## The LAN dev service (Docker Compose) +## The dev environment (Docker Compose, on vps1) Runs as a Docker Compose stack, not a bare systemd unit — `docker ps --filter -name=thermograph-dev` shows `thermograph-dev-backend-1`, `thermograph-dev-frontend-1` -(repo-split Stage 4 split the single "app" container in two — frontend has no -published port, reached only through backend's own reverse-proxy fallback) and -`thermograph-dev-db-1` (TimescaleDB). A legacy pre-container -`thermograph-dev.service` systemd --user unit may still exist from before this -cutover; it's stale and `deploy-dev.sh` stops/disables it on every run — don't -trust `systemctl --user status thermograph-dev` as a liveness check, use `docker -ps` instead. +name=thermograph-dev` on vps1 shows `thermograph-dev-backend-1`, +`thermograph-dev-frontend-1` (frontend has no published port, reached only +through backend's own reverse-proxy fallback) and `thermograph-dev-db-1` +(TimescaleDB, dev's **own** container — not the shared instance on vps2). ```bash docker ps --filter name=thermograph-dev # is it up? -docker compose -f docker-compose.yml -f docker-compose.dev.yml logs -f backend frontend # app logs (from ~/thermograph-dev) +docker compose -f docker-compose.yml -f docker-compose.dev.yml logs -f backend frontend # app logs (from /opt/thermograph-dev/infra) ``` -Bootstrap (or rebuild) it by hand: +Bootstrap (or rebuild) it: ```bash -bash deploy/provision-dev-lan.sh +sudo bash deploy/provision-dev.sh ``` -The dedicated checkout at `~/thermograph-dev` is separate from your working tree, -so a deploy never touches uncommitted edits in `~/Code/Thermograph`. +replacing the retired `deploy/provision-dev-lan.sh`, which bootstrapped dev on +the operator's desktop as a sudo-free `systemd --user` stack. Dev is +provisioned like any other fleet environment now — a checkout at +`/opt/thermograph-dev`, root-owned, deployed over SSH by CI like beta and +prod. The dedicated checkout is separate from your own working tree, so a +deploy never touches uncommitted edits there. ## Config and secrets Dev's configuration lives in the same SOPS vault as prod's and beta's — `deploy/secrets/dev.yaml`, encrypted to the same age recipient, rendered to `/etc/thermograph.env` by the same `render-secrets.sh` at deploy time. Changing a -dev value is `sops deploy/secrets/dev.yaml` → commit → deploy, exactly as for the -VPS boxes. **`deploy/secrets/README.md` is the reference**; what follows is only -what is specific to this machine. +dev value is `sops deploy/secrets/dev.yaml` → commit → deploy, exactly as for +beta and prod. **`deploy/secrets/README.md` is the reference**; what follows +is only what is specific to this environment. Dev renders `dev.yaml` **alone**. It does *not* layer `common.yaml`, which is the fleet's shared production credential set — registry token, both S3 keypairs (one read-write pair, on the bucket that also holds prod's backups), the VAPID private key that signs push to real subscribers, the IndexNow key, the metrics token. This -box is the CI runner and runs unreviewed `dev`-branch code; a plaintext render of -that set into `/etc/thermograph.env` would put all of it in every dev container's -environment. `deploy-dev.sh` exports `THERMOGRAPH_SECRETS_SKIP_COMMON=1` to prevent -it, and **aborts the deploy** if the renderer in the checkout cannot honour that. +box also runs Forgejo and its CI, and executes unreviewed `dev`-branch code; a +plaintext render of that set into `/etc/thermograph.env` would put all of it in +every dev container's environment. `deploy-dev.sh` exports +`THERMOGRAPH_SECRETS_SKIP_COMMON=1` to prevent it, and **aborts the deploy** if +the renderer in the checkout cannot honour that. This is a property of the +`dev` **environment** (encoded in `deploy/env-topology.sh`'s `TG_SKIP_COMMON`), +not just of `deploy-dev.sh` — so it holds regardless of which entry point is +used to deploy it. So dev holds its own `THERMOGRAPH_AUTH_SECRET` (its own, not prod's — the `daemon` -service refuses to start without one), its own `POSTGRES_PASSWORD`, a LAN +service refuses to start without one), its own `POSTGRES_PASSWORD`, a mesh-only `THERMOGRAPH_BASE_URL`, `THERMOGRAPH_COOKIE_SECURE=0` (plain HTTP — the shared value is `1` and would silently break login here), and the non-secret config it would otherwise have inherited. It holds **no** S3 keys, no VAPID keypair, no @@ -150,20 +190,26 @@ IndexNow key, no Discord or mail credentials: the lake falls through to the Open-Meteo archive without bucket creds, and the app generates and persists its own VAPID and IndexNow keys into the `appdata` volume. -Two dev-only mechanics, both because this box has **no passwordless sudo** and the -runner is a `systemd --user` service with no tty: +Two dev-only mechanics carried over from the old LAN setup, both worth keeping +even though dev is no longer sudo-free (vps1's `agent` user has passwordless +sudo like any other fleet host) — the render path just doesn't strictly need +them any more: -- the age key is read from `~/.config/sops/age/keys.txt`, not - `/etc/thermograph/age.key` — a root-owned `0400` key would send the renderer down - its `sudo cat` path, where it would hang on a prompt no CI job can answer; -- `/etc/thermograph.env` is **pre-created** owned by the deploying user, so the - renderer takes its in-place write path and no deploy needs `sudo` at all. +- the age key can be read from `~/.config/sops/age/keys.txt` as a fallback + when `/etc/thermograph/age.key` isn't present or readable — see + `deploy-dev.sh`'s own comment on this; +- `/etc/thermograph.env` being pre-created and owned by the deploying user + lets the renderer take its in-place write path without needing `sudo` at + all, which still matters under the Forgejo runner's `systemd --user` + context (no tty to answer a `sudo` prompt). -Setting this up on a fresh dev machine is four typed commands — see **"Setting up a -new dev machine"** in `deploy/secrets/README.md`. Until the -`/etc/thermograph/secrets-env` marker exists the box renders nothing and falls back -to `deploy-dev.sh`'s built-in `POSTGRES_PASSWORD` default, which is the safe state -and the one to return to (delete the marker) if the vault ever gets in the way. +Setting this up on a fresh dev host is a few typed commands — see **"Setting up a +new dev machine"** in `deploy/secrets/README.md` (written for the old desktop +setup; the mechanics are the same on vps1, just root-owned rather than +sudo-free). Until the `/etc/thermograph/secrets-env` marker exists the box +renders nothing and falls back to `deploy-dev.sh`'s built-in +`POSTGRES_PASSWORD` default, which is the safe state and the one to return to +(delete the marker) if the vault ever gets in the way. `REGISTRY_TOKEN` is deliberately not in the vault: dev pulls with the persistent `docker login` credential already in this host's docker config, like the prod/beta @@ -196,25 +242,28 @@ with `{"Do": "squash", "delete_branch_after_merge": true}`. ## Day-to-day - **Ship:** open a PR into `dev`, confirm CI is green, merge it explicitly — - the merge triggers the deploy here. -- **Manual redeploy:** trigger `deploy-dev.yml` via `workflow_dispatch` - (`POST /api/v1/repos/{owner}/{repo}/actions/workflows/deploy-dev.yml/dispatches`, - `{"ref": "dev"}`), or locally `APP_DIR=~/thermograph-dev bash deploy/deploy-dev.sh`. -- **Promote to prod:** merge/fast-forward `dev` into `main` and push — that - triggers the VPS pipeline in `DEPLOY.md`. + the merge triggers a deploy to vps1. +- **Manual redeploy:** trigger the `Deploy` workflow via `workflow_dispatch` + (`POST /api/v1/repos/{owner}/{repo}/actions/workflows/deploy.yml/dispatches`, + `{"ref": "dev"}`), or directly on vps1: + `APP_DIR=/opt/thermograph-dev bash deploy/deploy-dev.sh`. +- **Promote to beta:** merge/fast-forward `dev` into `main` and push — that + triggers the beta leg of the same `Deploy` workflow, per `DEPLOY.md`. ## Monitoring -Logs and dashboards moved to a **separate project** — a central Grafana + Loki -stack fed by a Grafana Alloy agent on every node (including this LAN dev box), -over the WireGuard mesh (`thermograph-observability` on Forgejo; UI at -`https://dashboard.thermograph.org`). The dev node's Alloy agent ships its -container/app logs under `host="dev"`, so LAN dev shows up alongside prod and -beta in the same dashboards. This replaced the old `scripts/dashboard.py`. +Logs and dashboards live in a **separate project** — a central Grafana + Loki +stack, hosted on **vps1 itself**, fed by a Grafana Alloy agent on every node +(prod and beta on vps2, dev and the desktop's own agent, all over the +WireGuard mesh; `thermograph-observability` on Forgejo; UI at +`https://dashboard.thermograph.org`). Dev's Alloy agent ships its +container/app logs under its own `host`/service labels, so dev shows up +alongside prod and beta in the same dashboards. This replaced the old +`scripts/dashboard.py`. For a quick terminal check without Grafana, the app still serves raw counters at the gated `GET /api/v2/metrics` route. Dev sets no `THERMOGRAPH_METRICS_TOKEN`, so -the route is direct-loopback-only — `curl localhost:8137/api/v2/metrics` from this -machine, which is where you already are. (A token exists on the VPS boxes to allow -reads through an SSH tunnel; dev has no reason to carry a copy, and the estate's -copy must not land here — see `deploy/secrets/README.md`.) +the route is direct-loopback-only — `curl localhost:8137/api/v2/metrics` from vps1 +itself. (A token exists on prod/beta to allow reads through an SSH tunnel; dev has +no reason to carry a copy, and the estate's copy must not land here — see +`deploy/secrets/README.md`.) diff --git a/infra/DEPLOY.md b/infra/DEPLOY.md index 35bee32..958176b 100644 --- a/infra/DEPLOY.md +++ b/infra/DEPLOY.md @@ -1,62 +1,71 @@ -# Deploying Thermograph to a prod VPS +# Deploying Thermograph to prod (and beta) on vps2 -> **Prod and beta are now provisioned by Terraform** (`terraform/README.md`) and -> run as a **`docker compose` stack** (backend + frontend + Postgres/TimescaleDB; -> repo-split Stage 4 split the single "app" service in two), not the -> manual venv + systemd model this document originally described. Current shape: -> prod = new box `169.58.46.181`, branch `release`, deployed with `terraform -> apply`; beta = old box `75.119.132.91`, branch `main`, deployed when `main` is -> pushed (`.forgejo/workflows/deploy.yml` SSHes in and runs `deploy/deploy.sh`, -> which `docker compose pull`s the image `build-push.yml` already pushed for -> that commit and `up`s it -- repo-split Stage 6's registry-pull cutover; -> `deploy.sh` no longer builds in place). The **manual walkthrough below -> (`provision.sh`, `thermograph.service`, the venv) is legacy** — kept as a -> from-scratch, no-Terraform reference; the process model is compose, so the -> systemd/venv specifics no longer match how prod/beta actually run. +> **Prod and beta both run as Docker Swarm stacks on vps2** (`169.58.46.181`, +> mesh `10.10.0.1`) — two separate stacks, two separate checkouts +> (`/opt/thermograph` and `/opt/thermograph-beta`), sharing one host because a +> beta green light is meant to be evidence about prod (same orchestrator, same +> Postgres build, same Caddy, same mesh position). `deploy/env-topology.sh` is +> the single source of truth for which environment lives where; every path +> below sources it rather than hardcoding a path or a port. The **manual +> walkthrough below (`provision.sh`, `thermograph.service`, the venv) is +> legacy** — kept as a from-scratch, no-orchestrator reference; prod and beta's +> actual process model is a Swarm stack (`deploy/stack/`), and dev's is plain +> compose on vps1 (see `DEPLOY-DEV.md`), so the systemd/venv specifics below no +> longer match how any live environment actually runs. -Pipeline (beta, on `main`): **push to `main` on Forgejo → `build-push.yml` builds -+ pushes the image, tagged by SHA → Forgejo Actions SSHes to beta → -`deploy/deploy.sh` (`git reset` + `docker login` + `docker compose pull && up`, -schema migration runs in the app entrypoint) → Caddy fronts it with Let's Encrypt -TLS.** Prod (on `release`) is deployed with `terraform apply`, not a push -trigger — its `remote-exec` provisioner does the same pull-instead-of-build. -(GitHub is retired/archived — Forgejo, self-hosted at `git.thermograph.org` / -`10.10.0.2:3080` over the WireGuard mesh, is now the sole git host and CI.) +Pipeline, both environments: **push to `main` (beta) or `release` (prod) on +Forgejo → `build-push.yml` builds + pushes the image, tagged by SHA → the +single `Deploy` workflow (`.forgejo/workflows/deploy.yml`) SSHes into **vps2** +with `THERMOGRAPH_ENV=beta` or `THERMOGRAPH_ENV=prod` → `deploy/deploy.sh` +resolves the right checkout/branch/stack from `env-topology.sh`, `git reset`s +it, renders that environment's secrets, and execs `deploy/stack/deploy-stack.sh` +(schema migration runs as a one-shot task before the roll) → each environment's +own loopback Caddy LB fronts it with TLS.** Credentials are keyed by **host** +(`VPS2_SSH_*`), not by environment — vps2 answers to both prod and beta, so the +environment is passed as data (`THERMOGRAPH_ENV`), not baked into which secret +is used. (GitHub is retired/archived — Forgejo, self-hosted at +`git.thermograph.org` on **vps1** over the WireGuard mesh, is now the sole git +host and CI.) Files that make this work: -| File | Where it lives in prod | Purpose | +| File | Where it lives on vps2 | Purpose | |------|------------------------|---------| -| `.forgejo/workflows/deploy.yml` | Forgejo | CI job that SSHes in and runs the deploy script (beta, on `main`) | -| `.forgejo/workflows/build-push.yml` | Forgejo | builds + pushes the SHA-tagged image `deploy.sh`/Terraform pull | -| `deploy/deploy.sh` | `/opt/thermograph/deploy/deploy.sh` | `git reset` + `docker login` + `docker compose pull && up` + health check + warm | -| `docker-compose.yml` | `/opt/thermograph/docker-compose.yml` | app + Postgres/TimescaleDB services (the process model) | -| `deploy/thermograph.env.example` | `/etc/thermograph.env` | Postgres password, VAPID/auth secrets, `WORKERS`, sizing, base path | -| `deploy/Caddyfile` | `/etc/caddy/Caddyfile` | reverse proxy + automatic HTTPS | -| `terraform/` | run from your machine | provisions the box + brings the stack up (replaces `provision.sh`) | -| `deploy/provision.sh`, `deploy/thermograph.service` | *(legacy)* | pre-compose venv+systemd bootstrap — superseded by Terraform | +| `.forgejo/workflows/deploy.yml` | Forgejo | CI job that SSHes in and runs the deploy script, for all three environments | +| `.forgejo/workflows/build-push.yml` | Forgejo | builds + pushes the SHA-tagged image `deploy.sh` pulls | +| `deploy/env-topology.sh` | `/opt/thermograph{,-beta}/infra/deploy/env-topology.sh` | resolves `THERMOGRAPH_ENV` to a checkout, branch, stack name, ports, DB role | +| `deploy/deploy.sh` | `/opt/thermograph{,-beta}/infra/deploy/deploy.sh` | `git reset` + render secrets + routes to the Swarm-stack deploy | +| `deploy/stack/thermograph-stack.yml` / `thermograph-beta-stack.yml` | same | prod's / beta's Swarm stack (db, web, worker, lake, daemon, frontend; beta has no `db` of its own) | +| `deploy/thermograph.env.example` | `/etc/thermograph.env` (prod) / `/etc/thermograph-beta.env` (beta) | Postgres password, VAPID/auth secrets, `WORKERS`, sizing, base path | +| `deploy/stack/lb/Caddyfile` / `Caddyfile.beta` | each stack's loopback LB container | reverse proxy from the host's real Caddy into the Swarm overlay | +| `terraform/` | run from your machine | provisions the box (host-level only; does not know about the two-environment split — see `ACCESS.md`) | +| `deploy/provision.sh`, `deploy/thermograph.service` | *(legacy)* | pre-compose venv+systemd bootstrap — superseded by the Swarm stack | -The app serves on **loopback only**; Caddy is the only thing exposed to the -internet (ports 80/443). The parquet cache in `data/cache/` lives inside the -`/opt/thermograph` checkout and is gitignored, so `git pull` never touches it. +Each app serves on **loopback only** (prod `127.0.0.1:8137`/`:8080`, beta +`127.0.0.1:8237`/`:8180`); the host's real Caddy is the only thing exposed to +the internet (ports 80/443), reverse-proxying into whichever loopback LB +matches the domain requested. Each environment's own checkout keeps its own +gitignored cache under `data/cache/`, so a `git pull` in one never touches the +other's. **Any deploy host needs a `/etc/hosts` entry for the registry.** -`git.thermograph.org`'s public DNS resolves to beta's public IP, and beta's -own Caddy rejects `/v2/*` (the registry API) from anything outside the +`git.thermograph.org`'s public DNS resolves to **vps1's** public IP, and +vps1's own Caddy rejects `/v2/*` (the registry API) from anything outside the WireGuard mesh (10.10.0.0/24) — confirmed live: `docker login`/`pull` from -prod failed with a 403 until adding `10.10.0.2 git.thermograph.org` to -`/etc/hosts` (10.10.0.2 is beta's mesh IP; every deploy host is already on -the mesh, this is purely a DNS-routing gap, not a connectivity one). Beta +vps2 failed with a 403 until adding `10.10.0.2 git.thermograph.org` to +`/etc/hosts` (10.10.0.2 is vps1's mesh IP; every deploy host is already on +the mesh, this is purely a DNS-routing gap, not a connectivity one). vps1 itself doesn't need this — it's the box Forgejo runs on, so `git.thermograph.org` just resolves to itself either way. `build-push.yml`'s own header comment documents the same requirement for the CI runner host. -**Current production layout** (`deploy/Caddyfile`): the app owns -**`thermograph.org`** at its root (`THERMOGRAPH_BASE=/`, so uvicorn serves `/`, -`/calendar`, `/api/v2/…` with no prefix), and **`emigriffith.dev`** serves a -static portfolio at its root with `emigriffith.dev/thermograph*` permanently -redirecting to `thermograph.org` (prefix stripped). Both domains' `A` records -point at the same VPS; Caddy provisions a separate cert for each. +**Current production layout**: prod's Caddy config owns **`thermograph.org`** +at its root (`THERMOGRAPH_BASE=/`, so uvicorn serves `/`, `/calendar`, +`/api/v2/…` with no prefix) and beta's owns **`beta.thermograph.org`** the same +way. **`emigriffith.dev`** is a separate static portfolio site, served from +**vps1**, not vps2 — it shares no host, checkout or Caddy config with either +app environment. `emigriffith.dev/thermograph*` permanently redirects to +`thermograph.org` (prefix stripped). > **Note:** `deploy.sh` (the CI deploy) only pulls code, reinstalls deps, and > restarts the app service — it does **not** touch Caddy or `/etc/thermograph.env`. @@ -202,24 +211,32 @@ If the repo is **public**, skip this — `git pull` needs no auth. ## Accounts & notifications Accounts, subscriptions, and notifications are **authoritative and not -regenerable** — back them up. On **prod/beta they live in Postgres/TimescaleDB** -(the compose `db` service, on the `pgdata` volume) alongside the climate record; -on **LAN dev** they fall back to SQLite (`data/accounts.sqlite`) when -`THERMOGRAPH_DATABASE_URL` isn't a Postgres URL. +regenerable** — back them up. **Prod, beta and dev all keep them in +Postgres/TimescaleDB** — prod and beta on the one shared instance on vps2 (as +the `thermograph` and `thermograph_beta` databases, separate roles, `CONNECT` +revoked from `PUBLIC` — see `deploy/db/provision-env-db.sh`), dev in its own +`db` container on vps1. SQLite (`data/accounts.sqlite`) is only a fallback for +running the app **outside** any of these stacks (a bare venv, the test suite) +when `THERMOGRAPH_DATABASE_URL` isn't a Postgres URL — it's not what any live +environment actually uses. -- **Back up the Postgres volume** (e.g. `pg_dump` the `thermograph` database) as - part of the VPS backup routine — losing it wipes all accounts and alerts. (On - a SQLite LAN dev box, back up `data/accounts.sqlite` and its `-wal`/`-shm` - sidecars instead.) -- **Multiple workers, one notifier (leader election).** Prod runs several uvicorn - workers (`WORKERS`, currently 8 on prod / 4 on beta). The subscription - evaluator and the recurring scheduler must run in exactly one, so the workers - elect a single leader — a host-local `flock` (`THERMOGRAPH_SINGLETON_LOCK`, - set by the compose app service to `/app/data/notifier.lock`), or a cluster-wide - Postgres advisory lock (`THERMOGRAPH_SINGLETON_PG=1`) if the notifier must be - single across *hosts*. See `backend/core/singleton.py`. `THERMOGRAPH_ROLE` - (`web`/`worker`/`all`, default `all`) further restricts which processes may own - it, so a stateless web tier can scale without scaling notifiers. +- **Back up the Postgres data** (e.g. `pg_dump` the relevant database) as part + of each environment's backup routine — losing it wipes all accounts and + alerts for that environment. (Only a bare-venv/offline run backs up + `data/accounts.sqlite` and its `-wal`/`-shm` sidecars instead.) +- **Multiple workers, one notifier (leader election).** Each environment runs + several uvicorn workers (`WORKERS`, sized per environment — see + `deploy/secrets/{prod,beta,dev}.yaml` and each stack file's replica/worker + counts). The subscription evaluator and the recurring scheduler must run in + exactly one, so the workers elect a single leader — a host-local `flock` + (`THERMOGRAPH_SINGLETON_LOCK`, set by the compose/stack app service to + `/app/data/notifier.lock`), or a cluster-wide Postgres advisory lock + (`THERMOGRAPH_SINGLETON_PG=1`) if the notifier must be single across + *replicas on multiple hosts* — this is what prod's Swarm stack uses, since + `web` scales to N replicas under the autoscaler. See + `backend/core/singleton.py`. `THERMOGRAPH_ROLE` (`web`/`worker`/`all`, + default `all`) further restricts which processes may own it, so a stateless + web tier can scale without scaling notifiers. - **Env vars** (all optional, sensible defaults): - `THERMOGRAPH_COOKIE_SECURE` — set to `1` when serving over HTTPS (the VPS/TLS deploy) so the session cookie is `Secure`. Leave unset for plain-HTTP LAN, where @@ -296,9 +313,11 @@ Why route through a local MTA rather than a provider's API: `ReadWritePaths=…`) needs no change. **Default is safe:** `THERMOGRAPH_MAIL_BACKEND` defaults to `console` — it logs -the message and sends nothing — so LAN dev and the test suite exercise the whole -signup path with no mail server and no risk of mailing a real person. Production -opts in with `=smtp`. +the message and sends nothing — so dev and the test suite exercise the whole +signup path with no mail server and no risk of mailing a real person. Prod +opts in with `=smtp`; beta stays on `console` too (see +`deploy/stack/thermograph-beta-stack.yml`'s header for why beta must never +hold live mail or Discord credentials). ### Postfix unit topology — and the check that lies @@ -431,8 +450,8 @@ Check placement with , which scores all four at onc confirmation or password-reset link is mailed. It currently defaults to a per-boot random value, so every outstanding link would break on restart. - **Digest signups** land in the `pending_digest` table in the accounts DB - (Postgres on prod/beta; authoritative, back it up). The form ships ahead of - delivery on purpose: + (Postgres on every environment; authoritative, back it up). The form ships + ahead of delivery on purpose: addresses are collected now and confirmed once SMTP is live. - **Logs:** `journalctl -u postfix -f`; queue with `mailq`. @@ -464,26 +483,33 @@ regenerated with `python gen_cities.py [n_global] [n_english] [n_extended]`. proved unreliable (wrong matches), so they're edited by hand; a city not in the list simply shows its flavor blurb. Add entries freely — a test checks the slugs are valid. -- **Archive warming runs automatically on every deploy.** `deploy.sh` (prod) and - `deploy-dev.sh` (dev) launch `warm_cities.py` detached in the background after the - health check, so the ~750 city pages serve from cache and a crawl can't burst the - archive API quota. It's idempotent (skips already-cached cells), so only the first - deploy does the full ~25-min warm; later deploys just top up new cities. A page - also self-heals (fetches its archive once) if hit before warming finishes. To run - it by hand: `cd backend && python warm_cities.py --pace 2`. Logs: prod - `logs/warm-cities.log`; dev `journalctl --user -u thermograph-warm-cities`. +- **Archive warming runs automatically on every deploy of prod and dev — not + beta.** `deploy/stack/deploy-stack.sh` (prod) and `deploy.sh`'s compose path + (dev, via `deploy-dev.sh`) launch `warm_cities.py` detached in the background + after the health check, so the ~750 city pages serve from cache and a crawl + can't burst the archive API quota. Beta deliberately skips this + (`TG_POST_DEPLOY=0` in `env-topology.sh`) — it would spend the shared + upstream archive quota filling a cache only a rehearsal environment reads. + Warming is idempotent (skips already-cached cells), so only the first deploy + does the full ~25-min warm; later deploys just top up new cities. A page + also self-heals (fetches its archive once) if hit before warming finishes. To + run it by hand: `cd backend && python warm_cities.py --pace 2`. Logs land at + `logs/warm-cities.log` inside each environment's own backend container (dev + is a normal container stack on vps1 now, not a bare systemd unit; there's no + `journalctl --user` path any more). - **Submit the sitemap** in Google Search Console: `https://thermograph.org/sitemap.xml`. - `jinja2` is a new dependency (already in `requirements.txt`). ## Monitoring Fleet-wide logs and dashboards live in a **separate project** — a central -Grafana + Loki stack fed by a Grafana Alloy agent on every node, over the -WireGuard mesh (`thermograph-observability` on Forgejo; UI at -**`https://dashboard.thermograph.org`**, Google SSO). It ingests every -container's stdout, Caddy's access logs, and the app's structured JSON logs -(`logs/{errors,access,audit}/*.jsonl`). This replaced the old SSH-tailed -`scripts/dashboard.py`. +Grafana + Loki stack, hosted on **vps1** alongside Forgejo, fed by a Grafana +Alloy agent on every node over the WireGuard mesh (`thermograph-observability` +on Forgejo; UI at **`https://dashboard.thermograph.org`**, Google SSO). It +ingests every container's stdout, Caddy's access logs, and the app's +structured JSON logs (`logs/{errors,access,audit}/*.jsonl`) — from prod and +beta on vps2 as well as dev on vps1, each distinguishable by its `host`/service +labels. This replaced the old SSH-tailed `scripts/dashboard.py`. The app still exposes raw counters at the gated `GET /api/v2/metrics` route (loopback-only; refuses any request carrying a proxy `X-Forwarded-*` header, so diff --git a/infra/Makefile b/infra/Makefile index fd349bb..2feaa42 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -30,8 +30,14 @@ db-up: db-down: docker compose stop db -# The LAN dev stack: uncapped (no CPU limits) and published on 0.0.0.0:8137 so -# phones on the Wi-Fi can reach it (the base file is prod: capped + loopback). +# The dev overlay: uncapped (no CPU limits). Default publish is 0.0.0.0:8137, +# right for a laptop-local rehearsal (e.g. phones on the same Wi-Fi) -- this is +# also the *fleet* dev environment's own overlay, but that environment now runs +# on vps1 (a public VPS, not a LAN box) and deploys via deploy/deploy-dev.sh, +# which overrides the bind address to the WireGuard mesh IP only +# (DEV_BIND_ADDR, see deploy/env-topology.sh) -- never invoke `make dev-up` +# directly on a fleet host, or it publishes on every interface. (The base file +# is prod/beta's shape: capped + loopback, fronted by their own Caddy LB.) DEV_COMPOSE = docker compose -f docker-compose.yml -f docker-compose.dev.yml dev-up: POSTGRES_PASSWORD=$${POSTGRES_PASSWORD:-thermograph-dev} $(DEV_COMPOSE) up -d --build diff --git a/infra/README.md b/infra/README.md index fdd28e8..ed6b477 100644 --- a/infra/README.md +++ b/infra/README.md @@ -3,8 +3,9 @@ Infrastructure for [Thermograph](https://thermograph.org): Terraform host provisioning, the SOPS+age secrets vault, WireGuard/Swarm networking, Forgejo, Caddy, mail, and the deploy scripts that run the already-built app images on each -host. This is a domain of the `emi/thermograph` monorepo — hosts' `/opt/thermograph` -is a checkout of the whole monorepo, and `infra/` never builds app source; it only +host. This is a domain of the `emi/thermograph` monorepo — a host's checkout +(`/opt/thermograph`, `/opt/thermograph-beta`, or `/opt/thermograph-dev`) is a +checkout of the whole monorepo, and `infra/` never builds app source; it only runs published images. - **`terraform/`** — provisions/configures hosts (SSH-driven by default; an @@ -13,37 +14,61 @@ runs published images. executable documentation, not a routine operation. - **`deploy/secrets/`** — the git-native SOPS+age secrets vault (every app secret, encrypted at rest, rendered at deploy time). See `deploy/secrets/README.md`. -- **`deploy/swarm/`, `deploy/forgejo/`** — the WireGuard/Swarm cluster hosting - Forgejo (git + CI + registry). See `ACCESS.md`. -- **`deploy/deploy.sh`** — the single deploy entry point for beta and prod. - Takes `SERVICE=backend|frontend|all` plus `BACKEND_IMAGE_TAG`/`FRONTEND_IMAGE_TAG`, - resets the host checkout, renders secrets, and routes to the right orchestrator. -- **`deploy/stack/`** — the **Swarm** path, live on **prod**: - `thermograph-stack.yml` (db, web, worker, lake, daemon, frontend, autoscaler, - autoscaler-lake), `deploy-stack.sh`, `autoscale.sh`, and the LB. Rolling updates - are start-first, health-gated, with auto-rollback. `STACK_TEST=1` rehearses the - whole stack on throwaway volumes and ports. -- **`docker-compose*.yml`** — the **compose** path, live on **beta** and LAN dev - (db, backend, lake, daemon, frontend). `docker-compose.dev.yml` is the LAN - overlay; `docker-compose.openmeteo.yml` is the self-hosted Open-Meteo overlay. +- **`deploy/swarm/`, `deploy/forgejo/`** — the WireGuard/Swarm mesh spanning + vps1, vps2 and the desktop, and Forgejo (git + CI + registry), pinned to + vps1. See `ACCESS.md`. +- **`deploy/env-topology.sh`** — the single source of truth for where each + environment (`dev`/`beta`/`prod`) lives: host, checkout path, branch, deploy + mode, stack/compose name, env file, LB ports, DB role/database, service-name + prefix. Every deploy path sources it and derives its behavior from it rather + than guessing from the host it happens to be running on — necessary since + vps2 alone now runs two environments. +- **`deploy/deploy.sh`** — the single deploy entry point for dev, beta and prod. + Takes `SERVICE=backend|frontend|all` plus `BACKEND_IMAGE_TAG`/`FRONTEND_IMAGE_TAG` + (and, on vps2, `THERMOGRAPH_ENV=beta|prod` to say which of the two checkouts + it's acting on), resets the host checkout, renders secrets, and routes to the + right orchestrator per `env-topology.sh`. +- **`deploy/deploy-dev.sh`** — a thin dev-specific wrapper around `deploy.sh` + (dev compose overlay, dev's secrets policy). See `DEPLOY-DEV.md`. +- **`deploy/stack/`** — the **Swarm** path, live on **vps2** for both prod and + beta: `thermograph-stack.yml` (prod: db, web, worker, lake, daemon, frontend, + autoscaler, autoscaler-lake) and `thermograph-beta-stack.yml` (beta: the same + service shape minus `db` and the autoscalers, every service name prefixed + `beta-`). `deploy-stack.sh`, `autoscale.sh` and `lb/` are shared by both. + Rolling updates are start-first, health-gated, with auto-rollback. + `STACK_TEST=1` rehearses the whole stack on throwaway volumes and ports. +- **`docker-compose*.yml`** — the **compose** path, live only on **dev** (vps1) + (db, backend, lake, daemon, frontend). `docker-compose.dev.yml` is dev's + mesh-only overlay; `docker-compose.openmeteo.yml` is the self-hosted + Open-Meteo overlay (prod only). `make dev-up` also runs this path locally as + a laptop convenience — that is not an "environment", just a local rehearsal. -Which path a host takes is decided by `/etc/thermograph/deploy-mode`: the string -`stack` makes `deploy.sh` exec `deploy/stack/deploy-stack.sh`; anything else is -compose. The workflows never need to know which mode a host runs. +Which path an environment takes is decided by `deploy/env-topology.sh` +(`TG_DEPLOY_MODE`, keyed by `dev`/`beta`/`prod`): dev is `compose`, beta and prod +are both `stack`. The old host-wide marker `/etc/thermograph/deploy-mode` still +exists as a fallback for a by-hand run with no explicit environment, but it +cannot describe vps2, which runs two environments in two different checkouts — +so it is no longer the thing that decides where files go. The workflows never +need to know which mode an environment runs; they only pass `THERMOGRAPH_ENV`. ## Branches & how changes reach each environment -- **`main`** — what **prod and beta** run. `infra-sync.yml` fires on a push to - `main` touching `infra/**`, fast-forwards each host's `/opt/thermograph` checkout - and re-renders `/etc/thermograph.env` from the vault. It deliberately does - **not** roll any service: image tags are the app domains' axis, not infra's. A - compose or stack change that must recreate containers takes effect on the next - app deploy, or a by-hand `SERVICE=all … deploy/deploy.sh`. -- **`dev`** — what LAN dev would run via `deploy/deploy-dev.sh`. The CI trigger - for this is currently inert (the LAN box still holds a split-era checkout); use - `make dev-up` locally. -- **`release`** — consumed by app deploys only. Both hosts track infra via `main`; - prod's *app images* are staged by `release`, but its checkout follows `main`. +- **`dev`** — deploys to **dev on vps1** (`/opt/thermograph-dev`). +- **`main`** — deploys to **beta on vps2** (`/opt/thermograph-beta`). +- **`release`** — deploys to **prod on vps2** (`/opt/thermograph`). -Note the asymmetry with the app domains: app code IS environment-staged -(`dev`→`main`→`release` maps to LAN→beta→prod via image tags); infra is not. +App code IS environment-staged this way (`dev`→`main`→`release` maps to +vps1/dev → vps2/beta → vps2/prod via image tags, one `Deploy` workflow keyed by +branch). **Infra itself is not environment-staged** the same way: `infra-sync.yml` +fires on a push touching `infra/**` — on `dev` it fast-forwards vps1's dev +checkout, on `main` it fast-forwards **both** of vps2's checkouts (beta and +prod) — re-rendering each environment's own env file from the vault. It +deliberately does **not** roll any service: image tags are the app domains' +axis, not infra's. A compose or stack change that must recreate containers +takes effect on the next app deploy, or a by-hand `SERVICE=all … +deploy/deploy.sh` (or `deploy-dev.sh`) run. + +Note the asymmetry this leaves: dev's infra checkout tracks `dev`, the same +branch its app images are staged by. Beta's and prod's infra checkouts both +track `main` — prod's *app images* are staged by `release`, but prod's *infra +checkout* follows `main`, same as beta's. diff --git a/infra/deploy/Caddyfile b/infra/deploy/Caddyfile deleted file mode 100644 index 6baf615..0000000 --- a/infra/deploy/Caddyfile +++ /dev/null @@ -1,120 +0,0 @@ -# /etc/caddy/Caddyfile on the VPS. -# Each domain's A/AAAA record must already point at this VPS — Caddy provisions a -# Let's Encrypt cert on first request and auto-renews. Nothing else to do for TLS -# (just make sure ports 80 and 443 are open). -# -# Layout: -# thermograph.org/* -> the Thermograph app (path-split across -# backend/frontend -- see below) -# emigriffith.dev/ -> static portfolio site (served straight from disk) -# emigriffith.dev/thermograph* -> permanent redirect to thermograph.org (the app moved) -# -# Thermograph now owns thermograph.org's root, so both services run with -# THERMOGRAPH_BASE=/ (see /etc/thermograph.env) — pages, assets and API all sit -# at "/" with no sub-path prefix. Repo-split Stage 4: backend and frontend are -# two containers now (docker-compose.yml), each on its own loopback port -- -# Caddy path-splits directly to whichever owns a given path, so the browser -# still sees one apparent origin. Repo-split Stage 7a flipped which side owns -# the enumerated list: frontend now owns everything (content pages, the -# interactive tool's SPA shells, every static asset, the dynamic IndexNow key -# file) except the short, stable set below, which mirrors backend/web/app.py's -# own routing exactly (a single catch-all proxy to frontend for everything -# else) -- unlike frontend's paths, backend's don't grow every time a new -# static asset filename is added. A gap in this list still just degrades to -# "one extra hop" through backend's own proxy fallback, never a 404. - -thermograph.org { - encode zstd gzip - - @backend_paths path /api/* /digest /discord/interactions - - # Active health check on the same cheap /healthz route each container's own - # HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting - # never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10). - # 15s (was 5s): plenty responsive for a process that only restarts on a deploy, - # and a quarter of the polling load. - handle @backend_paths { - reverse_proxy 127.0.0.1:8137 { - health_uri /healthz - health_interval 15s - health_timeout 3s - health_status 2xx - } - } - - handle { - reverse_proxy 127.0.0.1:8080 { - health_uri /healthz - health_interval 15s - health_timeout 3s - health_status 2xx - } - } - - # Access-log hygiene: the default JSON encoder serializes full request headers, - # the TLS block, and response headers on every line (measured ~1,133B/line) -- - # strip those with the `filter` format encoder. Also strip the query string from - # the logged URI: Caddy's default logger records request.uri *including* the - # query string, so every `?q=` a visitor typed sat in Loki next to - # their client IP for the full 30-day retention -- a real privacy leak, not just - # noise. Bot/crawler skipping stays out of here: `log_skip` needs Caddy >= 2.7 - # and an upgrade is out of scope, so that's handled downstream in Alloy's - # loki.process "caddy" stage instead (see observability/alloy/config.alloy). - log { - output file /var/log/caddy/thermograph.log { - roll_size 20MiB - roll_keep 5 - } - format filter { - wrap json - fields { - request>headers delete - request>tls delete - resp_headers delete - request>uri regexp \?.* "" - } - } - } -} - -emigriffith.dev { - encode zstd gzip - - # Thermograph moved to its own domain. Send the old sub-path there with a - # permanent redirect, stripping the /thermograph prefix so deep links map - # straight across (…/thermograph/calendar -> thermograph.org/calendar). The - # bare /thermograph (no trailing slash) goes to the new root. - handle_path /thermograph/* { - redir https://thermograph.org{uri} permanent - } - handle /thermograph { - redir https://thermograph.org/ permanent - } - - # Portfolio at the root. Point `root` at the built static site (for the Astro - # portfolio that's its `dist/` output). file_server serves index.html for - # directories and returns a real 404 for missing paths. - handle { - root * /var/www/emigriffith - file_server - } - - log { - output file /var/log/caddy/emigriffith.log - } -} - -# Old bookmarks to the raw IP (the pre-domain URL) would otherwise get bounced to -# HTTPS-on-the-IP, which has no cert and fails. Redirect them to the portfolio domain. -http://75.119.132.91 { - redir https://emigriffith.dev{uri} permanent -} - -# Optional: redirect www -> apex for either domain. Add the www CNAME/A record -# first, then uncomment the matching block. -# www.emigriffith.dev { -# redir https://emigriffith.dev{uri} permanent -# } -# www.thermograph.org { -# redir https://thermograph.org{uri} permanent -# } diff --git a/infra/deploy/Caddyfile.vps1 b/infra/deploy/Caddyfile.vps1 new file mode 100644 index 0000000..113e5db --- /dev/null +++ b/infra/deploy/Caddyfile.vps1 @@ -0,0 +1,61 @@ +# /etc/caddy/Caddyfile on **vps1** (75.119.132.91) — the operational-programs box. +# +# vps1 serves: +# emigriffith.dev -> static portfolio site (from disk) +# git.thermograph.org -> Forgejo (see deploy/forgejo/caddy-git.conf) +# dashboard.thermograph.org -> Grafana (see observability/caddy-grafana.conf) +# +# and hosts the **dev** environment, which is deliberately ABSENT from this +# file. Dev has no site block, no DNS record and no certificate: it publishes on +# the WireGuard address only (http://10.10.0.2:8137) and is reachable from the +# mesh and nowhere else. Dev runs whatever branch is in flight, on the same box +# as Forgejo and its CI — putting it behind a public hostname would be handing +# the internet an unreviewed build. If you ever need it reachable from a phone, +# join the phone to the mesh; do not add a site block here. +# +# beta.thermograph.org is NOT here either — beta moved to vps2, next to prod. +# That is the one edit in this file most likely to be made by mistake: a "beta" +# reference elsewhere in the repo that still points at 75.119.132.91 means this +# box, which no longer runs beta at all. +# +# Each domain's A/AAAA record must already point here — Caddy provisions a +# Let's Encrypt cert on first request and auto-renews. + +emigriffith.dev { + encode zstd gzip + + # Thermograph moved to its own domain. Send the old sub-path there with a + # permanent redirect, stripping the /thermograph prefix so deep links map + # straight across (…/thermograph/calendar -> thermograph.org/calendar). The + # bare /thermograph (no trailing slash) goes to the new root. + handle_path /thermograph/* { + redir https://thermograph.org{uri} permanent + } + handle /thermograph { + redir https://thermograph.org/ permanent + } + + # Portfolio at the root. Point `root` at the built static site (for the Astro + # portfolio that's its `dist/` output). file_server serves index.html for + # directories and returns a real 404 for missing paths. + handle { + root * /var/www/emigriffith + file_server + } + + log { + output file /var/log/caddy/emigriffith.log + } +} + +# Old bookmarks to the raw IP (the pre-domain URL) would otherwise get bounced to +# HTTPS-on-the-IP, which has no cert and fails. Redirect them to the portfolio domain. +http://75.119.132.91 { + redir https://emigriffith.dev{uri} permanent +} + +# Optional: redirect www -> apex. Add the www CNAME/A record first, then +# uncomment. +# www.emigriffith.dev { +# redir https://emigriffith.dev{uri} permanent +# } diff --git a/infra/deploy/Caddyfile.vps2 b/infra/deploy/Caddyfile.vps2 new file mode 100644 index 0000000..48e2da3 --- /dev/null +++ b/infra/deploy/Caddyfile.vps2 @@ -0,0 +1,145 @@ +# /etc/caddy/Caddyfile on **vps2** (169.58.46.181) — the public-facing box. +# +# vps2 serves BOTH environments an external user can reach: +# thermograph.org -> prod (loopback LB on 127.0.0.1:8137 / :8080) +# beta.thermograph.org -> beta (loopback LB on 127.0.0.1:8237 / :8180) +# +# and Centralis at mcp.thermograph.org, which is provisioned separately. +# +# Each domain's A/AAAA record must already point here — Caddy provisions a +# Let's Encrypt cert on first request and auto-renews. Nothing else to do for +# TLS (just make sure ports 80 and 443 are open). +# +# This file was split out of a single deploy/Caddyfile that described both +# boxes' sites at once. That was workable while each box served an unrelated +# set; it stopped being workable when the two ports 8137/8080 started meaning +# "prod on vps2" here and nothing at all on the other box. See Caddyfile.vps1 +# for git/dashboard/portfolio. +# +# Thermograph owns thermograph.org's root, so both services run with +# THERMOGRAPH_BASE=/ — pages, assets and API all sit at "/" with no sub-path +# prefix. Backend and frontend are separate containers, each on its own loopback +# port; Caddy path-splits directly to whichever owns a given path, so the +# browser still sees one apparent origin. The enumerated backend list below +# mirrors backend/web/app.py's own routing exactly (a single catch-all proxy to +# frontend for everything else) -- unlike frontend's paths, backend's don't grow +# every time a new static asset filename is added. A gap in this list still just +# degrades to "one extra hop" through backend's own proxy fallback, never a 404. + +thermograph.org { + encode zstd gzip + + @backend_paths path /api/* /digest /discord/interactions + + # Active health check on the same cheap /healthz route each container's own + # HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting + # never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10). + # 15s (was 5s): plenty responsive for a process that only restarts on a deploy, + # and a quarter of the polling load. + handle @backend_paths { + reverse_proxy 127.0.0.1:8137 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + handle { + reverse_proxy 127.0.0.1:8080 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + # Access-log hygiene: the default JSON encoder serializes full request headers, + # the TLS block, and response headers on every line (measured ~1,133B/line) -- + # strip those with the `filter` format encoder. Also strip the query string from + # the logged URI: Caddy's default logger records request.uri *including* the + # query string, so every `?q=` a visitor typed sat in Loki next to + # their client IP for the full 30-day retention -- a real privacy leak, not just + # noise. Bot/crawler skipping stays out of here: `log_skip` needs Caddy >= 2.7 + # and an upgrade is out of scope, so that's handled downstream in Alloy's + # loki.process "caddy" stage instead (see observability/alloy/config.alloy). + # + # The FILENAME is load-bearing now: Alloy attributes each access log to an + # environment by filename (thermograph.log -> host="prod", beta.log -> + # host="beta"). One Caddy fronts two environments here, so a single log file + # would file every beta request under prod. Renaming either file means + # updating loki.process "caddy" in observability/alloy/config.alloy. + log { + output file /var/log/caddy/thermograph.log { + roll_size 20MiB + roll_keep 5 + } + format filter { + wrap json + fields { + request>headers delete + request>tls delete + resp_headers delete + request>uri regexp \?.* "" + } + } + } +} + +# Beta — same shape as prod, pointed at beta's loopback LB. It moved here from +# its own box; the DNS A record for beta.thermograph.org must point at vps2. +# +# Deliberately NOT indexed: beta serves the same content as prod at a different +# hostname, which is a duplicate-content problem for search engines and an +# invitation for users to land on a rehearsal environment from a search result. +# The app itself never pings IndexNow from beta (see env-topology.sh's +# TG_POST_DEPLOY), and this header closes the other half. +beta.thermograph.org { + encode zstd gzip + + header { + X-Robots-Tag "noindex, nofollow" + } + + @backend_paths path /api/* /digest /discord/interactions + + handle @backend_paths { + reverse_proxy 127.0.0.1:8237 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + handle { + reverse_proxy 127.0.0.1:8180 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + log { + output file /var/log/caddy/beta.log { + roll_size 20MiB + roll_keep 5 + } + format filter { + wrap json + fields { + request>headers delete + request>tls delete + resp_headers delete + request>uri regexp \?.* "" + } + } + } +} + +# Optional: redirect www -> apex. Add the www CNAME/A record first, then +# uncomment. +# www.thermograph.org { +# redir https://thermograph.org{uri} permanent +# } diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md new file mode 100644 index 0000000..72934a5 --- /dev/null +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -0,0 +1,420 @@ +# Runbook — moving beta to vps2 and dev to vps1 + +**Status: not executed.** Everything this runbook describes is landed in the +repo and live nowhere. The estate still runs the old shape until someone works +through the steps below. + +## What changes + +| | before | after | +|---|---|---| +| `75.119.132.91` (**vps1**) | beta + Forgejo + Grafana/Loki | Forgejo + Grafana/Loki + **dev** | +| `169.58.46.181` (**vps2**) | prod + Centralis + Postfix + backups | **prod + beta** + Centralis + Postfix + backups | +| desktop | LAN dev server + CI runner | AI models + flex Swarm worker, **no environment** | +| databases | one Postgres per environment | one instance on vps2, `thermograph` + `thermograph_beta` | + +Mesh addresses do **not** move: vps1 stays `10.10.0.2`, vps2 stays `10.10.0.1`. +Every "beta = 75.119.132.91" reference anywhere is wrong after this — that +address is vps1, and sending a beta-intended command there is the most likely +way to do damage during this cutover. + +## Order, and why it is this order + +Beta moves first and completely, while dev stays where it is. Then dev moves. +The two halves are independent, so a problem in one never forces a rollback of +the other — and beta is the half that carries a public hostname and a TLS +certificate, so it gets done while you have the most attention. + +Prod is touched twice, and both touches are additive: a new database + role on +its Postgres instance, and a new site block in its Caddyfile. Prod's stack file, +its services, its volumes and its env file are not modified by this work at all. + +**Expected downtime:** beta, roughly the length of a dump/restore plus a stack +bring-up (~10–20 min). Dev, as long as you like. **Prod: none** — unless step 3 +or 8 is done wrong, which is what the verification lines are for. + +--- + +## 0. Before you start + +- [ ] **Lower the DNS TTL** on `beta.thermograph.org` to 300s, at least an hour + before step 7. Do this first; it is the only step with a lead time. +- [ ] **Create the new Forgejo Actions secrets** (Settings → Secrets). The + workflows on this branch reference them and will fail without them: + + VPS1_SSH_HOST=75.119.132.91 VPS1_SSH_USER=agent + VPS1_SSH_KEY= VPS1_SSH_PORT=22 + VPS2_SSH_HOST=169.58.46.181 VPS2_SSH_USER=agent + VPS2_SSH_KEY= VPS2_SSH_PORT=22 + + Keep the old `SSH_*` / `PROD_SSH_*` secrets until step 11 — they are the + rollback path, and deleting them early strands you. + + **Create these BEFORE merging.** `infra-sync.yml` fires on any push to + `dev` or `main` touching `infra/**`, and this change touches a great deal + of it. Without the new secrets those jobs SSH to an empty host and fail; + `sync-beta` additionally targets `/opt/thermograph-beta`, which does not + exist until step 1. Nothing is damaged either way — the jobs only fetch + and render — but you will get a red run and a misleading alert. + + The app `Deploy` workflow does **not** fire on this merge: it is + path-filtered to `backend/**` and `frontend/**`, and this change touches + neither. +- [ ] **Take a fresh prod backup and confirm it landed**, rather than trusting + the schedule: + + ``` + # from a machine with repo access + # Actions -> "Ops cron (backup + IndexNow)" -> Run workflow + ssh agent@169.58.46.181 'ls -lh ~/thermograph-backups | tail -3' + ``` +- [ ] **Confirm you can decrypt the vault** (`sops -d infra/deploy/secrets/beta.yaml` + from `infra/`). Steps 3 and 5 need the rendered beta password. +- [ ] Have the PR merged to `dev` and promoted to `main`. Hosts pull `main`, so + nothing below works from an unmerged branch. + +--- + +## 1. Give beta a checkout on vps2 + +``` +ssh agent@169.58.46.181 +sudo git clone --branch main http://10.10.0.2:3080/emi/thermograph.git /opt/thermograph-beta +sudo chown -R agent:agent /opt/thermograph-beta +``` + +Prod's checkout at `/opt/thermograph` is untouched. Two checkouts side by side +is the whole point: a `git reset --hard` for beta must never be able to move +prod's tree. + +**Verify:** `git -C /opt/thermograph-beta log --oneline -1` matches +`git -C /opt/thermograph log --oneline -1`. + +--- + +## 2. Render beta's env file on vps2 + +``` +ssh agent@169.58.46.181 +cd /opt/thermograph-beta/infra +. deploy/render-secrets.sh +render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env +``` + +The age key is already at `/etc/thermograph/age.key` on this box (prod uses it); +beta needs no second copy. + +**Verify** — beta's file must name beta's role and database, and prod's must be +untouched: + +``` +sudo grep -c . /etc/thermograph-beta.env # non-zero +sudo grep -o 'thermograph_beta' /etc/thermograph-beta.env | head -1 +sudo grep -o '@db:5432/thermograph$' /etc/thermograph.env # prod: still 'thermograph' +``` + +--- + +## 3. Create beta's role and database on the shared instance + +This is the first step that touches prod's Postgres. It only ever adds; it drops +nothing. + +``` +ssh agent@169.58.46.181 +sudo bash /opt/thermograph-beta/infra/deploy/db/provision-env-db.sh beta +``` + +**Verify the isolation is real** — this is the whole justification for one +instance serving two environments, so actually run it: + +``` +DBC=$(docker ps -q --filter label=com.docker.swarm.service.name=thermograph_db | head -1) +# beta's role must NOT be able to reach prod's database: +docker exec "$DBC" psql -U thermograph_beta -d thermograph -c 'select 1' # must FAIL +# beta's read-only role exists and really is read-only: +docker exec "$DBC" psql -U thermograph_beta_ro -d thermograph_beta -c 'select 1' # must SUCCEED +docker exec "$DBC" psql -U thermograph_beta_ro -d thermograph_beta \ + -c 'create table _x(i int)' # must FAIL +# prod's own role is UNTOUCHED — still a superuser: +docker exec "$DBC" psql -U thermograph -d postgres -tAc \ + "select rolsuper from pg_roles where rolname='thermograph'" # must be t +# and prod is still healthy: +curl -fsS -o /dev/null -w '%{http_code}\n' https://thermograph.org/api/health +``` + +The script **refuses to run for prod** — prod's `thermograph` role is the +instance's bootstrap superuser, not a guest, and provisioning it would demote +it. That guard is why the superuser check above should never fail; run it +anyway. + +If the first command SUCCEEDS, stop. `CONNECT` was not revoked and beta would be +able to read production data; re-run the provisioning script and re-check before +going further. + +**Rollback:** `DROP DATABASE thermograph_beta; DROP ROLE thermograph_beta;` — +prod is unaffected either way. + +--- + +## 4. Move beta's data (optional) + +Beta's existing database still lives on vps1. Decide honestly whether you want +it: beta's value is a rehearsal of prod, and a fresh schema with a handful of +test accounts is often *better* than carrying old beta state across. If you do +want it: + +``` +# on vps1 — dump the old beta database +ssh agent@75.119.132.91 \ + 'docker exec thermograph-db-1 pg_dump -U thermograph -d thermograph --format=custom' \ + > /tmp/beta-premove.dump + +# on vps2 — restore into beta's database as beta's role +scp /tmp/beta-premove.dump agent@169.58.46.181:/tmp/ +ssh agent@169.58.46.181 ' + DBC=$(docker ps -q --filter label=com.docker.swarm.service.name=thermograph_db | head -1) + docker cp /tmp/beta-premove.dump "$DBC":/tmp/ + docker exec "$DBC" pg_restore -U thermograph -d thermograph_beta --no-owner --role=thermograph_beta /tmp/beta-premove.dump +' +``` + +`--no-owner --role=thermograph_beta` matters: the dump's objects are owned by the +old `thermograph` role, and restoring them verbatim would leave beta's tables +owned by a role beta does not connect as. + +**Do not** restore a *prod* dump into beta. Beta's daemon runs the notification +timers, and beta's subscriber table would then be full of real people. Beta's +vault sets `THERMOGRAPH_MAIL_BACKEND=console` so nothing would actually send, +but that is one `sops edit` away from not being true. + +--- + +## 5. Bring beta's stack up on vps2 + +``` +ssh agent@169.58.46.181 +cd /opt/thermograph-beta +THERMOGRAPH_ENV=beta SERVICE=all \ + BACKEND_IMAGE_TAG= \ + FRONTEND_IMAGE_TAG= \ + infra/deploy/deploy.sh +``` + +Get the current tags from the old beta host first +(`cat /opt/thermograph/infra/deploy/.image-tags.env` on vps1), so beta comes up +on exactly the code it was already running. + +**Verify** — beta answers on its own loopback ports, and prod's are unmoved: + +``` +curl -fsS -o /dev/null -w 'beta %{http_code}\n' http://127.0.0.1:8237/healthz +curl -fsS -o /dev/null -w 'prod %{http_code}\n' http://127.0.0.1:8137/healthz +docker stack services thermograph-beta # 5 services, all 1/1 +docker stack services thermograph # prod: unchanged, still 1/1 +``` + +**Verify the DNS-collision fix actually held** — this is the failure mode that +would be subtle and awful in production: + +``` +# prod's frontend must resolve prod's web, not beta's +docker exec $(docker ps -q -f label=com.docker.swarm.service.name=thermograph_frontend | head -1) \ + getent hosts web +docker exec $(docker ps -q -f label=com.docker.swarm.service.name=thermograph_frontend | head -1) \ + getent hosts beta-web # should NOT resolve on prod's network +``` + +**Rollback:** `docker stack rm thermograph-beta` and remove the LB container +(`docker rm -f thermograph-beta-lb`). Beta on vps1 is still running and still +serving; nothing has moved yet. + +--- + +## 6. Add beta's site block to vps2's Caddy + +`infra/deploy/Caddyfile.vps2` is the reference copy. Append its +`beta.thermograph.org` block to the live `/etc/caddy/Caddyfile` on vps2. + +``` +ssh agent@169.58.46.181 +sudo caddy validate --config /etc/caddy/Caddyfile # BEFORE reloading +sudo systemctl reload caddy +``` + +Validate before reload, always: a malformed Caddyfile takes `thermograph.org` +down with it, and prod is on this box now. + +Caddy cannot issue the certificate until DNS moves (step 7), so expect the beta +hostname to fail TLS until then. That is fine and expected. + +--- + +## 7. Move DNS + +Point `beta.thermograph.org`'s A record at **169.58.46.181**. + +Then wait for propagation and the certificate: + +``` +dig +short beta.thermograph.org # 169.58.46.181 +curl -fsS -o /dev/null -w '%{http_code}\n' https://beta.thermograph.org/api/health +ssh agent@169.58.46.181 'sudo journalctl -u caddy -n 30 --no-pager | grep -i certificate' +``` + +**Rollback:** point the A record back at 75.119.132.91. Beta on vps1 is still +up (you have not stopped it yet — that is step 9), so this is a clean revert +for as long as you leave it running. + +--- + +## 8. Stand dev up on vps1 + +Independent of everything above; do it whenever. + +``` +ssh agent@75.119.132.91 +sudo bash /opt/thermograph-dev/infra/deploy/provision-dev.sh # clones if absent +``` + +The script writes `/etc/thermograph/secrets-env` = `dev` and removes any +`deploy-mode` marker. Then deploy: + +``` +cd /opt/thermograph-dev +SERVICE=all BACKEND_IMAGE_TAG= FRONTEND_IMAGE_TAG= \ + infra/deploy/deploy-dev.sh +``` + +**Verify it is mesh-only.** This is the security-relevant check of the whole +cutover — vps1 is a public box and dev runs unreviewed branches: + +``` +ss -ltnp | grep 8137 # MUST show 10.10.0.2:8137, never 0.0.0.0:8137 +curl -fsS -o /dev/null -w '%{http_code}\n' http://10.10.0.2:8137/healthz # from the mesh +curl --max-time 5 http://75.119.132.91:8137/healthz # MUST fail/refuse +``` + +Also confirm dev did **not** get the fleet's shared production credentials — +`dev.yaml` is rendered alone, and vps1 is exactly the box that must not hold +them: + +``` +sudo grep -c 'THERMOGRAPH_S3_SECRET_KEY\|REGISTRY_TOKEN\|VAPID_PRIVATE' /etc/thermograph.env # expect 0 +``` + +--- + +## 9. Retire beta from vps1 + +Only after beta has been serving from vps2 through step 7 for long enough that +you believe it. + +``` +ssh agent@75.119.132.91 +cd /opt/thermograph/infra +docker compose down # stops the old beta app stack +# remove beta's site block from /etc/caddy/Caddyfile (see Caddyfile.vps1) +sudo caddy validate --config /etc/caddy/Caddyfile && sudo systemctl reload caddy +``` + +**Keep the old `thermograph_pgdata` volume on vps1** until you are certain beta +on vps2 is healthy and backed up. It is the only copy of pre-move beta data. +Delete it deliberately, later, not as part of this runbook. + +Forgejo, Grafana, Loki and the portfolio site all keep running on this box +untouched — do not stop anything else here. + +--- + +## 10. Repoint the Alloy agents + +Both nodes need the new variables; the label model changed (see +`observability/alloy/config.alloy`). + +``` +# vps2 — two environments, so two log volumes and the beta overlay +ssh agent@169.58.46.181 +cd /opt/thermograph/observability/alloy +ALLOY_NODE=vps2 ALLOY_ENV=prod \ +APPLOGS_VOLUME=thermograph_applogs BETA_APPLOGS_VOLUME=thermograph-beta_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d + +# vps1 +ssh agent@75.119.132.91 +cd /opt/thermograph/observability/alloy +ALLOY_NODE=vps1 ALLOY_ENV=dev APPLOGS_VOLUME=thermograph-dev_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + docker compose -f docker-compose.agent.yml up -d +``` + +**Verify each environment is labelled as itself** — the point of the change is +that beta's logs on vps2 are not filed as prod: + +``` +# in Grafana Explore, or via the Loki API: +# {host="beta"} |= "" -> should show beta traffic, from node="vps2" +# {host="prod"} |= "" -> prod only +# {host="dev"} |= "" -> from node="vps1" +# count by (host) (count_over_time({job="docker"}[5m])) -> three values, not one +``` + +If `{host="beta"}` is empty while beta is clearly serving, the container-name +relabel did not match — check `docker ps --format '{{.Names}}'` on vps2 against +the `/thermograph-beta_.*` regex. + +--- + +## 11. Clean up + +- [ ] Delete the old `SSH_*` and `PROD_SSH_*` Forgejo secrets (only now — they + were the rollback path). +- [ ] Run the ops cron by hand and confirm **both** databases are dumped: + `ssh agent@169.58.46.181 'ls ~/thermograph-backups | tail -4'` should show + both a `thermograph-*.dump` and a `thermograph_beta-*.dump`. +- [ ] Confirm the Forgejo backup job still works — it now uses `VPS1_SSH_*`, + and it follows Forgejo (vps1), not beta. +- [ ] Restore the DNS TTL on `beta.thermograph.org`. +- [ ] Desktop: stop the old LAN dev stack and free the box for the AI models. + Leave it joined to the Swarm as a worker; nothing schedules onto it today + (every app service is pinned `node.role == manager`), which is what makes + it safe to also run models there. +- [ ] Update Centralis — see below. + +--- + +## Centralis (outside this repo) + +Centralis' own text still describes the old estate, and it is not in this +repository. These need a separate change in the Centralis repo: + +- The `onboarding` tool's estate paragraph ("Four machines on a WireGuard + mesh... beta (beta.thermograph.org, Forgejo, Grafana + Loki), the operator's + desktop (LAN dev, the main CI runner)"). +- The MCP server's own protocol-level `instructions` string, which carries a + second, separately-worded copy of the same paragraph. +- The `thermograph-orientation` skill's estate table, and `thermograph-ops`' + release-flow diagram ("LAN dev box"), its "ssh -L works for beta and is + impossible for prod" claim (beta is a Swarm overlay now, so it behaves like + prod), and its prod-only Swarm service-name list (beta has + `beta-`-prefixed ones now). +- Tool descriptions that hardcode the old shape: `fleet_status` ("Swarm services + on prod, compose containers on beta/dev"), `estate_status` and `rollback_to` + ("Centralis cannot SSH to the dev desktop"), `logs_query`'s per-environment + service lists (Forgejo/Grafana/Loki are listed under `beta`; they belong under + `dev`/vps1 now), and `secrets_render`'s assumption that a host's own + `secrets-env` marker identifies one environment — vps2 has two. + +--- + +## If it goes wrong + +| symptom | most likely cause | action | +|---|---|---| +| prod 502s after step 5 | beta's stack collided with prod's service DNS | `docker stack rm thermograph-beta`; prod recovers on its own. Check beta's services are `beta-`-prefixed | +| beta 502s, prod fine | beta's LB or stack not up | `docker ps \| grep beta-lb`; `docker stack ps thermograph-beta --no-trunc` | +| beta can't reach the database | role/database missing, or the wrong network | re-run `provision-env-db.sh beta`; confirm beta's tasks are on `thermograph_internal` | +| beta serves prod's data | beta's env file has prod's URL | check `/etc/thermograph-beta.env` names `thermograph_beta` twice; re-render (step 2). **Stop beta until fixed** | +| prod's nightly backup fails | `thermograph_beta` doesn't exist yet | either finish step 3 or revert the ops-cron change; the job fails loudly by design rather than skipping silently | +| dev reachable from the internet | `DEV_BIND_ADDR` not applied | `ss -ltnp \| grep 8137`; redeploy via `deploy-dev.sh`, which sets it from `env-topology.sh` | diff --git a/infra/deploy/backup/README.md b/infra/deploy/backup/README.md index 5bbc23e..1f2e3b3 100644 --- a/infra/deploy/backup/README.md +++ b/infra/deploy/backup/README.md @@ -6,11 +6,21 @@ single VPS. Driven by `.forgejo/workflows/ops-cron.yml` (03:00 UTC daily + ## What is backed up -| Prefix in bucket `era5-thermograph` | Source | Job | Retention off-box | -|---|---|---|---| -| `backups/db/prod/` | prod Postgres/TimescaleDB (`pg_dump --format=custom`) | `backup` (prod, `PROD_SSH_*`) | 30 days | -| `backups/forgejo/db/` | Forgejo Postgres (`forgejo_db`) | `forgejo-backup` (beta, `SSH_*`) | 30 days | -| `backups/forgejo/data/` | Forgejo data volume (repos/LFS/config) | `forgejo-backup` | 30 days | +| Prefix in bucket `era5-thermograph` | Source | Job | Host / secrets | Retention off-box | +|---|---|---|---|---| +| `backups/db/prod/` | prod's `thermograph` database (`pg_dump --format=custom`) | `backup` | vps2, `VPS2_SSH_*` | 30 days | +| `backups/db/beta/` | beta's `thermograph_beta` database, same shared instance | `backup` | vps2, `VPS2_SSH_*` | 30 days | +| `backups/forgejo/db/` | Forgejo Postgres (`forgejo_db`) | `forgejo-backup` | vps1, `VPS1_SSH_*` | 30 days | +| `backups/forgejo/data/` | Forgejo data volume (repos/LFS/config) | `forgejo-backup` | vps1, `VPS1_SSH_*` | 30 days | + +Both application databases live on the ONE shared TimescaleDB instance on +vps2 now (separate roles/databases, not separate boxes) — the `backup` job +dumps each by name so beta isn't silently left uncovered. Forgejo moved to +**vps1** with the vps1/vps2 rename (it used to be co-located with beta on the +box now called vps2's predecessor); its backup targets vps1 accordingly. See +`ops-cron.yml`'s own header comment for the history: an earlier revision keyed +these secrets by environment name rather than host, which is what let "the +prod backup" silently dump the wrong box for a while. Everything is streamed through **`age`** encryption before upload — encrypted to the vault recipient `age1xx4dz…`, so the private key each host already has at @@ -23,9 +33,10 @@ decrypts it. Nothing plaintext leaves the box. (also mirrored into the SOPS vault `deploy/secrets/{prod,beta}.yaml` for host-side use). Contabo needs **path-style** addressing (`force_path_style=true`, `provider=Other`, `region=default`). -- `rclone` + `age` must be installed on prod and beta (one-time: `apt-get install -y - rclone age`). The prod job apt-installs them if missing; the beta user has no sudo, - so they are pre-installed there. +- `rclone` + `age` must be installed on vps1 and vps2 (one-time: `apt-get install -y + rclone age`). The prod/beta job on vps2 apt-installs them if missing; check + whether the Forgejo-backup job's user on vps1 has the sudo to do the same, or + whether they need pre-installing there. ## Restore (DR runbook) @@ -33,17 +44,27 @@ Prereq on the restoring host: `rclone` + `age` + the age key at `/etc/thermograph/age.key` (or the operator key), and rclone configured for the `archive:` remote (env vars `RCLONE_CONFIG_ARCHIVE_*` or `/etc/rclone/rclone.conf`). -### Prod database +### Prod / beta database + +Both restore into the SAME shared instance on vps2 (`` is that +one instance's container/task, resolved the same way `dbq.sh` does) — only +the bucket prefix and the target database/role differ: + ```sh # list available encrypted dumps rclone ls archive:era5-thermograph/backups/db/prod/ +rclone ls archive:era5-thermograph/backups/db/beta/ # download newest, decrypt, restore into a fresh db OBJ=archive:era5-thermograph/backups/db/prod/.dump.age rclone cat "$OBJ" | sudo age -d -i /etc/thermograph/age.key \ | docker exec -i pg_restore -U thermograph -d thermograph --clean --if-exists +# beta: same instance, its own role/database +OBJ=archive:era5-thermograph/backups/db/beta/.dump.age +rclone cat "$OBJ" | sudo age -d -i /etc/thermograph/age.key \ + | docker exec -i pg_restore -U thermograph -d thermograph_beta --clean --if-exists ``` -### Forgejo +### Forgejo (restore on vps1) ```sh # database rclone cat archive:era5-thermograph/backups/forgejo/db/.dump.age \ diff --git a/infra/deploy/db/init/20-tuning.sh b/infra/deploy/db/init/20-tuning.sh index eabc6fe..4791d10 100755 --- a/infra/deploy/db/init/20-tuning.sh +++ b/infra/deploy/db/init/20-tuning.sh @@ -1,7 +1,10 @@ #!/bin/bash # Postgres memory / performance tuning, scaled to the container's DB_MEMORY budget so -# the same init serves every host (beta 8g; prod 16g on the 48 GB box) with no -# hardcoding. Runs once on a fresh data volume from /docker-entrypoint-initdb.d, after +# the same init works for any host with no hardcoding. There is ONE shared TimescaleDB +# instance now (prod and beta are separate databases/roles on it, not separate +# containers), sized from prod's vault values (16g on the 48 GB box) -- beta no longer +# gets a second database or a second tuning pass. Dev keeps its own, separate container +# (8g). Runs once on a fresh data volume from /docker-entrypoint-initdb.d, after # 10-timescaledb.sql enables timescaledb. Settings are written via ALTER SYSTEM # (persisted to postgresql.auto.conf, which the timescaledb image's own # timescaledb-tune postgresql.conf defers to); the container's post-init restart brings diff --git a/infra/deploy/db/provision-env-db.sh b/infra/deploy/db/provision-env-db.sh new file mode 100755 index 0000000..334612c --- /dev/null +++ b/infra/deploy/db/provision-env-db.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Give an environment its own role + database on the SHARED Postgres instance. +# +# sudo bash infra/deploy/db/provision-env-db.sh beta # run on vps2 +# +# Since beta moved onto vps2 there is ONE TimescaleDB instance serving two +# environments. That is a capacity decision — one Postgres to tune, back up and +# keep on one extension build — and it is emphatically NOT a decision to let the +# two environments see each other's data. This script is what makes the second +# half true: +# +# * a role per environment (thermograph_beta), with its own password taken +# from that environment's own vault render, +# * a database per environment (thermograph_beta) OWNED by that role, +# * CONNECT revoked from PUBLIC on it, so the split is enforced by Postgres +# rather than by everyone remembering to use the right URL, +# * and NO superuser, NO CREATEDB, NO CREATEROLE on that role. +# +# The prod role keeps its own database and cannot be reached with beta's +# credentials; beta's role cannot connect to prod's database at all. +# +# Idempotent by construction — safe to re-run after a password rotation (it +# re-applies the password) or on a fresh instance. It never drops anything. +# +# WHY THIS IS NOT WIRED INTO deploy.sh: creating roles and databases is a +# privileged, once-per-environment act, and a deploy that can mint database +# roles is a deploy that can mint them wrongly at 3am. Run it by hand from the +# runbook when an environment is first stood up, and after a password rotation. +set -euo pipefail + +ENV_NAME="${1:?usage: provision-env-db.sh (beta|prod)}" + +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../env-topology.sh" +thermograph_topology "$ENV_NAME" + +if [ "$TG_DEPLOY_MODE" != stack ]; then + echo "!! $ENV_NAME does not use the shared instance (dev keeps its own db container)" >&2 + exit 2 +fi + +# REFUSE to run for the environment that OWNS the instance. +# +# This script provisions a GUEST environment onto someone else's Postgres. Run +# for prod, it would target prod's `thermograph` role — which is the instance's +# bootstrap superuser, not a guest — and the `ALTER ROLE ... NOSUPERUSER` below +# would strip superuser from the role the whole instance is administered with. +# Prod's roles predate this script and are created by the container's own +# initdb; there is nothing here for prod to need. +if [ "$TG_DB_SERVICE" = "${TG_STACK_NAME}_db" ]; then + echo "!! '$ENV_NAME' OWNS this Postgres instance (${TG_DB_SERVICE} is its own stack's db)." >&2 + echo "!! This script provisions a guest environment onto a shared instance." >&2 + echo "!! Running it here would demote ${TG_DB_USER} from superuser. Refusing." >&2 + exit 2 +fi + +# The database SERVER is prod's, wherever we are provisioning FOR. +DB_CID=$(docker ps -q --filter "label=com.docker.swarm.service.name=${TG_DB_SERVICE}" | head -1) +if [ -z "$DB_CID" ]; then + echo "!! no running task for ${TG_DB_SERVICE} on this host" >&2 + echo "!! run this on vps2, with prod's stack up." >&2 + exit 1 +fi + +# The new role's password is whatever that environment's vault says it is, so +# the database and the app can never disagree about it. Read from the rendered +# env file (a deploy of that environment writes it) rather than from sops here: +# this script should not need the age key. +if [ ! -r "$TG_ENV_FILE" ]; then + PW=$(sudo grep -m1 '^POSTGRES_PASSWORD=' "$TG_ENV_FILE" 2>/dev/null | cut -d= -f2- || true) +else + PW=$(grep -m1 '^POSTGRES_PASSWORD=' "$TG_ENV_FILE" | cut -d= -f2- || true) +fi +if [ -z "${PW:-}" ]; then + echo "!! no POSTGRES_PASSWORD in $TG_ENV_FILE" >&2 + echo "!! deploy $ENV_NAME once first so the vault renders it, then re-run." >&2 + exit 1 +fi + +echo "==> Provisioning role/database '${TG_DB_USER}'/'${TG_DB_NAME}' on ${TG_DB_SERVICE}" + +# The bootstrap superuser of the instance is prod's POSTGRES_USER (`thermograph`), +# and psql runs inside the container over its local socket — the database is +# never exposed on a TCP port anyone outside the overlay can reach. +# +# The password reaches psql as an environment variable rather than a -v +# argument, so it is not in psql's argv inside the container. It IS briefly in +# the `docker exec` argv on the host; that is visible only to root on vps2, who +# can read the vault render anyway. It is never interpolated into SQL text: +# :'pw' is psql's quote-and-escape form, which is also what makes a password +# containing a quote safe. +# +# Note both statements below are generated and run via \gexec rather than a +# DO block. psql does NOT substitute :variables inside dollar-quoted strings, +# so a DO $$ ... :'role' ... $$ body would be sent to the server literally. +docker exec -i -e ENV_DB_PASSWORD="$PW" "$DB_CID" \ + psql -v ON_ERROR_STOP=1 -U thermograph -d postgres \ + -v role="$TG_DB_USER" -v dbname="$TG_DB_NAME" <<'SQL' +\set pw `echo "$ENV_DB_PASSWORD"` + +-- Role: create if absent. +SELECT format('CREATE ROLE %I LOGIN', :'role') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'role') +\gexec + +-- Always (re)apply the password and the negative privileges, so a vault +-- rotation is just "rotate, redeploy, re-run this" — and so a role that was +-- created by hand with more rights than it should have gets corrected. +ALTER ROLE :"role" WITH LOGIN PASSWORD :'pw' NOSUPERUSER NOCREATEDB NOCREATEROLE; + +-- CREATE DATABASE cannot run inside a transaction block, hence \gexec here too. +SELECT format('CREATE DATABASE %I OWNER %I', :'dbname', :'role') +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'dbname') +\gexec +SQL + +# Extension + privileges have to run INSIDE the new database, hence a second +# connection. CREATE EXTENSION needs superuser, which is why it is done here +# rather than left to the app's own migration to attempt as the env role. +docker exec -i "$DB_CID" \ + psql -v ON_ERROR_STOP=1 -U thermograph -d "$TG_DB_NAME" \ + -v role="$TG_DB_USER" -v dbname="$TG_DB_NAME" -v rorole="${TG_DB_USER}_ro" <<'SQL' +CREATE EXTENSION IF NOT EXISTS timescaledb; + +-- Nobody but this environment's roles connects to this database. Without this, +-- PUBLIC retains CONNECT and any role on the instance could read it. +REVOKE CONNECT ON DATABASE :"dbname" FROM PUBLIC; +GRANT CONNECT ON DATABASE :"dbname" TO :"role"; + +-- The role owns the database but not necessarily the public schema, which is +-- owned by the bootstrap superuser on a fresh database; Alembic needs to create +-- tables in it. +ALTER SCHEMA public OWNER TO :"role"; + +-- The read-only role that ad-hoc queries use (ops/dbq.sh). Every environment +-- gets one, named _ro, so a human or an agent poking at data cannot +-- write. Without it, beta's queries would have had to connect as the OWNER — +-- read-write on its own database — losing a guarantee prod and dev already had +-- purely because beta was newer. +SELECT format('CREATE ROLE %I LOGIN', :'rorole') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'rorole') +\gexec +ALTER ROLE :"rorole" WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE; + +GRANT CONNECT ON DATABASE :"dbname" TO :"rorole"; +GRANT USAGE ON SCHEMA public TO :"rorole"; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO :"rorole"; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO :"rorole"; +-- Future tables too: without this, every new migration would create a table the +-- read-only role cannot see, and the omission would only surface as a confusing +-- "permission denied" months later. +ALTER DEFAULT PRIVILEGES FOR ROLE :"role" IN SCHEMA public + GRANT SELECT ON TABLES TO :"rorole"; +ALTER DEFAULT PRIVILEGES FOR ROLE :"role" IN SCHEMA public + GRANT SELECT ON SEQUENCES TO :"rorole"; +SQL + +echo "==> OK: ${TG_DB_USER} owns ${TG_DB_NAME} (timescaledb enabled, PUBLIC revoked)" +echo " Verify isolation:" +echo " docker exec $DB_CID psql -U ${TG_DB_USER} -d thermograph -c 'select 1' # must FAIL" diff --git a/infra/deploy/deploy-dev.sh b/infra/deploy/deploy-dev.sh index 8149d8b..bf09ac1 100755 --- a/infra/deploy/deploy-dev.sh +++ b/infra/deploy/deploy-dev.sh @@ -1,41 +1,54 @@ #!/usr/bin/env bash -# LAN-dev deploy: roll the per-service registry-pull stack (see deploy.sh) onto -# the ~/thermograph-dev overlay instead of prod/beta's loopback-only stack. +# Dev deploy: roll the per-service registry-pull stack (see deploy.sh) onto the +# dev compose overlay instead of beta/prod's Swarm stacks. # -# This script assumes $APP_DIR is a checkout of the monorepo (deploy assets under infra/), -# the same way /opt/thermograph is on prod/beta. That is the live state: the LAN -# box's ~/thermograph-dev was reprovisioned as an infra checkout during the -# 2026-07-22 cutover (the app monorepo is archived), provision-dev-lan.sh's -# REPO_URL defaults to this repo, and the app repos' deploy-dev.yml workflows -# invoke this script on the thermograph-lan runner. This IS the live LAN-dev -# deploy path. +# Dev lives on vps1 now, at /opt/thermograph-dev — not on the operator's desktop. +# The desktop hosts no Thermograph environment any more; it runs the AI models +# and offers flex Swarm capacity. What moved is only WHERE and HOW dev is +# reached: it is a normal fleet host now (SSH deploy from CI, SOPS render at +# /etc/thermograph.env, mesh-only exposure on 10.10.0.2:8137), rather than a +# sudo-free systemd --user stack on someone's Wi-Fi. +# +# $APP_DIR is a checkout of the monorepo (deploy assets under infra/), the same +# way /opt/thermograph is for prod. A laptop can still point APP_DIR at a +# checkout under $HOME and get the old local behaviour. # # Design: a thin wrapper around deploy.sh, not a duplicate. deploy.sh already owns # the entire per-service registry-pull mechanism -- secrets sourcing, docker login, # the retry-pull loop, --no-deps single-service rolls vs. --remove-orphans `all`, # .image-tags.env persistence, and the 8137/8080 health checks. None of that is -# dev-specific; the only things LAN dev actually changes are WHERE it deploys -# (a separate checkout + branch) and WHICH compose files are in play (the base -# file plus docker-compose.dev.yml's uncapped/LAN-published overrides). Both are -# expressible as environment (APP_DIR/BRANCH that deploy.sh already reads, and -# docker compose's own COMPOSE_FILE variable), so re-exec'ing deploy.sh with that -# environment set covers it with no forked copy of the pull/roll/health logic to -# drift out of sync. If LAN dev ever needs deploy logic that genuinely diverges -# from prod/beta (not just "different files/directory"), promote this to a -# standalone script at that point rather than growing special cases into deploy.sh. +# dev-specific; the only things dev actually changes are WHERE it deploys (a +# separate checkout + branch), WHICH compose files are in play (the base file +# plus docker-compose.dev.yml's uncapped/mesh-published overrides), and its +# secrets policy (below). All are expressible as environment (APP_DIR/BRANCH +# that deploy.sh already reads, and docker compose's own COMPOSE_FILE +# variable), so re-exec'ing deploy.sh with that environment set covers it with +# no forked copy of the pull/roll/health logic to drift out of sync. If dev ever +# needs deploy logic that genuinely diverges from beta/prod (not just "different +# files/directory"), promote this to a standalone script at that point rather +# than growing special cases into deploy.sh. # # Usage, mirrors deploy.sh directly: # # roll just the backend onto a dev-tagged image: -# ssh dev-box 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> deploy/deploy-dev.sh' +# ssh vps1 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph-dev/infra/deploy/deploy-dev.sh' # # bring the whole dev stack up (both tags required, same as deploy.sh): -# ssh dev-box 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- deploy/deploy-dev.sh' +# ssh vps1 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- /opt/thermograph-dev/infra/deploy/deploy-dev.sh' set -euo pipefail -# Dev context: a separate checkout + branch from prod/beta's /opt/thermograph -# (main), so a dev deploy never touches or is touched by the prod/beta one. -APP_DIR="${APP_DIR:-$HOME/thermograph-dev}" -BRANCH="${BRANCH:-dev}" +# Dev context: its own checkout and its own branch, on its own host (vps1). +# Both come from deploy/env-topology.sh so there is one place that says where +# dev lives. $APP_DIR is still honoured when set explicitly — that is how a +# laptop points this at a checkout under $HOME. +_dev_self="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=infra/deploy/env-topology.sh +. "$_dev_self/env-topology.sh" +thermograph_topology dev +APP_DIR="${APP_DIR:-$TG_APP_DIR}" +BRANCH="${BRANCH:-$TG_BRANCH}" export APP_DIR BRANCH +# deploy.sh resolves the environment itself; say so explicitly rather than +# letting it fall back to a host marker that, on vps1, names dev anyway. +export THERMOGRAPH_ENV=dev # Point every `docker compose` invocation inside deploy.sh at the LAN-dev overlay # (uncapped CPU, backend published on 0.0.0.0:8137, frontend unpublished -- see @@ -72,23 +85,33 @@ export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-thermograph-dev}" # genuinely needs, and nothing else. export THERMOGRAPH_SECRETS_SKIP_COMMON=1 -# (1b) Docker on this box is the snap package, confined to $HOME (plus a short -# allowlist) -- it silently can't see /etc/thermograph.env at all, so -# env_file: /etc/thermograph.env resolves to nothing for every container no -# matter how correctly it's rendered. Mirror the render to a path under -# $APP_DIR too; docker-compose.dev.yml points env_file at this copy instead of -# /etc/thermograph.env. Untracked (see infra/.gitignore); re-rendered on every -# deploy, never committed. -export THERMOGRAPH_SECRETS_ENV_FILE_MIRROR="$APP_DIR/infra/deploy/dev-secrets.env" +# (1b) Snap-packaged Docker is confined to $HOME (plus a short allowlist) and +# silently can't see /etc/thermograph.env at all, so env_file: /etc/thermograph.env +# resolves to nothing for every container no matter how correctly it's rendered. +# Where that is the case, mirror the render to a path under $APP_DIR; +# docker-compose.dev.yml carries a matching optional env_file entry. +# +# CONDITIONAL, not unconditional: this writes a plaintext copy of the render +# into the checkout, and on vps1 (ordinary apt Docker, which reads /etc fine) +# that copy would be pure liability. Detect the confinement rather than add a +# knob someone has to remember — the docker binary resolving under /snap is +# exactly the condition that breaks the /etc path. +_docker_bin="$(command -v docker || true)" +if [ -n "$_docker_bin" ] && case "$(readlink -f "$_docker_bin")" in /snap/*) true ;; *) false ;; esac; then + export THERMOGRAPH_SECRETS_ENV_FILE_MIRROR="$APP_DIR/infra/deploy/dev-secrets.env" +fi # (2) The age key is read from wherever it is READABLE, not necessarily # /etc/thermograph/age.key. render-secrets.sh falls back to `sudo cat` for a -# root-owned 0400 key, and this box has no passwordless sudo -- under the Forgejo -# runner (a systemd --user service, no tty) that sudo cannot prompt, so the -# fleet-standard placement would make every dev deploy fail at the render. Prefer -# the standard path when it is readable; otherwise the operator's own keyring, which -# is where dev's key already lives (this box is the machine `sops` is run on), so -# provisioning dev needs no second copy of the estate's recovery root. +# root-owned 0400 key, which needs passwordless sudo. +# +# On vps1 that is the normal case: the `agent` user has passwordless sudo and +# the key sits at the fleet-standard path, so this block does nothing. The +# fallback exists for the laptop case — a checkout where the key lives only in +# the operator's own keyring (that machine is where `sops` is run), and where a +# non-interactive shell could not prompt for sudo even if the key were in /etc. +# Preferring the standard path when readable keeps vps1 on the fleet convention +# and means provisioning dev needs no second copy of the estate's recovery root. if [ -z "${THERMOGRAPH_AGE_KEY:-}" ] && [ ! -r /etc/thermograph/age.key ] \ && [ -r "$HOME/.config/sops/age/keys.txt" ]; then export THERMOGRAPH_AGE_KEY="$HOME/.config/sops/age/keys.txt" diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index 51748eb..98fd9de 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -26,11 +26,59 @@ # two axes are independent and rarely change together. set -euo pipefail -APP_DIR="${APP_DIR:-/opt/thermograph}" +# --- which environment is this? ------------------------------------------------ +# vps2 runs beta AND prod, so "the host" no longer answers this — the caller does, +# via THERMOGRAPH_ENV (the deploy workflow always passes it). A by-hand run on a +# single-environment box still falls back to the host marker, and a box with +# neither still defaults to prod's historical paths, so nothing about an existing +# single-env host changes. +# +# Resolved from the SCRIPT'S OWN LOCATION, not a hardcoded /opt/thermograph: +# invoking /opt/thermograph-beta/infra/deploy/deploy.sh must act on the beta +# checkout even if something in the environment says otherwise. That is also the +# check that catches the one genuinely dangerous mistake on a two-environment +# host — running prod's checkout with THERMOGRAPH_ENV=beta, or the reverse. +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +SELF_APP_DIR=$(cd "$SELF_DIR/../.." && pwd) + +# Guarded exactly like render-secrets.sh below: the deploy that INTRODUCES this +# file runs with a checkout that predates it (it arrives with the git reset +# further down, after which deploy.sh re-execs). Missing => keep the pre-split +# behaviour rather than fail. +if [ -f "$SELF_DIR/env-topology.sh" ]; then + # shellcheck source=infra/deploy/env-topology.sh + . "$SELF_DIR/env-topology.sh" + ENV_NAME=$(thermograph_env_name) + # Nothing to go on anywhere: this is a pre-split host whose paths are prod's. + [ -n "$ENV_NAME" ] || ENV_NAME=prod + thermograph_topology "$ENV_NAME" + if [ "$SELF_APP_DIR" != "$TG_APP_DIR" ] && [ -z "${APP_DIR:-}" ]; then + echo "!! environment/checkout mismatch: THERMOGRAPH_ENV=$ENV_NAME expects" >&2 + echo "!! $TG_APP_DIR but this script lives in $SELF_APP_DIR." >&2 + echo "!! On vps2 that means beta and prod have been crossed. Refusing to deploy." >&2 + echo "!! If this checkout really is $ENV_NAME (a rehearsal copy, a relocated" >&2 + echo "!! checkout), say so explicitly: APP_DIR=$SELF_APP_DIR ..." >&2 + exit 2 + fi + APP_DIR="${APP_DIR:-$TG_APP_DIR}" + BRANCH="${BRANCH:-$TG_BRANCH}" + ENV_FILE="$TG_ENV_FILE" + DEPLOY_MODE="$TG_DEPLOY_MODE" + [ "$TG_SKIP_COMMON" = 1 ] && export THERMOGRAPH_SECRETS_SKIP_COMMON=1 + # The second pass (after the re-exec) must resolve the SAME environment, and + # the stack path needs it too. + export THERMOGRAPH_ENV="$ENV_NAME" +else + ENV_NAME="" + APP_DIR="${APP_DIR:-/opt/thermograph}" + BRANCH="${BRANCH:-main}" + ENV_FILE=/etc/thermograph.env + DEPLOY_MODE="" +fi + # Monorepo layout: git operations act on the checkout root ($APP_DIR); all # compose files and deploy assets live under infra/. INFRA_DIR="$APP_DIR/infra" -BRANCH="${BRANCH:-main}" # Which service this deploy rolls: backend | frontend | all. Defaults to `all` # (a full-stack bring-up) so a by-hand run with both tags still works; the # per-repo deploy.yml workflows always pass an explicit single service. @@ -69,16 +117,20 @@ fi # after which deploy.sh re-execs), so a missing helper simply falls back to the # existing /etc/thermograph.env. Then source it so a by-hand run interpolates the # same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/. +# +# $ENV_FILE, not a hardcoded /etc/thermograph.env: on vps2 prod renders +# prod.yaml there while beta renders beta.yaml to /etc/thermograph-beta.env. +# One file per environment, never shared. if [ -f "$INFRA_DIR/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$INFRA_DIR/deploy/render-secrets.sh" - render_thermograph_secrets "$INFRA_DIR" + render_thermograph_secrets "$INFRA_DIR" "$ENV_NAME" "$ENV_FILE" fi -# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it -# cannot exist at lint time, so don't ask shellcheck to follow it. +# $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at +# lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null -. /etc/thermograph.env 2>/dev/null || true +. "$ENV_FILE" 2>/dev/null || true set +a # Pre-warm the ~750 city-page archives so /climate pages serve from cache and a @@ -122,12 +174,18 @@ if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then exec "$0" "$@" fi -# Stack-mode routing: a host whose /etc/thermograph/deploy-mode says "stack" -# (prod, after the Swarm cutover) deploys via the Swarm stack path instead of -# compose. Checked AFTER the reset+re-exec so the stack script is always the -# freshly-pulled one, and the SERVICE/tag contract passes through unchanged -- -# the app repos' workflows never need to know which mode a host runs. -if [ "$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)" = "stack" ]; then +# Stack-mode routing: prod and beta are both Swarm stacks, dev is compose. The +# mode is a property of the ENVIRONMENT (env-topology.sh), not of the host -- +# vps2 runs two stacks, and a host-wide /etc/thermograph/deploy-mode marker +# cannot describe a host that runs more than one environment. The marker is +# still honoured when the topology file is absent (a checkout that predates it). +# Checked AFTER the reset+re-exec so the stack script is always the freshly +# pulled one, and the SERVICE/tag contract passes through unchanged -- the +# deploy workflow never needs to know which mode an environment runs. +if [ -z "$DEPLOY_MODE" ]; then + DEPLOY_MODE=$(cat /etc/thermograph/deploy-mode 2>/dev/null || true) +fi +if [ "$DEPLOY_MODE" = "stack" ]; then exec bash "$INFRA_DIR/deploy/stack/deploy-stack.sh" fi @@ -137,6 +195,20 @@ fi # rename the project and recreate the stack under a second name. cd "$INFRA_DIR" +# Compose-mode environments (dev) carry their project name, file list and bind +# address in the topology table. Set only when not already in the environment, +# so deploy-dev.sh's own exports and a by-hand override both still win. +if [ -n "${TG_COMPOSE_PROJECT:-}" ]; then + export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-$TG_COMPOSE_PROJECT}" + export COMPOSE_FILE="${COMPOSE_FILE:-$TG_COMPOSE_FILE}" +fi +# Which address the dev overlay publishes on. Defaults to loopback in the +# compose file; dev on vps1 sets the mesh address here, because a public VPS +# must never publish an unreviewed branch's stack on 0.0.0.0. +if [ -n "${TG_BIND_ADDR:-}" ]; then + export DEV_BIND_ADDR="${DEV_BIND_ADDR:-$TG_BIND_ADDR}" +fi + # Registry-pull cutover: pull the image each app repo's build-push.yml already # built and pushed, instead of building in place. This checkout is # thermograph-infra, not an app repo, so there's no "current commit" to derive diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh new file mode 100644 index 0000000..77b6c82 --- /dev/null +++ b/infra/deploy/env-topology.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# WHERE EACH ENVIRONMENT LIVES — the single source of truth, sourced by every +# deploy path. Not executable on its own; `. env-topology.sh` then call +# `thermograph_topology `. +# +# Why this file exists at all. Until the vps1/vps2 split, "which environment is +# this?" was answerable from the machine you were standing on: one host ran one +# environment, so a host-wide marker (/etc/thermograph/secrets-env) and a +# host-wide deploy mode (/etc/thermograph/deploy-mode) were enough, and every +# path could hardcode /opt/thermograph, /etc/thermograph.env, ports 8137/8080. +# vps2 now runs BOTH beta and prod. Every one of those assumptions becomes a +# collision: two checkouts, two rendered env files, two stacks, two loopback +# port pairs, two image-tag files, two deploy locks — on one box. So the +# environment is now an INPUT (THERMOGRAPH_ENV, passed by the deploy caller), +# and everything else is derived here rather than re-guessed per script. +# +# The host markers survive as the fallback for a by-hand run with no explicit +# env (`deploy.sh` on prod still means prod), but they are no longer the thing +# that decides where files go. +# +# THE ESTATE +# +# vps1 75.119.132.91 10.10.0.2 "operational programs" +# Forgejo (git + CI + registry), Grafana/Loki/Alloy, emigriffith.dev, +# and the DEV environment with its own Postgres. Nothing here is +# reachable by an external user except git/dashboard/the portfolio site; +# dev itself is mesh-only, deliberately (see DEV_BIND_ADDR below). +# +# vps2 169.58.46.181 10.10.0.1 "the deployed environment" +# Everything an external user can touch: prod (thermograph.org) AND beta +# (beta.thermograph.org), Centralis, Postfix, the backups, and the ONE +# TimescaleDB instance that serves both environments on separate +# databases with separate roles. +# +# desktop 10.10.0.3 operator's box +# AI model hosting (voice-to-text, the LLM behind upcoming features) and +# flex capacity as a Swarm worker. It hosts NO Thermograph environment — +# that is the whole point of the vps1/vps2 split. A laptop-style +# `make dev-up` still works locally; it is just not the dev server. +# +# WHY BETA SITS NEXT TO PROD RATHER THAN NEXT TO DEV +# +# Beta is a pre-production rehearsal, so what it needs to rehearse is prod's +# environment: the same orchestrator (Swarm, not compose), the same Postgres +# major and extension build, the same Caddy in front, the same mail path, the +# same mesh position. Co-locating beta with dev bought resemblance to the thing +# it is NOT trying to predict. Co-locating it with prod means a beta green light +# is evidence about prod. The cost — a bad beta deploy shares a machine with +# prod — is bounded by the per-environment isolation this file defines: separate +# stacks, separate volumes, separate DB roles, separate CPU limits. + +# thermograph_topology +# +# Exports the TG_* variables describing that environment. Every value is a +# derived fact about the estate, not a preference: change one here and the +# deploy scripts, the stack files and the runbooks all follow. +thermograph_topology() { + local env_name="${1:?thermograph_topology needs an environment: dev|beta|prod}" + + # Reset first: this function is sourced into long-lived shells (deploy.sh + # sources it before and after its self re-exec), and a stale TG_STACK_NAME + # from a previous call would silently target the wrong stack. + TG_ENV=""; TG_HOST=""; TG_APP_DIR=""; TG_BRANCH=""; TG_DEPLOY_MODE="" + TG_STACK_NAME=""; TG_STACK_FILE=""; TG_LB_CONFIG="" + TG_COMPOSE_PROJECT=""; TG_COMPOSE_FILE="" + TG_ENV_FILE=""; TG_STACK_ENV_FILE=""; TG_LB_NAME="" + TG_LB_HTTP_PORT=""; TG_LB_FE_PORT=""; TG_DB_NAME=""; TG_DB_USER="" + TG_TAGS_FILE=""; TG_LOCK_FILE=""; TG_BIND_ADDR=""; TG_SKIP_COMMON=0 + TG_SVC_PREFIX=""; TG_DATA_NETWORK=""; TG_DB_SERVICE=""; TG_POST_DEPLOY=0 + TG_SSH_HOST=""; TG_SSH_TARGET="" + + case "$env_name" in + prod) + TG_ENV=prod + TG_HOST=vps2 + # Unchanged from before the split — prod keeps every path it already has, + # so nothing about the live prod host moves during the cutover. + TG_SSH_HOST=169.58.46.181 + TG_SSH_TARGET=agent@169.58.46.181 + TG_APP_DIR=/opt/thermograph + TG_BRANCH=main + TG_DEPLOY_MODE=stack + TG_STACK_NAME=thermograph + TG_STACK_FILE=deploy/stack/thermograph-stack.yml + TG_ENV_FILE=/etc/thermograph.env + TG_STACK_ENV_FILE=/etc/thermograph/stack.env + TG_LB_NAME=thermograph-lb + TG_LB_CONFIG=deploy/stack/lb/Caddyfile + TG_LB_HTTP_PORT=8137 + TG_LB_FE_PORT=8080 + TG_DB_NAME=thermograph + TG_DB_USER=thermograph + # Prod's services keep their bare names (web, worker, frontend, ...): the + # cutover must not touch prod's stack file, its env or its LB config. + TG_SVC_PREFIX="" + # Where the database lives. Prod's own overlay, which is `attachable` and + # which beta joins as an external network. + TG_DATA_NETWORK=thermograph_internal + TG_DB_SERVICE=thermograph_db + # Post-deploy city warm + IndexNow ping. Prod only — see the beta note. + TG_POST_DEPLOY=1 + ;; + beta) + TG_ENV=beta + TG_HOST=vps2 + # Same box, same public IP as prod — beta and prod are two stacks on one + # Swarm manager now, not two separate hosts. A beta SSH target is + # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh). + TG_SSH_HOST=169.58.46.181 + TG_SSH_TARGET=agent@169.58.46.181 + # A SECOND checkout on the same box. Separate from prod's so the two can + # sit on different commits of this repo, and so `git reset --hard` in one + # deploy can never yank the tree out from under the other's running + # deploy. Everything below is likewise a distinct name/port/path from + # prod's, on purpose: co-location is only safe if nothing is shared by + # accident. The one deliberate exception is the database SERVER (see + # TG_DB_* — separate role and database on a shared instance). + TG_APP_DIR=/opt/thermograph-beta + TG_BRANCH=main + TG_DEPLOY_MODE=stack + TG_STACK_NAME=thermograph-beta + TG_STACK_FILE=deploy/stack/thermograph-beta-stack.yml + TG_ENV_FILE=/etc/thermograph-beta.env + TG_STACK_ENV_FILE=/etc/thermograph/beta-stack.env + TG_LB_NAME=thermograph-beta-lb + TG_LB_CONFIG=deploy/stack/lb/Caddyfile.beta + # NOT 8137/8080: prod's loopback LB already owns those on this host. + TG_LB_HTTP_PORT=8237 + TG_LB_FE_PORT=8180 + TG_DB_NAME=thermograph_beta + # Its own role, not prod's `thermograph` superuser-of-its-own-database. + # One instance is a capacity decision; it is not a decision to let a beta + # deploy running an unmerged branch read or write the production database. + TG_DB_USER=thermograph_beta + # Beta's Swarm services are named beta-web, beta-worker, ... Swarm + # registers a service's SHORT name as a DNS alias on every network it + # joins, so two stacks that both call a service `web` on one shared + # network make `web` ambiguous — prod's frontend could resolve a beta + # task. Prefixing beta's names removes the collision without editing a + # single line of prod's stack. + TG_SVC_PREFIX="beta-" + # Beta has no db service of its own: it joins prod's overlay (declared + # `external` in its stack file) purely to reach `db`, and keeps its own + # `internal` overlay for beta-to-beta traffic. + TG_DATA_NETWORK=thermograph_internal + TG_DB_SERVICE=thermograph_db + # No warm, no IndexNow on beta. IndexNow announces URLs to Bing/DDG/ + # Yandex — from beta that means asking search engines to index + # beta.thermograph.org, which is the opposite of what beta is for. The + # city warm is worse than useless here: it spends the shared upstream + # archive quota to fill a cache that only a rehearsal environment reads. + # (Both ran on beta under compose; that was a side effect of beta and + # prod sharing one script, not a decision.) + TG_POST_DEPLOY=0 + ;; + dev) + TG_ENV=dev + TG_HOST=vps1 + # vps1 — Forgejo/Grafana/dev, mesh-only for dev itself (see TG_BIND_ADDR). + TG_SSH_HOST=75.119.132.91 + TG_SSH_TARGET=agent@75.119.132.91 + TG_APP_DIR=/opt/thermograph-dev + # The only environment that tracks `dev`; beta and prod both run this + # repo's `main` (infra is not environment-staged — app code is, via image + # tags). See the branch model in infra/README.md. + TG_BRANCH=dev + # Compose, not Swarm: dev's value is a fast, legible, single-box stack + # you can `docker compose logs` at. It is the one environment not + # pretending to be prod. + TG_DEPLOY_MODE=compose + TG_COMPOSE_PROJECT=thermograph-dev + TG_COMPOSE_FILE="docker-compose.yml:docker-compose.dev.yml" + TG_ENV_FILE=/etc/thermograph.env + # Dev keeps its OWN Postgres container (a `db` service in its compose + # project). It is not on the shared instance and must not be: the shared + # instance lives on vps2 and holds real user data. + TG_DB_NAME=thermograph + TG_DB_USER=thermograph + # Mesh-only. vps1 is a public VPS, so the dev overlay must not publish on + # 0.0.0.0 the way it did on the operator's LAN box — dev runs whatever + # branch is in flight, including unreviewed ones. Reachable at + # http://10.10.0.2:8137 from anything on the WireGuard mesh, and from + # nowhere else. There is deliberately no dev DNS record and no Caddy site. + TG_BIND_ADDR=10.10.0.2 + # Dev NEVER layers common.yaml — that file is the internet-facing fleet's + # shared production credential set (both S3 keypairs, the VAPID signing + # key, REGISTRY_TOKEN, the metrics token), and dev runs whatever branch is + # in flight on the same box as Forgejo and its CI runner. This lives here + # rather than only in deploy-dev.sh so the protection is a property of the + # ENVIRONMENT, not of which entry point happened to be used. See the long + # note in render-secrets.sh. + TG_SKIP_COMMON=1 + TG_DB_SERVICE=db + # Dev warms nothing and pings nothing: it must never spend the upstream + # archive quota, and it must never tell a search engine it exists. + TG_POST_DEPLOY=0 + ;; + *) + echo "!! unknown environment '$env_name' (expected dev|beta|prod)" >&2 + return 2 + ;; + esac + + # Derived, never hand-set: keeping these next to the definitions above is what + # stops a second environment on one host from sharing a lock or a tag file. + TG_LOCK_FILE="$TG_APP_DIR/infra/deploy/.deploy.lock" + if [ "$TG_DEPLOY_MODE" = stack ]; then + TG_TAGS_FILE="$TG_APP_DIR/infra/deploy/.stack-image-tags.env" + else + TG_TAGS_FILE="$TG_APP_DIR/infra/deploy/.image-tags.env" + fi + + export TG_ENV TG_HOST TG_APP_DIR TG_BRANCH TG_DEPLOY_MODE + export TG_STACK_NAME TG_STACK_FILE TG_LB_CONFIG TG_COMPOSE_PROJECT TG_COMPOSE_FILE + export TG_ENV_FILE TG_STACK_ENV_FILE TG_LB_NAME + export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER + export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON + export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY + export TG_SSH_HOST TG_SSH_TARGET +} + +# thermograph_env_name +# +# Which environment is this run for? Explicit input first (the deploy workflows +# pass THERMOGRAPH_ENV, and on vps2 it is the ONLY thing distinguishing a beta +# deploy from a prod one), then the host marker for a by-hand run on a +# single-environment box. Empty output means "cannot tell" and the caller must +# fail rather than guess — guessing wrong on vps2 means deploying beta's image +# tag onto prod. +thermograph_env_name() { + local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" + if [ -n "${THERMOGRAPH_ENV:-}" ]; then + printf '%s\n' "$THERMOGRAPH_ENV" + return 0 + fi + cat "$marker" 2>/dev/null || true +} diff --git a/infra/deploy/forgejo/README.md b/infra/deploy/forgejo/README.md index ab7315d..f15711f 100644 --- a/infra/deploy/forgejo/README.md +++ b/infra/deploy/forgejo/README.md @@ -1,22 +1,24 @@ # Forgejo on the Swarm cluster Runs as `deploy/forgejo/docker-stack.yml` — the only Swarm-scheduled workload -this cluster carries (the Thermograph app itself stays on the -Terraform-managed `docker compose` deploys; see `terraform/README.md`). -Pinned to the **beta** node (old VPS) via the `role=forge` label from +this cluster carries (prod and beta's app stacks are separate `docker stack +deploy`s that happen to run on the manager node, vps2 — see +`deploy/stack/README` context in `DEPLOY.md`). Pinned to the **vps1** node +(`75.119.132.91`) via the `role=forge` label from `deploy/swarm/label-forge-node.sh`. -The Actions **runner** is deliberately *not* part of this stack — it runs on -the **desktop** as a plain systemd service (`register-lan-runner.sh` below), -per `thermograph-docs/runbooks/implementation-handoff.md` Track B step 5. That's the -canonical placement; an earlier revision of this stack ran the runner as a -Swarm-scheduled Docker-in-Docker sidecar pinned to beta, which is gone now. +The Actions **runner** is deliberately *not* part of this stack — see +`DEPLOY-DEV.md` and the note in `register-lan-runner.sh`'s own header for +where it runs today. That's the canonical placement per +`thermograph-docs/runbooks/implementation-handoff.md` Track B step 5; an +earlier revision of this stack ran the runner as a Swarm-scheduled +Docker-in-Docker sidecar pinned to the Forgejo node, which is gone now. ## Prerequisites -1. All three nodes have joined the swarm (`deploy/swarm/`) and beta is +1. All three nodes have joined the swarm (`deploy/swarm/`) and vps1 is labeled `role=forge`. -2. `docker node ls` (from the manager) shows all three `Ready`. +2. `docker node ls` (from the manager, vps2) shows all three `Ready`. ## One-time setup: Swarm secret @@ -43,27 +45,27 @@ its first health check just fails harmlessly until it is. This stack has **no auto-deploy trigger** — nothing in `.forgejo/workflows/` redeploys it on push. A change to `docker-stack.yml` only takes effect once -someone re-runs `docker stack deploy` by hand on the manager (prod). +someone re-runs `docker stack deploy` by hand on the manager (vps2). `db`/`forgejo` both carry `resources.limits` (defaults: db 1 CPU/1g, forgejo 2 CPU/2g — several times observed steady-state usage), overridable with `FORGEJO_DB_CPUS`/`FORGEJO_DB_MEMORY`/`FORGEJO_CPUS`/`FORGEJO_MEMORY` env vars before `docker stack deploy`, same convention as the app stack. -## DNS + TLS: reusing beta's existing Caddy, not a second reverse proxy +## DNS + TLS: reusing vps1's existing Caddy, not a second reverse proxy -Forgejo is pinned to beta (`role=forge`) — but beta is also **today's live -thermograph.org host**, and its Caddy already owns ports 80/443 -(`/etc/caddy/Caddyfile` on that box). A second ingress (Traefik) trying to -bind the same ports would collide with it. So there's no Traefik in this -stack: `forgejo`'s web port publishes to `127.0.0.1:3080` only (host-local), -and beta's *existing* Caddy gets one more site block reverse-proxying to it — -same pattern as its `thermograph.org` block, same automatic-HTTPS. +Forgejo is pinned to vps1 (`role=forge`) — the same box that also runs +Grafana/Loki/Alloy and the `emigriffith.dev` portfolio site, each fronted by +that host's own Caddy. A second ingress (Traefik) trying to bind the same +ports would collide with it. So there's no Traefik in this stack: `forgejo`'s +web port publishes to `127.0.0.1:3080` only (host-local), and vps1's +*existing* Caddy gets one more site block reverse-proxying to it — same +pattern as its other site blocks, same automatic-HTTPS. 1. Point the Forgejo domain (default `git.thermograph.org`; override with - `FORGEJO_DOMAIN=...` before `docker stack deploy`) at **beta's** public IP - — that's where the task actually runs, not prod's or the desktop's. -2. Append `deploy/forgejo/caddy-git.conf` to beta's `/etc/caddy/Caddyfile`, + `FORGEJO_DOMAIN=...` before `docker stack deploy`) at **vps1's** public IP + — that's where the task actually runs, not vps2's or the desktop's. +2. Append `deploy/forgejo/caddy-git.conf` to vps1's `/etc/caddy/Caddyfile`, adjusting the domain if you didn't use the default, then `systemctl reload caddy`. 3. That file also resolves the registry-exposure hazard (#15 in @@ -75,28 +77,28 @@ same pattern as its `thermograph.org` block, same automatic-HTTPS. ## Registry access from mesh clients -Any node that needs `docker login`/push/pull against the registry (the -desktop's CI runner building/pushing images, later any Swarm node pulling -them) must reach `git.thermograph.org` **over the WireGuard tunnel**, not -beta's public IP — otherwise Caddy's `/v2/*` block above refuses the -connection. Public DNS resolves the domain to beta's public IP, so add a -`/etc/hosts` override on each such node pinning it to beta's WireGuard -address instead: +Any node that needs `docker login`/push/pull against the registry (the CI +runner building/pushing images, any Swarm node pulling them, prod or beta on +vps2 pulling app images) must reach `git.thermograph.org` **over the +WireGuard tunnel**, not vps1's public IP — otherwise Caddy's `/v2/*` block +above refuses the connection. Public DNS resolves the domain to vps1's public +IP, so add a `/etc/hosts` override on each such node pinning it to vps1's +WireGuard address instead: ``` echo "10.10.0.2 git.thermograph.org" | sudo tee -a /etc/hosts ``` -(`10.10.0.2` is beta's WG address per `deploy/swarm/README.md`'s peer +(`10.10.0.2` is vps1's WG address per `deploy/swarm/README.md`'s peer numbering — adjust if you assigned it differently.) The git/web UI keeps working normally for everyone else since only `/v2/*` is restricted. -## Register the Actions runner (on the desktop, not through Swarm) +## Register the Actions runner Once Forgejo answers at its domain: ```bash -# On the desktop: +# On the runner host (see DEPLOY-DEV.md for where that is today): # Forgejo web UI -> Site Administration -> Actions -> Runners -> Create new Runner # (or, for a repo-scoped runner: -> Settings -> Actions -> Runners) # copy the registration token, then: @@ -104,8 +106,8 @@ bash deploy/forgejo/register-lan-runner.sh https:// ``` See that script's header for exactly what it replaces (the pre-Forgejo GitHub -self-hosted runner on this same machine) and why it registers with two -labels where there used to be two separate runners. +self-hosted runner) and why it registers with two labels where there used to +be two separate runners. `config.yaml`'s `runner.capacity` is raised from the tool's default of 1 to 8 (override with `CAPACITY=`) — a single PR push fires `pr-build`, @@ -113,18 +115,16 @@ labels where there used to be two separate runners. no `needs:` between them), so capacity 1 serializes work that could run in parallel, and even capacity 3 (an earlier, undocumented hand-tune) leaves `build-backend`/`build-frontend`/`validate-observability` queued behind -those three before they get a slot. The desktop has 16 cores / 34GB free -today; 8 concurrent jobs is comfortable headroom without starving LAN dev's -own compose stack. +those three before they get a slot. **Adding more runner capacity should mean raising this number, or adding a -second runner on the desktop itself — not putting a runner on prod or -beta.** `container.docker_host: automount` gives job containers the *host's* -Docker socket; on prod or beta that would mean any CI job has root-equivalent -access to whatever else is running there (the live app stack, or Forgejo -itself). An earlier revision of this stack actually did run the runner as a -Swarm-hosted container on beta and was deliberately reverted to the desktop -for this reason — see the note at the top of `docker-stack.yml`. +second runner alongside it — not putting a runner on prod or beta (vps2).** +`container.docker_host: automount` gives job containers the *host's* Docker +socket; on vps2 that would mean any CI job has root-equivalent access to both +the prod and beta stacks running there. An earlier revision of this stack +actually ran the runner as a Swarm-hosted container on the box Forgejo was +pinned to and was deliberately reverted for this same class of reason — see +the note at the top of `docker-stack.yml`. ## Custom CI job image (`ci-runner/`) @@ -155,8 +155,8 @@ docker push git.thermograph.org/emi/thermograph/ci-runner:vN `register-lan-runner.sh`'s `LABELS` default points at the current tag, so fresh registrations pick it up automatically. The live runner is cut over by -editing the `labels` array in `~/forgejo-runner/.runner` on the desktop (same -runner id/token, no re-registration needed) and restarting the service — +editing the `labels` array in `~/forgejo-runner/.runner` on the runner host +(same runner id/token, no re-registration needed) and restarting the service — **verify a real job runs green under the new image before relying on it**, same way v1's break was caught. Only after that verification should the now-redundant `apt-get install docker.io` / `python3-yaml` steps be removed @@ -165,7 +165,7 @@ still running on the stock `node:20-bookworm` image. ## Why Postgres here and not the Thermograph app's TimescaleDB -Separate instance, separate network (`forgejo_net`, not the app's compose +Separate instance, separate network (`forgejo_net`, not the app's overlay network), separate volume. Forgejo is a distinct product with its own schema and its own backup/restore lifecycle — sharing a database with the app would couple two things that should be able to fail, migrate, and restore @@ -177,7 +177,7 @@ independently. docker service ls # forgejo_db, forgejo_forgejo both Running, 1/1 curl -I https://git.thermograph.org/ # 200, valid cert (Caddy's, not a new one) curl -I https://git.thermograph.org/v2/ # 403 from anywhere off the WireGuard mesh -# On the desktop, after registering the runner: +# On the runner host, after registering the runner: systemctl --user status forgejo-runner # active, both labels registered ``` diff --git a/infra/deploy/forgejo/caddy-git.conf b/infra/deploy/forgejo/caddy-git.conf index 8118682..55cb9c7 100644 --- a/infra/deploy/forgejo/caddy-git.conf +++ b/infra/deploy/forgejo/caddy-git.conf @@ -3,10 +3,11 @@ # ends up as a confusing orphaned comment in production config with no # surrounding context (docker-stack.yml isn't visible from there). # -# Target: /etc/caddy/Caddyfile on beta (the box `forgejo` is pinned to — see -# docker-stack.yml). Reuses beta's existing Caddy instead of a second reverse -# proxy/ACME flow: Forgejo publishes its web UI to 127.0.0.1:3080 only -# (host-local), and this block is the only thing that ever talks to it. +# Target: /etc/caddy/Caddyfile on vps1 (the box `forgejo` is pinned to — see +# docker-stack.yml; this is the box also called "vps1", 75.119.132.91). +# Reuses vps1's existing Caddy instead of a second reverse proxy/ACME flow: +# Forgejo publishes its web UI to 127.0.0.1:3080 only (host-local), and this +# block is the only thing that ever talks to it. # # Registry exposure (hazard #15, see docker-stack.yml's header comment): # git.thermograph.org/v2/* is the built-in OCI registry API. It's blocked @@ -15,7 +16,7 @@ # else (web UI, git-over-HTTP, PR pages) stays public like the rest of the # site. # -# Update the domain below to match whatever you actually pointed at beta's +# Update the domain below to match whatever you actually pointed at vps1's # IP, if not git.thermograph.org. # --- copy from here down into /etc/caddy/Caddyfile --- diff --git a/infra/deploy/forgejo/docker-stack.yml b/infra/deploy/forgejo/docker-stack.yml index 9891c3b..a86d41b 100644 --- a/infra/deploy/forgejo/docker-stack.yml +++ b/infra/deploy/forgejo/docker-stack.yml @@ -9,18 +9,18 @@ # (deploy/forgejo/register-lan-runner.sh) — the same place the pre-Forgejo # GitHub self-hosted runner already lived, not a Swarm-scheduled container. # -# No Traefik here. Forgejo is pinned to beta (node.labels.role == forge) -# because that's what was chosen, but beta is ALSO today's live thermograph.org +# No Traefik here. Forgejo is pinned to vps1 (node.labels.role == forge) +# because that's what was chosen, and vps1 is ALSO the emigriffith.dev portfolio # host — Caddy already owns its ports 80/443 (see /etc/caddy/Caddyfile on that # box). A second reverse proxy binding those same ports would either fail to # start or fight Caddy. Instead: forgejo's web port publishes to -# 127.0.0.1:3080 only (host-local, mode: host), and beta's existing Caddy gets +# 127.0.0.1:3080 only (host-local, mode: host), and vps1's existing Caddy gets # a new site block reverse-proxying git.thermograph.org -> 127.0.0.1:3080, -# same pattern as its thermograph.org block. TLS is Caddy's existing -# automatic-HTTPS (HTTP-01), not a second ACME flow. +# same pattern as its other blocks. TLS is Caddy's existing automatic-HTTPS +# (HTTP-01), not a second ACME flow. # -# Deploy from the manager node (prod), after all three nodes (prod, beta, -# desktop) have joined the swarm and beta is labeled role=forge: +# Deploy from the manager node (vps2, which runs prod), after all three nodes +# (vps1, vps2, desktop) have joined the swarm and vps1 is labeled role=forge: # # docker stack deploy -c deploy/forgejo/docker-stack.yml forgejo # @@ -32,7 +32,7 @@ # firewall the /v2/ registry API paths so only WireGuard-mesh clients can # reach them. That's implemented in the Caddy site block (see # deploy/forgejo/README.md), not here — nothing in this stack file is -# registry-specific, the restriction lives entirely in Caddy's config on beta. +# registry-specific, the restriction lives entirely in Caddy's config on vps1. services: db: @@ -96,9 +96,10 @@ services: FORGEJO__service__ENABLE_PASSWORD_SIGNIN_FORM: "false" FORGEJO__service__REGISTER_MANUAL_CONFIRM: "true" FORGEJO__service__DEFAULT_USER_IS_RESTRICTED: "true" - # Outbound mail via prod's Postfix null client over the WireGuard mesh - # (10.10.0.1:25 — mynetworks permits beta); self-signed cert on :25, so - # trust it. Send-only; used for admin/approval and notification mail. + # Outbound mail via vps2's Postfix null client (prod's box) over the + # WireGuard mesh (10.10.0.1:25 — mynetworks permits vps1, 10.10.0.2, + # where this Forgejo runs); self-signed cert on :25, so trust it. + # Send-only; used for admin/approval and notification mail. FORGEJO__mailer__ENABLED: "true" FORGEJO__mailer__PROTOCOL: "smtp" FORGEJO__mailer__SMTP_ADDR: "10.10.0.1" @@ -117,17 +118,17 @@ services: networks: [forgejo_net] ports: # SSH for git@ clones — published on whichever node the task lands on - # (pinned to beta by the placement constraint below, so effectively - # always beta's public IP, port 2222). + # (pinned to vps1 by the placement constraint below, so effectively + # always vps1's public IP, port 2222). - target: 2222 published: 2222 protocol: tcp mode: host # Web UI/API. Swarm's port schema has no host_ip scoping, so this binds - # 0.0.0.0:3080 on beta — NOT actually localhost-only by itself. A + # 0.0.0.0:3080 on vps1 — NOT actually localhost-only by itself. A # DOCKER-USER iptables rule (see deploy/forgejo/README.md) is what # actually restricts it, since Docker's own iptables rules bypass ufw - # for published ports. beta's Caddy reverse-proxies to 127.0.0.1:3080. + # for published ports. vps1's Caddy reverse-proxies to 127.0.0.1:3080. - target: 3000 published: 3080 protocol: tcp diff --git a/infra/deploy/forgejo/register-lan-runner.sh b/infra/deploy/forgejo/register-lan-runner.sh index 1296a1c..c204798 100755 --- a/infra/deploy/forgejo/register-lan-runner.sh +++ b/infra/deploy/forgejo/register-lan-runner.sh @@ -4,17 +4,26 @@ # runner (see DEPLOY-DEV.md) — this replaces that runner, it doesn't add a # second one. Sudo-free, systemd --user, same pattern as the app service. # -# This is THE runner (docs/runbooks/implementation-handoff.md Track B step 5 -# — canonical placement is the desktop, not a Swarm-hosted container), so it -# registers with BOTH labels the workflows need, where one runner used to be -# two: general CI/build/deploy.yml jobs (`docker`, containerized via this -# machine's own already-installed Docker — no Docker-in-Docker sidecar needed, -# unlike the Swarm-hosted design this replaces, since a real host with a real -# Docker install needs no such indirection) and the LAN-deploy job -# (`thermograph-lan`, bare/host-native — it writes to ~/thermograph-dev and -# restarts a systemd --user service, which only works running directly on the +# It registers with BOTH labels the workflows historically needed: general +# CI/build/deploy jobs (`docker`, containerized via this machine's own +# already-installed Docker — no Docker-in-Docker sidecar needed, since a real +# host with a real Docker install needs no such indirection) and the LAN-deploy +# job (`thermograph-lan`, bare/host-native — it wrote to ~/thermograph-dev and +# restarted a systemd --user service, which only worked running directly on the # host, not inside a container). # +# `thermograph-lan` IS NOW OBSOLETE. Dev moved off this machine to vps1 and is +# deployed over SSH by the same `Deploy` workflow that ships beta and prod; +# there is no host-native LAN deploy job left for that label to serve. Nothing +# breaks by keeping it registered — no workflow requests it — but do not build +# anything new on it. +# +# TODO(cutover): whether this machine keeps serving the `docker` label at all is +# an open call. It is no longer the only runner (the Swarm-hosted one is +# always-on), and the box's new job is AI model hosting plus flex Swarm-worker +# capacity. Keeping it is fine — it is real CI capacity — but the estate no +# longer depends on it, so decide deliberately rather than by inertia. +# # bash deploy/forgejo/register-lan-runner.sh # # Get from the Forgejo web UI: diff --git a/infra/deploy/openmeteo/README.md b/infra/deploy/openmeteo/README.md index fc455dd..238358f 100644 --- a/infra/deploy/openmeteo/README.md +++ b/infra/deploy/openmeteo/README.md @@ -184,6 +184,10 @@ credit; keep it in place. ## 9. Not on dev/beta -This overlay runs only on the self-hosting host (prod). Dev and beta leave -`THERMOGRAPH_ARCHIVE_URL` unset and use the public Open-Meteo archive API — do -not bring the overlay up there. +This overlay runs only for prod specifically — not for every environment on +prod's host. Beta now shares vps2 with prod, but that doesn't extend the +self-hosted archive to it: beta is its own Swarm stack +(`thermograph-beta-stack.yml`), which never sets `THERMOGRAPH_ARCHIVE_URL`, so +it reaches the public Open-Meteo archive API like dev does. Dev leaves it +unset for the same reason on vps1. Do not bring this overlay up for beta or +dev, and don't assume co-residency with prod on vps2 changes that. diff --git a/infra/deploy/provision-dev-lan.sh b/infra/deploy/provision-dev-lan.sh deleted file mode 100755 index 2daaf21..0000000 --- a/infra/deploy/provision-dev-lan.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# One-time bootstrap for the Thermograph LAN dev server on THIS machine. -# -# Sudo-free: the app runs as a Docker Compose stack (see deploy-dev.sh), and -# `linger` keeps the runner/services running across logout/reboot. Re-runnable -# (idempotent). -# -# bash deploy/provision-dev-lan.sh -set -euo pipefail - -APP_DIR="${APP_DIR:-$HOME/thermograph-dev}" -REPO_URL="${REPO_URL:-http://10.10.0.2:3080/emi/thermograph-infra.git}" -BRANCH="${BRANCH:-dev}" -here="$(cd "$(dirname "$0")" && pwd)" - -echo "==> Enabling linger so the service survives logout/reboot" -loginctl enable-linger "$USER" \ - || echo " (couldn't enable linger; the service still runs while you're logged in)" - -echo "==> Cloning/refreshing $APP_DIR and starting the service" -APP_DIR="$APP_DIR" REPO_URL="$REPO_URL" BRANCH="$BRANCH" bash "$here/deploy-dev.sh" - -cat <&2 + exit 1 +fi + +echo "==> Checkout: $APP_DIR on $BRANCH" +if [ -d "$APP_DIR/.git" ]; then + git -C "$APP_DIR" remote set-url origin "$REPO_URL" + git -C "$APP_DIR" fetch --prune origin "$BRANCH" + git -C "$APP_DIR" checkout -B "$BRANCH" "origin/$BRANCH" +else + git clone --branch "$BRANCH" "$REPO_URL" "$APP_DIR" +fi + +echo "==> Marking this checkout's environment" +mkdir -p /etc/thermograph +# The host marker still exists for by-hand runs. vps1 runs exactly one +# environment, so a marker is sufficient here — unlike vps2, where beta and prod +# share a box and THERMOGRAPH_ENV must be passed explicitly. +printf 'dev\n' > /etc/thermograph/secrets-env +# Dev is compose, not Swarm. env-topology.sh is what actually decides this; the +# marker is only the fallback for a checkout that predates it. +rm -f /etc/thermograph/deploy-mode + +if [ ! -f /etc/thermograph/age.key ]; then + cat >&2 <<'EOF' +!! No age key at /etc/thermograph/age.key. +!! Dev cannot render its vault without it, and deploy-dev.sh will fall back to +!! the un-vaulted defaults. Copy the key over (0400 root:root), then re-run: +!! install -m 0400 -o root -g root age.key /etc/thermograph/age.key +EOF +fi + +cat < FRONTEND_IMAGE_TAG=sha- \\ + $APP_DIR/infra/deploy/deploy-dev.sh + +Reach it (mesh only): + http://${TG_BIND_ADDR}:8137 + +Confirm it is NOT publicly exposed — this must show ${TG_BIND_ADDR}:8137 and +never 0.0.0.0:8137: + ss -ltnp | grep 8137 +EOF diff --git a/infra/deploy/provision-mail.sh b/infra/deploy/provision-mail.sh index 61d4788..8c76406 100755 --- a/infra/deploy/provision-mail.sh +++ b/infra/deploy/provision-mail.sh @@ -35,6 +35,12 @@ # sudo MAIL_DOMAIN=thermograph.org bash deploy/provision-mail.sh # sudo MAIL_DOMAIN=thermograph.org RELAYHOST='[smtp.provider.com]:587' \ # RELAY_USER=apikey RELAY_PASSWORD=secret bash deploy/provision-mail.sh +# +# MAIL_ENV picks which environment's deploy mode (env-topology.sh) sizes the +# Docker mail gateway below -- see the "WHICH gateway" comment. Defaults to +# prod; only matters if this box ever runs an environment in compose mode +# (it doesn't today -- prod and beta are both Swarm on this box; compose-mode +# dev lives on the other box entirely and doesn't run Postfix). set -euo pipefail MAIL_DOMAIN="${MAIL_DOMAIN:-thermograph.org}" @@ -43,6 +49,12 @@ RELAYHOST="${RELAYHOST:-}" RELAY_USER="${RELAY_USER:-}" RELAY_PASSWORD="${RELAY_PASSWORD:-}" +MAIL_ENV="${MAIL_ENV:-prod}" +SELF_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/env-topology.sh" +thermograph_topology "$MAIL_ENV" + if [[ $EUID -ne 0 ]]; then echo "run as root (sudo)" >&2 exit 1 @@ -68,17 +80,21 @@ postconf -e "myorigin = ${MAIL_DOMAIN}" # docker-compose.yml). Set DOCKER_MAIL_GATEWAY="" for a pure loopback-only null # client (app running natively on the host, not in a container). # -# WHICH gateway depends on the host's deploy mode: plain compose (beta, LAN) -# uses the pinned compose bridge (172.19.0.1/172.19.0.0/16, the defaults); -# Swarm-stack mode (prod) uses the docker_gwbridge gateway instead -- -# overlay tasks have no compose-bridge gateway -- so prod is provisioned with -# DOCKER_MAIL_GATEWAY=172.18.0.1 DOCKER_MAIL_SUBNET=172.18.0.0/16 (plus its -# MESH_MAIL_* listener below). Do NOT list an address that doesn't exist on -# the host: Postfix's master fails to bind and takes ALL listeners down -- -# exactly what happened when the compose bridge (172.19.0.1) vanished at the -# stack cutover while still listed in inet_interfaces. Also note: postfix on -# this distro is an umbrella unit; restart `postfix@-`, not `postfix`, for -# inet_interfaces changes to take effect. +# WHICH gateway to listen on comes from MAIL_ENV's deploy mode (env-topology.sh, +# TG_DEPLOY_MODE) -- not a hardcoded beta/prod split. Beta moved onto this SAME +# box as prod and is Swarm too now, so "compose" no longer means "beta"; it +# means dev, which lives on the other box entirely and never runs this script. +# Compose mode uses the pinned compose bridge (172.19.0.1/172.19.0.0/16, the +# defaults); stack mode uses the docker_gwbridge gateway instead -- overlay +# tasks have no compose-bridge gateway -- so a stack-mode MAIL_ENV (prod or +# beta; same box, same gateway) gets DOCKER_MAIL_GATEWAY=172.18.0.1 +# DOCKER_MAIL_SUBNET=172.18.0.0/16 (plus the MESH_MAIL_* listener below). Do +# NOT list an address that doesn't exist on the host: Postfix's master fails +# to bind and takes ALL listeners down -- exactly what happened when the +# compose bridge (172.19.0.1) vanished at the stack cutover while still listed +# in inet_interfaces. Also note: postfix on this distro is an umbrella unit; +# restart `postfix@-`, not `postfix`, for inet_interfaces changes to take +# effect. # # THE SAME TRAP FIRES AT BOOT, NOT JUST ON RENUMBERING (prod outage 2026-07-24). # A Docker bridge address does not exist until dockerd creates it, and the stock @@ -92,15 +108,23 @@ postconf -e "myorigin = ${MAIL_DOMAIN}" # it orders postfix@ after docker.service and wg-quick@wg0.service and retries # on failure. If you add an address here that some other daemon creates, add # that daemon to the drop-in too. -DOCKER_MAIL_GATEWAY="${DOCKER_MAIL_GATEWAY-172.19.0.1}" -DOCKER_MAIL_SUBNET="${DOCKER_MAIL_SUBNET-172.19.0.0/16}" -# Optional WireGuard-mesh listener: other mesh nodes (e.g. beta's Forgejo, whose +if [ "$TG_DEPLOY_MODE" = stack ]; then + DOCKER_MAIL_GATEWAY="${DOCKER_MAIL_GATEWAY-172.18.0.1}" + DOCKER_MAIL_SUBNET="${DOCKER_MAIL_SUBNET-172.18.0.0/16}" +else + DOCKER_MAIL_GATEWAY="${DOCKER_MAIL_GATEWAY-172.19.0.1}" + DOCKER_MAIL_SUBNET="${DOCKER_MAIL_SUBNET-172.19.0.0/16}" +fi +# Optional WireGuard-mesh listener: other mesh nodes (e.g. vps1's Forgejo, whose # mailer posts to 10.10.0.1:25 — see deploy/forgejo/docker-stack.yml) can relay -# through this box. Prod runs with MESH_MAIL_LISTEN=10.10.0.1 and -# MESH_MAIL_PEERS=10.10.0.2/32; both default OFF so a plain run stays a strict -# null client. Without these, re-running this script on prod would silently drop -# the mesh listener and break Forgejo's outbound mail — the live config was -# originally hand-applied and this script is the source of truth for it now. +# through this box. This host (vps2 -- prod AND beta) runs with +# MESH_MAIL_LISTEN=10.10.0.1 and MESH_MAIL_PEERS=10.10.0.2/32; both default OFF +# so a plain run stays a strict null client. 10.10.0.2 is vps1 (Forgejo, +# Grafana, dev) -- mesh IPs did not move in the vps1/vps2 split, only which +# environment runs where, so this is NOT "beta's" address. Without these, +# re-running this script on vps2 would silently drop the mesh listener and +# break Forgejo's outbound mail — the live config was originally hand-applied +# and this script is the source of truth for it now. MESH_MAIL_LISTEN="${MESH_MAIL_LISTEN-}" MESH_MAIL_PEERS="${MESH_MAIL_PEERS-}" postconf -e "inet_protocols = ipv4" diff --git a/infra/deploy/provision-secrets.sh b/infra/deploy/provision-secrets.sh index dd5fa4c..c70a8b5 100755 --- a/infra/deploy/provision-secrets.sh +++ b/infra/deploy/provision-secrets.sh @@ -1,16 +1,37 @@ #!/usr/bin/env bash -# Provision a host to render secrets from the SOPS vault at deploy time. Idempotent; -# run once per box (prod/beta) as a sudo-capable user. +# Provision a NAMED ENVIRONMENT to render secrets from the SOPS vault at deploy +# time. Idempotent; run once per environment (not once per box) as a +# sudo-capable user — vps2 carries two (prod AND beta), so it needs this run +# TWICE, each pointing SECRETS_ENV_FILE at that environment's own marker. # # SECRETS_ENV=prod AGE_KEY_SRC=/path/to/age.key bash deploy/provision-secrets.sh +# SECRETS_ENV=beta AGE_KEY_SRC=/path/to/age.key \ +# SECRETS_ENV_FILE=/etc/thermograph/beta-secrets-env bash deploy/provision-secrets.sh # -# Installs `sops` + `age`, then installs the age PRIVATE key at /etc/thermograph/age.key -# (0400) and writes /etc/thermograph/secrets-env. After this, deploy.sh renders -# /etc/thermograph.env from deploy/secrets/*.yaml. See deploy/secrets/README.md. +# Installs `sops` + `age` (idempotent, and shared across every environment on the +# box — one age key decrypts every deploy/secrets/.yaml, since they're all +# encrypted to the same recipient), then installs the age PRIVATE key at +# /etc/thermograph/age.key (0400) and writes SECRETS_ENV_FILE (default +# /etc/thermograph/secrets-env — right for a single-environment host, dev +# included). After this, deploy.sh renders that environment's own +# /etc/thermograph*.env from deploy/secrets/*.yaml. See deploy/secrets/README.md. +# +# SECRETS_ENV_FILE only matters for a by-hand run with THERMOGRAPH_ENV unset: +# env-topology.sh's thermograph_env_name() falls back to reading it to decide +# which environment a bare `deploy.sh` means, and a host running two +# environments (vps2) cannot answer that from one shared marker — a by-hand +# deploy of beta there must export THERMOGRAPH_SECRETS_ENV_FILE to match +# whatever path this script wrote. CI-driven deploys pass THERMOGRAPH_ENV +# explicitly and never consult this file at all. set -euo pipefail -SECRETS_ENV="${SECRETS_ENV:?set SECRETS_ENV=prod|beta}" +SECRETS_ENV="${SECRETS_ENV:?set SECRETS_ENV=prod|beta|dev}" AGE_KEY_SRC="${AGE_KEY_SRC:?set AGE_KEY_SRC=/path/to/the/age/private/key}" +# Where to write the env marker. Default matches the original, single-environment +# behavior; a SECOND environment on the same box (vps2's beta, alongside prod's +# default marker) must pass a distinct path so provisioning one can never +# overwrite the other's marker. +SECRETS_ENV_FILE="${SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" SOPS_VERSION="${SOPS_VERSION:-v3.13.2}" AGE_VERSION="${AGE_VERSION:-v1.3.1}" BIN="${BIN:-/usr/local/bin}" @@ -35,11 +56,16 @@ install_age() { install_sops install_age -echo "==> Installing age key -> /etc/thermograph/age.key (0400) and marker (${SECRETS_ENV})" -sudo mkdir -p /etc/thermograph +echo "==> Installing age key -> /etc/thermograph/age.key (0400) and marker ${SECRETS_ENV_FILE} (${SECRETS_ENV})" +sudo mkdir -p /etc/thermograph "$(dirname "$SECRETS_ENV_FILE")" sudo install -m 0400 "$AGE_KEY_SRC" /etc/thermograph/age.key -printf '%s\n' "$SECRETS_ENV" | sudo tee /etc/thermograph/secrets-env >/dev/null -sudo chmod 0644 /etc/thermograph/secrets-env +printf '%s\n' "$SECRETS_ENV" | sudo tee "$SECRETS_ENV_FILE" >/dev/null +sudo chmod 0644 "$SECRETS_ENV_FILE" echo "==> Done. Verify: sops --version && ls -l /etc/thermograph/" echo " Next: seed deploy/secrets/*.yaml, dry-run render, then deploy (see README)." +if [ "$SECRETS_ENV_FILE" != /etc/thermograph/secrets-env ]; then + echo " NOTE: non-default marker -- a by-hand deploy of ${SECRETS_ENV} on this box" + echo " must export THERMOGRAPH_SECRETS_ENV_FILE=${SECRETS_ENV_FILE} (CI-driven" + echo " deploys pass THERMOGRAPH_ENV explicitly and never consult this file)." +fi diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index b3b5b3c..fb5731c 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -15,12 +15,21 @@ # (prod|beta|dev). A host without them keeps whatever /etc/thermograph.env it already # has — so merging this is a safe no-op until a host is deliberately migrated. See # deploy/secrets/README.md. +# render_thermograph_secrets [env_name] [out_file] +# +# env_name and out_file are explicit since vps2 began running two environments. +# A host-wide marker file cannot name two environments, and one output path +# cannot hold two renders: on vps2 prod renders prod.yaml to /etc/thermograph.env +# while beta renders beta.yaml to /etc/thermograph-beta.env. Both arguments keep +# their pre-split defaults (marker file, /etc/thermograph.env), so a +# single-environment host and a by-hand run behave exactly as before. render_thermograph_secrets() { local repo="${1:-.}" # repo root holding deploy/secrets local key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" - local env_name - env_name=$(cat "$marker" 2>/dev/null || true) + local env_name="${2:-}" + local out="${3:-/etc/thermograph.env}" + [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true) # The per-host file is required; common.yaml is optional so the initial cutover can # seed each host as an exact copy of its live env (byte-identical render, no value @@ -31,7 +40,7 @@ render_thermograph_secrets() { # (a) Not configured for SOPS at all — no env marker, or no age key. The LAN # dev box is legitimately in this state. Saying so and succeeding is right. if [ -z "$env_name" ] || [ ! -f "$key" ]; then - echo "==> SOPS secrets not configured here; using existing /etc/thermograph.env" + echo "==> SOPS secrets not configured here; using existing $out" return 0 fi @@ -57,7 +66,7 @@ render_thermograph_secrets() { return 1 fi - echo "==> Rendering /etc/thermograph.env from deploy/secrets (common + ${env_name})" + echo "==> Rendering $out from deploy/secrets (common + ${env_name})" # The age private key is root-owned (0400). Read it directly if we can, else via # sudo into SOPS_AGE_KEY — so the key never has to be readable by the deploy user. # (The render needs sudo to write /etc/thermograph.env below anyway.) @@ -119,12 +128,12 @@ render_thermograph_secrets() { # failed in-place `cat` write to status 0. Capture the status instead, so a # half-written /etc/thermograph.env fails loudly rather than deploying stale. local rc=0 - if [ -f /etc/thermograph.env ] && [ -w /etc/thermograph.env ]; then - cat "$tmp" > /etc/thermograph.env || rc=1 - elif install -m 0640 "$tmp" /etc/thermograph.env 2>/dev/null; then : - elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" /etc/thermograph.env 2>/dev/null; then : + if [ -f "$out" ] && [ -w "$out" ]; then + cat "$tmp" > "$out" || rc=1 + elif install -m 0640 "$tmp" "$out" 2>/dev/null; then : + elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then : else - echo "!! cannot write /etc/thermograph.env (need file write access or passwordless sudo)" >&2 + echo "!! cannot write $out (need file write access or passwordless sudo)" >&2 rc=1 fi diff --git a/infra/deploy/secrets/README.md b/infra/deploy/secrets/README.md index 7242fd3..4b85e59 100644 --- a/infra/deploy/secrets/README.md +++ b/infra/deploy/secrets/README.md @@ -16,8 +16,8 @@ no per-host duplication. |------|-------|------------| | `common.yaml` | The 16 values identical on **prod and beta** — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config. **Not layered under `dev`** — see below | ✅ | | `prod.yaml` | Prod's own — the three held-back credentials below, sizing (`APP_CPUS`, `DB_CPUS`, `DB_MEMORY`, `WORKERS`), `THERMOGRAPH_BASE_URL`, and the Discord + mail credentials that exist nowhere else | ✅ | -| `beta.yaml` | Beta's own — the three held-back credentials, sizing, `THERMOGRAPH_BASE_URL` | ✅ | -| `dev.yaml` | The LAN dev box's own — **self-contained**, 12 values, no production credential among them | ✅ | +| `beta.yaml` | Beta's own — the three held-back credentials, `THERMOGRAPH_BASE_URL`, `APP_CPUS`/`WORKERS`, plus `THERMOGRAPH_MAIL_BACKEND=console` and `THERMOGRAPH_DISCORD_BOT=0`. Its `POSTGRES_PASSWORD` and `THERMOGRAPH_DATABASE_URL` name beta's **own role and database** (`thermograph_beta`) on the instance it shares with prod. `DB_CPUS`/`DB_MEMORY` were **removed**: beta no longer runs a database of its own, and leaving them would have implied it sized one — the shared instance is sized by `prod.yaml`'s values (see `deploy/stack/thermograph-stack.yml`'s `db` service) | ✅ | +| `dev.yaml` | Dev's own (vps1) — **self-contained**, 12 values, no production credential among them | ✅ | | `centralis.prod.yaml` | **Centralis's** nine variables, rendered to `/etc/centralis.env` on prod. A different service, a different output file, a different renderer — see below | ✅ | | `example.yaml` | Format reference / CI fixture (fake values) | ✅ | | `../../.sops.yaml` | Which age recipient files are encrypted to (plaintext config) | — | @@ -25,28 +25,55 @@ no per-host duplication. ### Three credentials are deliberately NOT in `common.yaml` `POSTGRES_PASSWORD`, `THERMOGRAPH_AUTH_SECRET` and `THERMOGRAPH_DATABASE_URL` -hold **identical values on prod and beta today**, so by the mechanical rule they -belong in `common.yaml`. They are kept per-host anyway. +hold **different values on prod and beta today** (beta has its own database +role/password on the shared instance, and its own auth secret), so unlike the +rest of `common.yaml` they were never candidates for sharing on values alone. +They stay per-environment for a reason that matters more now than it used to: -These are the credentials that let one environment act as another: with them, a -foothold on beta is a foothold on prod's database and prod's session signing. -Beta is the *more* exposed box — it serves public Forgejo and Grafana. They match -only because beta was seeded from prod, not because the two are meant to be one -system. +**These are the credentials that let one environment act as another.** With +matching values, a foothold in one environment's rendered env file is a +foothold on the other's database and the other's session signing. That used +to matter because beta and prod were separate boxes; it matters for a +different reason now that they're co-resident on vps2 — keeping the +credentials themselves separate is what stops co-location on one host from +also becoming equivalence at the credential level. `deploy/db/provision-env-db.sh` +enforces the same boundary from the database side: beta's role +(`thermograph_beta`) is `NOSUPERUSER`/`NOCREATEDB`/`NOCREATEROLE`, owns only +its own database, and `CONNECT` is revoked from `PUBLIC` on it — so even a +leaked beta credential cannot reach prod's data, and the reverse. -Keeping them per-host costs nothing now (same values, one extra line each) and -buys the ability to diverge: rotating prod's database password stops implying -"and beta's too". Moving them into `common.yaml` would encode the equivalence as -intentional and turn breaking it into a migration rather than an edit. +**What actually separates beta and prod now, stated plainly:** they share a +host by design (see `deploy/env-topology.sh`'s header), so a host-level +compromise of vps2 is shared between them — separate SSH credentials no +longer buy a host boundary between beta and prod, the way they once did when +beta and prod were different machines. What remains is isolation at the +**database** level (separate roles, separate databases, `CONNECT` revoked from +`PUBLIC`) and the **file** level (separate checkouts, separate rendered env +files, separate stack env files, separate loopback ports). Keeping +`POSTGRES_PASSWORD`/`THERMOGRAPH_AUTH_SECRET`/`THERMOGRAPH_DATABASE_URL` out of +a shared file is part of that file-level isolation: nothing on vps2 would ever +hand you prod's auth secret merely because you have beta's. -If you ever *do* want them to differ, change one file. That is the whole point. +**vps1 is the box that must never hold prod credentials.** It runs Forgejo, +Forgejo's CI runner, Grafana, and the `dev` environment — which runs whatever +branch is currently in flight, reviewed or not. That is the actual reason +`dev` renders `dev.yaml` **alone** and never layers `common.yaml` (see below): +not squeamishness about a dev box, but that vps1 is uniquely positioned to +leak whatever it's handed, and `common.yaml` is the fleet's shared production +credential set. -`dev.yaml` is the case that already differs: all three of its copies are its own, -generated on the desktop. The rule earned its keep the first time it was used. +Keeping the three held-back credentials per-environment costs nothing (one +line each) and buys the ability for them to diverge further, or for a future +environment to be added without inheriting anyone else's values by accident. + +`dev.yaml` is the case that already differs completely: all three of its +values are its own, generated on vps1 and never shared with prod or beta. At deploy the renderer concatenates `common.yaml` then `.yaml`, so a **host value wins** (env_file / `source` take the last occurrence of a duplicate key). -`` comes from `/etc/thermograph/secrets-env` on the box (`prod`/`beta`/`dev`). +`` comes from an explicit `THERMOGRAPH_ENV`/argument on vps2 (which runs +two environments) or `/etc/thermograph/secrets-env` as a fallback elsewhere — +see `deploy/env-topology.sh`. ### `dev` is vaulted, but renders `dev.yaml` **alone** @@ -65,17 +92,17 @@ is a **plaintext concatenation**, so layering it would put every one of those in `/etc/thermograph.env` on the dev box and, through `env_file:`, into the environment of every container in the dev stack. -The dev box is the operator's desktop. It is also the Forgejo CI runner, whose -`docker`-labelled jobs get the host docker socket automounted, and the stack it -runs is the `dev` branch — code that has not been through the gate that guards -prod. Handing that the estate's read-write object-storage keys and the push -signing key is a strictly larger blast radius than anything the vault buys back. -An override in `dev.yaml` would not help: last-wins governs *consumers*, but the -production value is still physically a line in the file. +The dev box is **vps1** — the same box that runs Forgejo and its CI. Its +`docker`-labelled runner jobs get the host docker socket automounted, and the +stack it runs is the `dev` branch — code that has not been through the gate +that guards prod. Handing that the estate's read-write object-storage keys and +the push signing key is a strictly larger blast radius than anything the vault +buys back. An override in `dev.yaml` would not help: last-wins governs +*consumers*, but the production value is still physically a line in the file. One value in `common.yaml` is also simply **wrong** for dev: `THERMOGRAPH_COOKIE_SECURE=1` is right for the TLS hosts and silently breaks login -over dev's plain-HTTP LAN URL. `dev.yaml` sets `0`. +over dev's plain-HTTP mesh-only URL. `dev.yaml` sets `0`. So `dev.yaml` is self-contained. Where the value is shared-but-harmless (`PORT`, `THERMOGRAPH_BASE`, `TIMESCALEDB_TAG`) it carries its own copy — a @@ -87,8 +114,8 @@ duplicated constant is the cheap half of this trade. |-----|-----| | `POSTGRES_PASSWORD`, `THERMOGRAPH_DATABASE_URL` | dev's own. Deliberately the weak, well-known `thermograph-dev` — the value dev's `pgdata` volume is already initialized with, and the DB is never published off the compose network. Kept per-host for the reason below, and *not* shared with prod | | `THERMOGRAPH_AUTH_SECRET` | dev's **own**, freshly generated. Fixes a live crash loop: the `daemon` service refuses to start without it (it derives the `/internal/*` token from it), so dev's daemon had been restarting every 60s. Prod's value must never be here — it signs sessions and verification links | -| `THERMOGRAPH_BASE_URL` | `http://10.0.1.216:8137`. Unset, the app defaults to `https://thermograph.org`, so dev's IndexNow pings and Discord links claimed to be prod | -| `THERMOGRAPH_COOKIE_SECURE=0` | plain HTTP on the LAN | +| `THERMOGRAPH_BASE_URL` | `http://10.10.0.2:8137` (vps1's mesh address). Unset, the app defaults to `https://thermograph.org`, so dev's IndexNow pings and Discord links claimed to be prod | +| `THERMOGRAPH_COOKIE_SECURE=0` | plain HTTP on the mesh-only URL | | `THERMOGRAPH_MAIL_BACKEND=console`, `THERMOGRAPH_DISCORD_BOT=0` | explicit fail-safes. Both are already the code defaults; stating them means dev cannot start mailing or open a Discord gateway by inheriting a value | | `PORT`, `THERMOGRAPH_BASE`, `TIMESCALEDB_TAG`, `DB_MEMORY`, `WORKERS` | non-secret config that would otherwise have come from `common.yaml` | @@ -104,8 +131,8 @@ duplicated constant is the cheap half of this trade. (`backend/indexnow.py`). Prod's key is what authenticates URL submissions *for thermograph.org*. - **`THERMOGRAPH_METRICS_TOKEN`.** With no token `/api/v2/metrics` is - direct-loopback-only, which is exactly right on a LAN box; dev's Alloy agent - ships logs, not metrics, so nothing scrapes it. + direct-loopback-only, which is exactly right on a mesh-only box; dev's Alloy + agent ships logs, not metrics, so nothing scrapes it. - **`REGISTRY_TOKEN`.** dev pulls with the persistent `docker login` credential already in the host's docker config, like the prod/beta CI paths. A vault copy would only duplicate a credential the box already has into a more readable file. @@ -224,12 +251,17 @@ files), so layering would only push the VAPID private key, both S3 keypairs and - `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time. - **Back it up** in the password manager. The public recipient is in `.sops.yaml`. -On the **dev desktop** those two are the same file, on purpose. `render-secrets.sh` -falls back to `sudo cat` for a root-owned key, the desktop has no passwordless sudo, -and the Forgejo runner is a `systemd --user` service with no tty — a `/etc` copy -would make every CI-triggered dev deploy hang on a prompt it cannot answer. -`deploy-dev.sh` points `THERMOGRAPH_AGE_KEY` at the operator's keyring instead, so -the box keeps **one** copy of the recovery root rather than two. +On **vps1** (dev) those two can end up being the same file. `render-secrets.sh` +falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes +the account driving a CI-triggered dev deploy has no passwordless sudo and no +tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) — +inherited from the old desktop setup, where that was true of the whole box. +Whether it's still true of vps1's CI-runner account specifically (as opposed +to the `agent` login, which does have passwordless sudo there per `ACCESS.md`) +wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY` +at the operator's keyring as a fallback either way, so the box keeps **one** +copy of the recovery root rather than two regardless of which account ends up +mattering. ## Prerequisites (once per machine) @@ -246,8 +278,10 @@ go install github.com/getsops/sops/v3/cmd/sops@latest # if you have Go ```sh sops deploy/secrets/common.yaml # opens DECRYPTED in $EDITOR; re-encrypts on save git commit -am "rotate " && git push forgejo -# beta auto-deploys on push to main. prod (no CI) — one command: -ssh agent@169.58.46.181 'cd /opt/thermograph && git pull && deploy/deploy.sh' +# beta and prod both auto-deploy on push (main -> beta, release -> prod) via +# the Deploy workflow. To force it by hand on vps2, say which environment: +ssh agent@169.58.46.181 'cd /opt/thermograph && git pull && THERMOGRAPH_ENV=prod deploy/deploy.sh' +ssh agent@169.58.46.181 'cd /opt/thermograph-beta && git pull && THERMOGRAPH_ENV=beta deploy/deploy.sh' ``` A host-specific value (e.g. `POSTGRES_PASSWORD`) is the same, editing `prod.yaml` / @@ -264,10 +298,13 @@ commit, and the next `deploy-dev.sh` run (merge to `dev`, or by hand) renders it ## Setting up a new dev machine -The desktop is hands-on: Centralis cannot SSH to it, so this is typed, not -automated. `provision-secrets.sh` is **not** the right tool here — it installs the -age key root-owned at `/etc/thermograph/age.key`, which the runner cannot read -(above). +Dev is provisioned like any fleet host now (`deploy/provision-dev.sh`, run as +root on vps1), not the sudo-free desktop bootstrap this predates. The +secrets-specific steps below are still worth doing by hand rather than +folding into that script: `provision-secrets.sh` is **not** the right tool +here — it installs the age key root-owned at `/etc/thermograph/age.key`, +which the CI-runner-driven render may not be able to read (see the age-key +note above). ```sh # 1. sops + age on PATH, and the age private key in the operator's keyring: @@ -293,17 +330,17 @@ SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt \ # 5. Deploy. deploy-dev.sh refuses to run if render-secrets.sh in the checkout # cannot honour THERMOGRAPH_SECRETS_SKIP_COMMON, rather than leak. -APP_DIR=~/thermograph-dev bash deploy/deploy-dev.sh +APP_DIR=/opt/thermograph-dev bash deploy/deploy-dev.sh ``` To take a dev box **back** off the vault, delete `/etc/thermograph/secrets-env`: presence detection turns rendering off and `deploy-dev.sh` falls back to its built-in `POSTGRES_PASSWORD` default. -A *different* dev machine (a second desktop, a laptop) needs its own values, not a -copy of this one's: give it its own `.yaml` and its own marker name, so the two -can never be confused for one host — the same argument as the three held-back -credentials above. +A *different* dev machine (a laptop running `make dev-up` locally, or a future +second dev host) needs its own values, not a copy of vps1's: give it its own +`.yaml` and its own marker name, so the two can never be confused for one +host — the same argument as the three held-back credentials above. ## Add a new secret @@ -332,17 +369,24 @@ git commit -am "rotate age identity" && push + deploy ## First-time bootstrap (seed from the live boxes) +**Historical** — this ran once, before the vps1/vps2 split, when beta was its +own box at `75.119.132.91`. That address is **vps1** now (dev + Forgejo + +Grafana), not beta; re-running the beta line below as written would seed +`beta.yaml` from the wrong host. Kept for the shape of the process, which is +unchanged if a new environment is ever bootstrapped the same way — just point +`seed-from-live.sh` at wherever that environment actually lives today. + Order matters — values must match the live env exactly so `AUTH_SECRET` / VAPID / `POSTGRES_PASSWORD` don't rotate unintentionally: 1. Install `sops`+`age` and drop `/etc/thermograph/age.key` (0400) + - `/etc/thermograph/secrets-env` on prod & beta (`provision-secrets.sh`). + `/etc/thermograph/secrets-env` on the target hosts (`provision-secrets.sh`). 2. Seed each host's file from its live env with the helper (pulls over SSH, encrypts, and verifies the render round-trips — run it on your own machine, it reads live - secrets): + secrets), e.g. as originally run: ```sh deploy/secrets/seed-from-live.sh prod agent@169.58.46.181 - deploy/secrets/seed-from-live.sh beta agent@75.119.132.91 + deploy/secrets/seed-from-live.sh beta agent@75.119.132.91 # beta's address AT THE TIME; now vps1's ``` That writes exact per-host copies (`prod.yaml`/`beta.yaml`); factor shared values into `common.yaml` later. Commit. diff --git a/infra/deploy/secrets/beta.yaml b/infra/deploy/secrets/beta.yaml index 2f3fa15..7a24542 100644 --- a/infra/deploy/secrets/beta.yaml +++ b/infra/deploy/secrets/beta.yaml @@ -1,23 +1,23 @@ -APP_CPUS: ENC[AES256_GCM,data:Wg==,iv:9kB0WruqLCIkJp7i+t5kGB4qv8NzHwgTQlFmYwhTEXc=,tag:q6cKDI86YrFJjGRnEkdweA==,type:str] -DB_CPUS: ENC[AES256_GCM,data:7w==,iv:xDOeZo3dJ5gnXIEGyOBGZm4RUTNyzinMkvxWoY/EbQs=,tag:ikmAjzeM/U4w5HDZ+C7mgA==,type:str] -DB_MEMORY: ENC[AES256_GCM,data:apg=,iv:4+GHbccVBFUGtrP12a2oEya7Hz0KUhJAFzdwpqQJ2sc=,tag:X0h2nYza80c9PjeNgqX7Uw==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:X5HmOvHVtm2mjeLZO7FaEWP0Qq767D0quycr2iI3/Mw=,iv:BQsjJaDPW9RxnMldE1rCU3yP5WBOXvc9VjnTim3kMZs=,tag:yfdi+T7lBbBK544bSvEZtQ==,type:str] -THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:YjgknnUhAU0gW6YuMCmlZYfVhOgye52MLhVF7nKMuc0ToGn0OTqqoiXN0A==,iv:Q9XCoKa5Y/7V9b72tktGZ12jgPumC4kJKjrtCM39ihw=,tag:3kEpMjLCEFQcpSir3N9f6w==,type:str] -THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:bus0/ibfLohpsszHZKKoGe5P0/Su5i36Hk2nzg==,iv:QI3crdbipse62xdakCGxPIA4wmdq/T4KzZSvi/NE1Xg=,tag:/S4Bzk2aDoA6ahDlW7qG8w==,type:str] -THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:xMd8m6VajAjtwQy8DDqTs4VbyASG8ua8FUZHfPmmEzZEb4cTtYILJ+z6LXhcyvdenDsPg2/GJx5BApvVAKSrWrnSzNWnp+pAs7x+VnNzZ2giZ9meiQ==,iv:MOU+SB/taEc/ExRc07ChF1ksOQiaDj7TPMQq+aV4Ltg=,tag:XP2h9KBGJE7JzripO7yjcg==,type:str] -WORKERS: ENC[AES256_GCM,data:BQ==,iv:XYgQ+Sn9OnbUR8LVW+4r9vhnQQJZmliVomfyRmUqMsw=,tag:JzM3JsNR08z1pePSTl/dhw==,type:str] +APP_CPUS: ENC[AES256_GCM,data:qg==,iv:T3eyF+9ssUCWDiFr8Wg0tRpgeV3BSLIjOSBE2i7WOlc=,tag:wisw0gQ6E8QCNom/Zd3mWQ==,type:str] +POSTGRES_PASSWORD: ENC[AES256_GCM,data:r5ygtwgycLjjGXSehpLt/RpUkhLyZ4lHPt6KiqNd+ng=,iv:qgqfU1mtm7ZuqSbeUEB9cgT56bOAkc0UFb3BMxED+xg=,tag:WiEoVHrA96WNFROqmFyElA==,type:str] +THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:aZySO3pAVB24S7uCsblRtd65U+OkBatK2tmuzeALdlPsqRhMqRioXQLgMQ==,iv:5zxNwj39E/AOJXzV/AEX9w1n3LSnl9PnS3cp5wVRi2E=,tag:jQRGr2436E4qqBE53iltqQ==,type:str] +THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:9IxftZqwSBG/1MY+jcALQeeTbIk+tJ/8GXGUgQ==,iv:EvsDUExWQHpidriMDDj0flM8Rz25iFycVPJLrK240+s=,tag:7rKr5KUDMe4J7U4OHUvPRg==,type:str] +THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:SnD/achZHduuWwwrQYOseqw0mquV1B4k4c2eom81dqtib6SxGj1OabHYj0rJd+o9Y5sH+kTh+zkFHpBi0J7c5GEGzhBqLctt530uMlOstvu1Hh7qL+DareyBF+SHwG4=,iv:qPFUnfRpQ6gIfBx6phyl5RWsNgEFYdNqW8oVzDZgXU8=,tag:djaqFRpDetNoH7Ux1PmQ3Q==,type:str] +WORKERS: ENC[AES256_GCM,data:gQ==,iv:jkwIzTpiEtEtbt7acP8DWRbl51baTFuQiw4oJxb3eyA=,tag:X2Gqy3icYRKrI6F99lnuyQ==,type:str] +THERMOGRAPH_MAIL_BACKEND: ENC[AES256_GCM,data:OrchUc9tpw==,iv:SmAZ+USGesaCyaOdKopqKmPeG7y/WtDry9G3aDVgP/Q=,tag:EGh5foV6uClrIJCdkjP3Hg==,type:str] +THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Yw==,iv:jsHid5lQ75shgfIJkrsLnlhmcJ1aJWuxkUyhmpg52Dc=,tag:UqtZjQpYwQH5uvEYC9mp7w==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFWHVZU2xSS1pmcy91S2w5 - MWNhQW90TDdQdDdJV1BHM2Y0a1d6aFJOS2hnCjRodHZlY0orODdpS3QwTVdTdXM5 - Z3RJOFhyR3AzbjljaExkQjN0YUJ2ckUKLS0tIGpua1Z2TDJrRzE3WFZlMWNpL1Ja - VG45eVM0MldLejRNd0pweWNna3JYWlUKWuNU+6PqKlbr7F0ckrNxsMF2OyXh1fMu - cLFBIQg/7vO7O7PJ0VIy0Ugfq6gj2Gv91qKJUGeOXOw3tv1Y6HHbjA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqdFZ3MWVCSDdvdnVLWlZH + My9JMzF2TmhiSlJjbWRYbUIzSm0zb1dtdTI4CmI4N2o0YzR1NGNoTmI5OXBJM1Bj + OVF3WWEvYmlWczR2dUJpYTROR2pvK2sKLS0tIDM5blFFdnZwRHpDOEZ6aHhyL0dn + d1BDWVV6YXNQNWo3dWRqN2FVUC85UmMK8jjGvQxKDsnlr7i95Ar509nihDgRb/JW + qzLM5fwMU16HOPMiAPd/zH3rUyKcZxMtITR6QLcuSNLByURg4ZMBnQ== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-25T00:38:24Z" - mac: ENC[AES256_GCM,data:lTpEmA8gweS+QlAakIziXxAEiHBUsNDk5lGwVG+uaXNhifIYWbu+QWkklP4wES9o3pu4TNCzFboRWkBbnzhW8HYRglkSChsmmFXDChz67IYlaveCFwczxffp39iypnV/EKaBCa8m1eDJ5kugzr2s7+blagCH0TwpNXBqZShyGNE=,iv:KBF6CreB4oTJGwNetTj27Grox30X3s3qEEYb1cJXmIE=,tag:YfRb5yAoY8K/TXspoZYdcA==,type:str] + lastmodified: "2026-07-25T21:43:52Z" + mac: ENC[AES256_GCM,data:/+kmyMP20n7fQRO2FdiHWaLl8MAfnMjtF8OE5HEUCLpRGet3s+IqYb8DajFIF9SCibW7aEqb6ir9m0KBx0IRW4TTsvK6JYFyRZKKtt4oiCzRaGwUMwhI75DE6xYdHxEbdAC6Q+qFUKGtRcIGhSX1LrlGEloSJggdEFSgfTlJLNg=,iv:UX1zMFw70ODiPolKSjEj8adKxa49Kf5dzfDA7ePtxmE=,tag:dA0OqIPTvXBlMPNH4oCNtg==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/secrets/dev.yaml b/infra/deploy/secrets/dev.yaml index eb84992..76e5b51 100644 --- a/infra/deploy/secrets/dev.yaml +++ b/infra/deploy/secrets/dev.yaml @@ -1,27 +1,27 @@ -PORT: ENC[AES256_GCM,data:0LjaWw==,iv:3e8Q71QFlMAhQStgoiVh/c2ELI3+7UkWseAFv8BnElU=,tag:/wYfKmqErJpsLBjg1gs4Tw==,type:str] -THERMOGRAPH_BASE: ENC[AES256_GCM,data:Ww==,iv:e2yNIAAw4dtlahfbA+PCM/jyODKFwWFNfVpVJ1JoREc=,tag:QzoGwDgIwdA9OFAMIPej7Q==,type:str] -TIMESCALEDB_TAG: ENC[AES256_GCM,data:7Oj7OoxoaG51whg=,iv:PPbfdn4DHMxTPA5FUqv+lEY29wQsVX+g0aVuWvraYrU=,tag:NAyJZlRXU1iKo4jwBctUtw==,type:str] -DB_MEMORY: ENC[AES256_GCM,data:/3U=,iv:jFI/9NQaz8oIqBWAKydWUEEI7bI3SmLnsoBlVgfKw7I=,tag:oBoFANuTgZV0jXC9qR5IHQ==,type:str] -WORKERS: ENC[AES256_GCM,data:VQ==,iv:+d84ByEU22XVQi43NcRn8cHmbW9XudJMj0guidsHxBQ=,tag:XM//7y0VQCwini4CGL7pSg==,type:str] -THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:FK2kOaZG8ESwDi3VMOlo0XvBKvdADQ==,iv:c6GvCrAnAf+ZFh53xg3ZOlf934Ls+PyR/+CXRi933lE=,tag:cYLXvxP5hk7/6gmz/CCP0A==,type:str] -THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:zg==,iv:sVr33CvZoZmn0LcHlP+FVA8esjGy0aKe1A2iPrrrT0I=,tag:3YneNF7m05fCRGfMgYzByA==,type:str] -THERMOGRAPH_MAIL_BACKEND: ENC[AES256_GCM,data:pRBtOu6cTw==,iv:88QgyY6rwwp/6Y32r6Us1M+b7YpSLPfKIsCL2j/YuIY=,tag:FrchZZrMcCSeOFRWbePM4Q==,type:str] -THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:NQ==,iv:Zl888EtEoQQd8FdqGb4/N7gGMT4yEymQfWd2fZZCNVA=,tag:Xspr8/taeQwh1qHhfFbLfw==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:l076hUutOAKw7+X6wboX,iv:9WVz3SVUikWREVjB6GsJ+EXWO3PvX+g+OaZNGN9f+jU=,tag:OkVuEqEbTSikpCNEySPX5A==,type:str] -THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:kXCYmZXewa8JSY5iJd0cHDtsKrxyKEl5tSVwWeQEd8u7XU+/dn4BExT4RDB3vUWVFVoHXYIXedMnDO8h0QfDCxSI13s=,iv:s1bDjdH4E4T+kF126saOrKlx3AnKEkRWF68KEPcCUe0=,tag:nCzD9U/7g28bJmUXkiYh3Q==,type:str] -THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:EjwKGKh1od5vgvm8LLv1l6TXO7nshUwh5ihXDpETl+3ldtF8L4H3CWszsw==,iv:4mqwEXWpGpLYbIUotHj9Qf7FSD0ATwkxwhfuL7pG+Zc=,tag:0iLU0ee86H0CgmgPCCtatQ==,type:str] +PORT: ENC[AES256_GCM,data:VCheDw==,iv:u9IC7psxzcsLuuWhU89fUJfVsmCI7lECbjxzKmC/nGM=,tag:ZTG/kG2vSNvUUlItAjsCHg==,type:str] +THERMOGRAPH_BASE: ENC[AES256_GCM,data:Aw==,iv:Pcmbm6D126jncQnmBquZ/t6qMaaUF7UDnQV6q85CsrY=,tag:SGakFybX3TNKAWvFo9gsEw==,type:str] +TIMESCALEDB_TAG: ENC[AES256_GCM,data:ghewNqRFuYJfBgE=,iv:q1GsQM3QZBW4A95U9NY90RR9WRmPV6miIxGLgLmPTps=,tag:eJARZNyqnJbHhtXXUI1AWw==,type:str] +DB_MEMORY: ENC[AES256_GCM,data:k2U=,iv:ZRCjRUHbNvVrCdGWqmOoJBtsJLeLVSVm32szpQ23HN4=,tag:KE5jUadHZnKFPis0friNbg==,type:str] +WORKERS: ENC[AES256_GCM,data:yQ==,iv:6VE9mjS7QLuOWazReqyt1DhAhamkvWSm92B2d/ZCZYo=,tag:hDl1/E11xwUyPVhFNhzOuQ==,type:str] +THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:/sOgmM0eCf7dWldK0iZaDWE0W4aN,iv:dpiLeHLanO2zwtMmPMbsqR4aTVaXfT+V2Ux9K8Alh3w=,tag:PDd/L2nt4X750xCKjHnSPw==,type:str] +THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:Bw==,iv:dsLcLu126A3d7dTqBgQY1SOvJfKOBT/ye4ucIB4w6B4=,tag:/G7Qvlzb6KKVVjqmn3d/Nw==,type:str] +THERMOGRAPH_MAIL_BACKEND: ENC[AES256_GCM,data:ryWhCkSdxg==,iv:Lpl1CIjTINV8rF8BW5MHZD4Ji802i96zcyqF3xRMLcY=,tag:I0nbb6IP6LM/Y+KaHjBDpg==,type:str] +THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Rw==,iv:o5tr5lQGXFkB6kTJ/dDARBWECDYRdofFlZ7v/HdMe5k=,tag:Lv97fH6u/nIKZlJr5HjbZw==,type:str] +POSTGRES_PASSWORD: ENC[AES256_GCM,data:IBVel17GKtrW5yzqDjBi,iv:yEPSSYR/Ng9rjSvSl0veNRDPLayF0SvWZ1T8BSY1l50=,tag:hbB4Uo6YluQPgl9XyvgfKw==,type:str] +THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:KjsSo+IglfY5MhtEsX3HJpExAtBwzmSu3E+O7Y22B2WuIrTKGDxG81WwvQGABYfKVel6kzzv8uKczq1qEtOwHRdMsRI=,iv:VPJNe1GrJa7BeAA/N/fqyBjBF+ciF8btE4j/TwrV3cA=,tag:tkyzkR1LxY8z28bjb0bJ7w==,type:str] +THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:ccEQxcNxMEn8/Gf2knQaVMSoBp7BRJ7zWhr9KmawaSMrwHGU7kADDWCoMw==,iv:nDpMHhP0ubjzz2LIY0QMGFLGP1M10zOIJb6pPgri0PI=,tag:+Z2j5XLdV+LuSK7hWf5r0Q==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzV0lrcUpjeU81TmJDRVpa - elppeU1mMUlpc3lKSTVtU3MxUi83bzN2dVVRCmxZbDJFUkJQVjhCSmljNFpPaWVu - dkNqTkJTd1ZEaHZsZitpTWlJdElCK28KLS0tIEdSTXN6S1Y2eE12U3o2RGxBNXNN - bDBlYkNxMDVTVEFLYVIyVkhIRnNwOGMKv/TMt307XqvzBLCOCA5C4kXFV9iJeVBn - 7gyzb5MRbHbDoNAY/5ckU3341uWUXUQ+IrPvVt2snAdXIlWghm+HwA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyb2dCeVgrbGxGSnNpaUE3 + RlovN0RrejVmREY0ckNGaExpNWkvSjcyRW1RCjYvc1IrMWwvdmJGU2g5bU52Mmgw + SVY1dnFHd0cyNkM1ZWhJaWlIZldzZlEKLS0tIGtLZDRSMS82UjRFRVdZU3J1ckMz + T0g1WnRUUGFFRTVyMlF6WnZiSEoweEEKPLwcTNiE4s24J0kComnNJj5jXIBRBzMh + PVktsQm9m1ojaG0hgCZsFaMOOnE5O0Vi41jgG7WWLSNZRaVggvRbJg== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-25T00:59:09Z" - mac: ENC[AES256_GCM,data:Ga1lMooQrBhU+YasAbN3THZvjrct21KRbuxOoXf+72K5mrgPmqZyFDdB71B8dOUzmz4cqTPw3+jQx2s2ZrZCEiR3qmb897I8hWbTiFWnURSC7rjytxIe6kLhDBjCtso8ThKUMqNjAglqjkytHXljP7Cg4zCuGLEgczoakEJVB7s=,iv:NFHljXdG619XJMjy+1OuUknu+QaG5nH68fvo/zAf/TE=,tag:MyFf/edujKzypOwloVUGAA==,type:str] + lastmodified: "2026-07-25T21:44:03Z" + mac: ENC[AES256_GCM,data:ok66XdbCyLyU/2UxQ6Fo63S16r6VETT7Qk/KVJ2b4rrdd4hfOMMAvmUfFmBNkxclxBp+UMOlGJ3XpGRZRzcmWR9o1eXpFlussgDbWHU+KLoIVqAVkUsltVM9aDkg+3dl6/vInftWGw24LbfDh1WhYQ9+OwyKxZntj0sNHAShc8E=,iv:eBTeVtkAv6qcldT0x+zyO2yWVrCm0nhRUJjzaAfxLBM=,tag:BtgIWfkLSoZDcbJML9uO7g==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/secrets/seed-encrypt-on-host.sh b/infra/deploy/secrets/seed-encrypt-on-host.sh index 36769c1..45eaf59 100755 --- a/infra/deploy/secrets/seed-encrypt-on-host.sh +++ b/infra/deploy/secrets/seed-encrypt-on-host.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Seed deploy/secrets/.yaml by encrypting a box's live /etc/thermograph.env +# Seed deploy/secrets/.yaml by encrypting an environment's live env file # ENTIRELY ON THE BOX. The plaintext never leaves the box; only the already-encrypted # ciphertext is written back here. Encryption uses the age PUBLIC key (safe to hardcode) # so the machine you run this from needs nothing but SSH access + the agent key — no @@ -7,7 +7,13 @@ # # Run from any checkout of this branch that can SSH to the target with the agent key: # deploy/secrets/seed-encrypt-on-host.sh prod agent@169.58.46.181 -# deploy/secrets/seed-encrypt-on-host.sh beta agent@75.119.132.91 +# deploy/secrets/seed-encrypt-on-host.sh beta agent@169.58.46.181 +# deploy/secrets/seed-encrypt-on-host.sh dev agent@75.119.132.91 +# +# prod and beta share a box (vps2); they differ only by which env FILE is read +# (/etc/thermograph.env vs /etc/thermograph-beta.env), resolved from +# deploy/env-topology.sh. Hardcoding that path meant "seed beta" would encrypt +# PROD's live secrets into beta.yaml. # Then commit + push the resulting deploy/secrets/.yaml. Verify faithfulness with # the key-gaps skill after. See README.md. set -euo pipefail @@ -17,14 +23,18 @@ ENV_NAME="${1:?usage: seed-encrypt-on-host.sh [ssh-key]}" SSH_TARGET="${2:?ssh target, e.g. agent@169.58.46.181}" SSH_KEY="${3:-$HOME/.ssh/thermograph_agent_ed25519}" OUT="deploy/secrets/${ENV_NAME}.yaml" +# shellcheck source=infra/deploy/env-topology.sh +. deploy/env-topology.sh +thermograph_topology "$ENV_NAME" +REMOTE_ENV_FILE="$TG_ENV_FILE" # Public age recipient — NOT a secret (matches .sops.yaml). The private key never # leaves the operator's machine / the hosts' /etc/thermograph/age.key. AGE_PUB="age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2" SOPS_URL="https://github.com/getsops/sops/releases/download/v3.13.2/sops-v3.13.2.linux.amd64" -echo "==> Encrypting ${SSH_TARGET}:/etc/thermograph.env on-host -> ${OUT}" -ssh -i "$SSH_KEY" "$SSH_TARGET" "AGE_PUB='$AGE_PUB' SOPS_URL='$SOPS_URL' bash -s" > "$OUT" <<'REMOTE' +echo "==> Encrypting ${SSH_TARGET}:${REMOTE_ENV_FILE} on-host -> ${OUT}" +ssh -i "$SSH_KEY" "$SSH_TARGET" "AGE_PUB='$AGE_PUB' SOPS_URL='$SOPS_URL' ENV_FILE='$REMOTE_ENV_FILE' bash -s" > "$OUT" <<'REMOTE' set -euo pipefail exec 3>&1 # real stdout carries ONLY the ciphertext { # setup noise -> stderr, so it can't corrupt the file @@ -34,7 +44,9 @@ exec 3>&1 # real stdout carries ONLY the ciphertext fi } >&2 tmp="$(mktemp)"; tmy="$(mktemp)"; trap 'rm -f "$tmp" "$tmy"' EXIT -sudo cat /etc/thermograph.env > "$tmp" # plaintext stays on this box only +# $ENV_FILE comes from the ssh command line above (env-topology.sh resolved it +# for the named environment) — NOT hardcoded, because vps2 holds two of them. +sudo cat "$ENV_FILE" > "$tmp" # plaintext stays on this box only python3 -c ' import json, sys for line in open(sys.argv[1], encoding="utf-8"): diff --git a/infra/deploy/secrets/seed-from-live.sh b/infra/deploy/secrets/seed-from-live.sh index 365a636..d220c19 100755 --- a/infra/deploy/secrets/seed-from-live.sh +++ b/infra/deploy/secrets/seed-from-live.sh @@ -1,11 +1,19 @@ #!/usr/bin/env bash -# Seed (or re-seed) an encrypted secrets file from a box's live /etc/thermograph.env. +# Seed (or re-seed) an encrypted secrets file from an environment's live env file. # Run on the operator's machine — it reads production secrets, so it is deliberately # NOT something the agent runs for you. Requires: sops + age on PATH, the age private # key at ~/.config/sops/age/keys.txt (or SOPS_AGE_KEY_FILE), and SSH access to the box. # # deploy/secrets/seed-from-live.sh prod agent@169.58.46.181 ~/.ssh/thermograph_agent_ed25519 -# deploy/secrets/seed-from-live.sh beta agent@75.119.132.91 ~/.ssh/thermograph_agent_ed25519 +# deploy/secrets/seed-from-live.sh beta agent@169.58.46.181 ~/.ssh/thermograph_agent_ed25519 +# deploy/secrets/seed-from-live.sh dev agent@75.119.132.91 ~/.ssh/thermograph_agent_ed25519 +# +# Note prod and beta share a box (vps2) and therefore differ only by which env +# FILE is read: /etc/thermograph.env vs /etc/thermograph-beta.env. That path is +# resolved from deploy/env-topology.sh rather than hardcoded — hardcoding it +# meant "seed beta" would read PROD's live secrets and write them into +# beta.yaml, which is how one environment silently ends up holding another's +# credentials. # # Writes deploy/secrets/.yaml ENCRYPTED (exact copy of the live env, so the # deploy render is faithful), then verifies the render round-trips to the same @@ -21,13 +29,17 @@ ENV_NAME="${1:?usage: seed-from-live.sh [ssh-key]}" SSH_TARGET="${2:?ssh target, e.g. agent@169.58.46.181}" SSH_KEY="${3:-$HOME/.ssh/thermograph_agent_ed25519}" OUT="deploy/secrets/${ENV_NAME}.yaml" +# shellcheck source=infra/deploy/env-topology.sh +. deploy/env-topology.sh +thermograph_topology "$ENV_NAME" +REMOTE_ENV_FILE="$TG_ENV_FILE" command -v sops >/dev/null || { echo "!! sops not on PATH" >&2; exit 1; } tmp="$(mktemp)"; trap 'rm -f "$tmp" "$tmp.env"' EXIT umask 077 -echo "==> Pulling ${SSH_TARGET}:/etc/thermograph.env (sudo)" -ssh -i "$SSH_KEY" "$SSH_TARGET" 'sudo cat /etc/thermograph.env' > "$tmp" +echo "==> Pulling ${SSH_TARGET}:${REMOTE_ENV_FILE} (sudo)" +ssh -i "$SSH_KEY" "$SSH_TARGET" "sudo cat $REMOTE_ENV_FILE" > "$tmp" n=$(grep -cE '^[A-Za-z_][A-Za-z0-9_]*=' "$tmp" || true) [ "$n" -gt 0 ] || { echo "!! no KEY=VALUE lines read (permission? path?)" >&2; exit 1; } echo " ${n} vars" @@ -47,7 +59,7 @@ for line in open(src, encoding="utf-8"): continue pairs[k] = v with open(dst, "w", encoding="utf-8") as fh: - fh.write(f"# SOPS-encrypted — {env} host secrets, seeded from the live /etc/thermograph.env.\n") + fh.write(f"# SOPS-encrypted — {env} host secrets, seeded from that environment’s live env file.\n") for k, v in pairs.items(): fh.write(f"{k}: {json.dumps(v)}\n") # JSON-escaped scalar is valid YAML print(f" {len(pairs)} keys") diff --git a/infra/deploy/stack/deploy-stack.sh b/infra/deploy/stack/deploy-stack.sh index c3badc2..f53c149 100755 --- a/infra/deploy/stack/deploy-stack.sh +++ b/infra/deploy/stack/deploy-stack.sh @@ -18,8 +18,20 @@ # rehearsal on the same host that cannot touch live data or ports. set -euo pipefail -APP_DIR="${APP_DIR:-/opt/thermograph}" SERVICE="${SERVICE:-all}" + +# Two stacks now run on vps2 (prod and beta), so nothing here may assume "the" +# stack: the name, the stack file, the loopback ports, the env file, the +# database role and the service-name prefix all come from env-topology.sh, +# keyed by the environment deploy.sh already resolved and exported. +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../env-topology.sh" +ENV_NAME=$(thermograph_env_name) +[ -n "$ENV_NAME" ] || ENV_NAME=prod +thermograph_topology "$ENV_NAME" + +APP_DIR="${APP_DIR:-$TG_APP_DIR}" cd "$APP_DIR" case "$SERVICE" in @@ -27,10 +39,26 @@ case "$SERVICE" in *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac +# Which stack-file services this environment's roll targets. Prod's are bare +# (web, worker, ...); beta's carry a `beta-` prefix so its short DNS names never +# collide with prod's on the shared overlay — see env-topology.sh. +SVC_WEB="${TG_SVC_PREFIX}web" +SVC_WORKER="${TG_SVC_PREFIX}worker" +SVC_FRONTEND="${TG_SVC_PREFIX}frontend" +SVC_LAKE="${TG_SVC_PREFIX}lake" +SVC_DAEMON="${TG_SVC_PREFIX}daemon" + +STACK_FILE="$APP_DIR/infra/$TG_STACK_FILE" +ENV_FILE="$TG_ENV_FILE" +STACK_ENV_FILE="$TG_STACK_ENV_FILE" + if [ "${STACK_TEST:-0}" = "1" ]; then + # Test mode rehearses PROD's stack only; it is not a second-environment path. STACK_NAME="thermograph-test" LB_NAME="thermograph-test-lb" LB_HTTP_PORT=18137; LB_FE_PORT=18080 + DATA_NETWORK="thermograph-test_internal" + DB_SERVICE="thermograph-test_db" export PGDATA_VOLUME="thermograph-test_pgdata" export APPDATA_VOLUME="thermograph-test_appdata" export APPLOGS_VOLUME="thermograph-test_applogs" @@ -38,29 +66,42 @@ if [ "${STACK_TEST:-0}" = "1" ]; then docker volume create "$APPDATA_VOLUME" >/dev/null docker volume create "$APPLOGS_VOLUME" >/dev/null else - STACK_NAME="${STACK_NAME:-thermograph}" - LB_NAME="thermograph-lb" - LB_HTTP_PORT=8137; LB_FE_PORT=8080 + STACK_NAME="${STACK_NAME:-$TG_STACK_NAME}" + LB_NAME="$TG_LB_NAME" + LB_HTTP_PORT="$TG_LB_HTTP_PORT"; LB_FE_PORT="$TG_LB_FE_PORT" + # Where the database is. Prod owns it inside its own overlay; beta reaches + # the same service across that overlay, which its stack declares external. + DATA_NETWORK="$TG_DATA_NETWORK" + DB_SERVICE="$TG_DB_SERVICE" fi export STACK_NAME +echo "==> Environment: $ENV_NAME (stack $STACK_NAME on $TG_HOST, checkout $APP_DIR)" # --- secrets ------------------------------------------------------------------ # Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then # install the uid-10001-readable copy the tasks' env-entrypoint shim sources. # 10001 = the app images' `thermograph` user; the file is 0400 to that uid. +# +# Per-environment paths throughout: on vps2 prod uses /etc/thermograph.env + +# /etc/thermograph/stack.env while beta uses /etc/thermograph-beta.env + +# /etc/thermograph/beta-stack.env. Sharing either file would give beta prod's +# database credentials and vice versa. if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$APP_DIR/infra/deploy/render-secrets.sh" - render_thermograph_secrets "$APP_DIR/infra" + render_thermograph_secrets "$APP_DIR/infra" "$ENV_NAME" "$ENV_FILE" fi -# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it -# cannot exist at lint time, so don't ask shellcheck to follow it. +# $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at +# lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null -. /etc/thermograph.env 2>/dev/null || true +. "$ENV_FILE" 2>/dev/null || true set +a -sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \ - || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env +sudo install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" \ + || install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" +# The stack file bind-mounts this path into every task; export it so the +# `volumes:` interpolation resolves to the right environment's copy. +export STACK_ENV_FILE # --- image tags ----------------------------------------------------------------- # Same persisted-tags contract as deploy.sh: incoming env wins, the file @@ -90,7 +131,14 @@ FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}: # Hazard #7: the db image under an existing volume must never drift. Resolve # the digest-pinned ref from whatever is running (stack task or compose # container), falling back to the local latest-pg18's digest on first bring-up. -if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then +# +# Only the environment that OWNS the database needs this — i.e. the one whose +# own stack declares the db service. Beta's stack declares none (it uses prod's +# instance), so resolving an image pin there answers a question beta's stack +# file never asks. +OWNS_DB=0 +[ "$DB_SERVICE" = "${STACK_NAME}_db" ] && OWNS_DB=1 +if [ -z "${TIMESCALEDB_IMAGE:-}" ] && [ "$OWNS_DB" = 1 ]; then cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1) if [ -n "$cid" ]; then @@ -102,7 +150,11 @@ if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img" fi export TIMESCALEDB_IMAGE -echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" +if [ "$OWNS_DB" = 1 ]; then + echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" +else + echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE (db: shared $DB_SERVICE)" +fi # --- registry ------------------------------------------------------------------ if [ -n "${REGISTRY_TOKEN:-}" ]; then @@ -124,15 +176,22 @@ done # in the stack). Runs on the stack's overlay so `db` resolves. First-ever # deploy: the network doesn't exist yet — create it exactly as the stack will # (attachable overlay) so the name is adopted, then migrate, then deploy. -NET="${STACK_NAME}_internal" -# Never pre-create $NET: docker stack deploy must own it (a pre-existing -# unlabeled network makes it fail with "already exists"). On first deploy the -# migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE below). +# +# The migrate task joins the network the DATABASE is on, which for beta is +# prod's overlay rather than beta's own — and it connects as this environment's +# own role to this environment's own database (beta: thermograph_beta on both +# counts), so a beta migration can never touch prod's schema. +NET="$DATA_NETWORK" +MIGRATE_URL="postgresql+asyncpg://${TG_DB_USER}:${POSTGRES_PASSWORD}@db:5432/${TG_DB_NAME}" +# Never pre-create $NET when this stack owns it: docker stack deploy must own it +# (a pre-existing unlabeled network makes it fail with "already exists"). On +# first deploy the migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE +# below). Beta's data network is prod's and always already exists. if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then - if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then - echo "==> One-shot migrate ($BACKEND_IMAGE)" + if docker service inspect "$DB_SERVICE" >/dev/null 2>&1; then + echo "==> One-shot migrate ($BACKEND_IMAGE -> ${TG_DB_NAME})" docker run --rm --network "$NET" \ - -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ + -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate else echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below" @@ -141,38 +200,38 @@ fi # --- deploy ---------------------------------------------------------------------- FIRST_DEPLOY_MIGRATE=0 -docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 -if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then +docker service inspect "$DB_SERVICE" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 +if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_${SVC_WEB}" >/dev/null 2>&1; then echo "==> docker stack deploy ($STACK_NAME)" - docker stack deploy --with-registry-auth -c "$APP_DIR/infra/deploy/stack/thermograph-stack.yml" "$STACK_NAME" + docker stack deploy --with-registry-auth -c "$STACK_FILE" "$STACK_NAME" # First-ever deploy ran no migrate above (db didn't exist): wait for db, # migrate, then force web/worker to restart cleanly against the schema. if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then echo "==> Waiting for db, then first-boot migrate" for i in $(seq 1 60); do - cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) - [ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break + cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${DB_SERVICE}" | head -1) + [ -n "$cid" ] && docker exec "$cid" pg_isready -U "$TG_DB_USER" -d "$TG_DB_NAME" >/dev/null 2>&1 && break sleep 5 done docker run --rm --network "$NET" \ - -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ + -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate - docker service update --force --detach=false "${STACK_NAME}_web" - docker service update --force --detach=false "${STACK_NAME}_worker" + docker service update --force --detach=false "${STACK_NAME}_${SVC_WEB}" + docker service update --force --detach=false "${STACK_NAME}_${SVC_WORKER}" fi else case "$SERVICE" in backend) - echo "==> Rolling web + worker + lake to $BACKEND_IMAGE" - docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web" - docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker" + echo "==> Rolling $SVC_WEB + $SVC_WORKER + $SVC_LAKE to $BACKEND_IMAGE" + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WEB}" + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WORKER}" # lake and daemon ship in the same image; a stack file predating either # has no service yet — the next SERVICE=all stack deploy creates it, so # don't fail here. The daemon especially must roll with web: they share # the /internal/* contract, and a version skew between them is exactly # what pinning one BACKEND_IMAGE_TAG exists to prevent (seen live: the # first post-creation backend roll left the daemon a release behind). - for extra in lake daemon; do + for extra in "$SVC_LAKE" "$SVC_DAEMON"; do if docker service inspect "${STACK_NAME}_${extra}" >/dev/null 2>&1; then docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${extra}" else @@ -181,8 +240,8 @@ else done ;; frontend) - echo "==> Rolling frontend to $FRONTEND_IMAGE" - docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend" + echo "==> Rolling $SVC_FRONTEND to $FRONTEND_IMAGE" + docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_${SVC_FRONTEND}" ;; esac fi @@ -192,13 +251,20 @@ fi # 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated # only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME` # to force a refresh after editing lb/Caddyfile. +# +# Note the network: the LB proxies to THIS stack's own service VIPs, so it joins +# this stack's own overlay — which for beta is beta's `internal`, not the shared +# data network its tasks also sit on. And its config is per-environment, because +# beta's services are named beta-web/beta-frontend. +LB_NETWORK="${STACK_NAME}_internal" +LB_CONFIG="$APP_DIR/infra/${TG_LB_CONFIG:-deploy/stack/lb/Caddyfile}" if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then docker rm -f "$LB_NAME" >/dev/null 2>&1 || true echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)" docker run -d --name "$LB_NAME" --restart unless-stopped \ - --network "$NET" \ + --network "$LB_NETWORK" \ -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \ - -v "$APP_DIR/infra/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \ + -v "$LB_CONFIG:/etc/caddy/Caddyfile:ro" \ caddy:2-alpine >/dev/null fi @@ -233,8 +299,14 @@ docker images --format '{{.Repository}}:{{.Tag}}' \ # Post-deploy warm + IndexNow, via any web task (skip in test mode: no data, # and the warmer would burn upstream quota against an empty cache). -if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then - wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1) +# +# TG_POST_DEPLOY gates this per environment: prod does both, beta does neither +# (IndexNow from beta would ask search engines to index beta.thermograph.org, +# and the warm would spend the shared upstream archive quota on a rehearsal +# cache). See env-topology.sh. +if [ "${STACK_TEST:-0}" != "1" ] && [ "${TG_POST_DEPLOY:-1}" = 1 ] \ + && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then + wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_${SVC_WEB}" | head -1) if [ -n "$wcid" ]; then echo "==> Warming city archives (detached) + IndexNow" docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true diff --git a/infra/deploy/stack/lb/Caddyfile.beta b/infra/deploy/stack/lb/Caddyfile.beta new file mode 100644 index 0000000..756d21d --- /dev/null +++ b/infra/deploy/stack/lb/Caddyfile.beta @@ -0,0 +1,34 @@ +# Beta's loopback LB bridge config — the beta counterpart of ./Caddyfile. +# +# Two differences from prod's, both load-bearing: +# +# 1. It proxies to beta-web / beta-frontend, not web / frontend. Beta's Swarm +# services carry that prefix so their short DNS names cannot collide with +# prod's on the shared overlay (see thermograph-beta-stack.yml). +# 2. deploy-stack.sh publishes this container on 127.0.0.1:8237 and :8180, +# not 8137/8180 — prod's LB already owns 8137/8080 on this host. The +# LISTEN ports below stay 8137/8080: those are inside the container, and +# the host mapping is what differs. Keeping the container ports identical +# to prod's means the two configs differ only in the upstream names. +# +# The host Caddy on vps2 terminates TLS for beta.thermograph.org and proxies to +# 127.0.0.1:8237 / :8180 — see deploy/Caddyfile. + +{ + auto_https off + admin off +} + +:8137 { + reverse_proxy beta-web:8137 { + # Fail fast to the client if the VIP has no healthy task; Swarm's own + # task healthchecks handle ejecting dead replicas from the VIP. + lb_try_duration 5s + } +} + +:8080 { + reverse_proxy beta-frontend:8080 { + lb_try_duration 5s + } +} diff --git a/infra/deploy/stack/thermograph-beta-stack.yml b/infra/deploy/stack/thermograph-beta-stack.yml new file mode 100644 index 0000000..1d1aca2 --- /dev/null +++ b/infra/deploy/stack/thermograph-beta-stack.yml @@ -0,0 +1,278 @@ +# Docker Swarm stack for BETA, co-resident with prod on vps2. +# +# Deployed by the same deploy/stack/deploy-stack.sh as prod, which picks this +# file (and beta's ports, env file, LB and DB role) out of deploy/env-topology.sh +# when THERMOGRAPH_ENV=beta. Beta moved here from its own box so that a beta +# green light is evidence about prod: same orchestrator, same host kernel, same +# Postgres build, same Caddy, same mail path, same mesh position. +# +# --------------------------------------------------------------------------- +# WHY THIS IS A SEPARATE FILE AND NOT AN OVERLAY ON thermograph-stack.yml +# --------------------------------------------------------------------------- +# `docker stack deploy` accepts multiple -c files and MERGES them. Merging can +# add and override, but it cannot REMOVE a service — and the single most +# important fact about beta is a removal: it has no `db`. It uses prod's. An +# overlay would therefore still create a second Postgres, which is the exact +# thing this design exists to avoid. The same goes for the two autoscalers, +# which beta deliberately does not run. +# +# The cost is a file that must be kept roughly in step with prod's by hand. +# Keep them in step for anything that affects whether the APP works (env vars, +# entrypoints, healthchecks, the migrate contract). Do NOT keep them in step on +# scale, replicas or resource limits — those differ on purpose (below). +# +# --------------------------------------------------------------------------- +# THE THREE THINGS THAT MAKE CO-RESIDENCY SAFE +# --------------------------------------------------------------------------- +# 1. SERVICE NAMES ARE PREFIXED (beta-web, not web). Swarm registers a service's +# short name as a DNS alias on every network it joins. Beta's tasks share the +# `data` network with prod's, so two services both called `web` would make +# `web` ambiguous — prod's frontend could resolve a beta task, and vice +# versa. The prefix removes the collision without touching prod's stack file. +# +# 2. THE DATABASE IS SHARED, THE DATA IS NOT. Beta connects to prod's `db` +# service as the role `thermograph_beta`, to the database `thermograph_beta`. +# That role owns only its own database (see deploy/db/provision-env-db.sh), +# so a beta deploy running an unmerged branch — or a migration that goes +# wrong — cannot read or write production data. One server is a capacity +# decision, not a trust decision. +# +# 3. NOTHING ELSE IS SHARED BY ACCIDENT. Separate checkout (/opt/thermograph-beta), +# separate rendered env file (/etc/thermograph-beta.env), separate stack env +# (/etc/thermograph/beta-stack.env), separate volumes, separate loopback +# ports (8237/8180 — prod owns 8137/8080), separate LB container, separate +# deploy lock and image-tag file. Every one of those is derived in +# env-topology.sh rather than repeated here by hand. +# +# --------------------------------------------------------------------------- +# WHAT BETA DELIBERATELY DOES NOT DO +# --------------------------------------------------------------------------- +# - No autoscaling: fixed 1 replica per service. Beta exists to answer "does +# this code work", not "does it scale"; a second replica would only add a +# variable prod's rehearsal doesn't need, on a host prod is also using. +# - No IndexNow ping and no city-archive warm (deploy-stack.sh gates both on +# TG_POST_DEPLOY). Pinging IndexNow from beta asks Bing/DuckDuckGo/Yandex to +# index beta.thermograph.org; the warm spends the shared upstream archive +# quota to fill a cache only a rehearsal reads. +# - No real mail and no Discord gateway. Both are governed by beta's vault +# (THERMOGRAPH_MAIL_BACKEND=console, THERMOGRAPH_DISCORD_BOT=0) rather than +# pinned here, so the operator can opt in with `sops edit` if a release ever +# genuinely needs to rehearse them. Discord in particular allows ONE gateway +# connection per bot token — beta and prod must never both hold one. + +services: + beta-web: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + # Beta's OWN role and OWN database on the shared instance. `db` resolves + # across the external `data` network to prod's db service. + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph_beta:${POSTGRES_PASSWORD}@db:5432/thermograph_beta + THERMOGRAPH_BASE: / + PORT: 8137 + THERMOGRAPH_SERVICE_ROLE: backend + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://beta-frontend:8080 + WORKERS: ${BETA_WEB_WORKERS:-2} + THERMOGRAPH_DATA_DIR: /state + # Never the notifier/scheduler — that is beta-worker's job, exactly as in + # prod, so the two files stay honest about which process owns what. + THERMOGRAPH_ROLE: web + # Migrations run as the one-shot task in deploy-stack.sh. + RUN_MIGRATIONS: "0" + # Overlay tasks reach the HOST's Postfix via the docker_gwbridge gateway. + # Same host, same Postfix as prod — but see the mail note in the header: + # beta's vault selects the console backend, so nothing is actually sent. + THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + THERMOGRAPH_LAKE_URL: http://beta-lake:8141 + volumes: + - appdata:/state + - applogs:/app/logs + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + networks: + - internal + - data + deploy: + replicas: 1 + # vps2 is the Swarm manager and every volume here is local to it. The + # desktop is a worker on this mesh and must never be scheduled the app. + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_WEB_CPUS:-2}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + + beta-worker: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph_beta:${POSTGRES_PASSWORD}@db:5432/thermograph_beta + THERMOGRAPH_BASE: / + PORT: 8137 + THERMOGRAPH_SERVICE_ROLE: backend + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://beta-frontend:8080 + WORKERS: "1" + THERMOGRAPH_DATA_DIR: /state + THERMOGRAPH_ROLE: worker + # The advisory lock is taken in beta's OWN database, so it can never + # contend with prod's worker despite the shared server. + THERMOGRAPH_SINGLETON_PG: "1" + RUN_MIGRATIONS: "0" + THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + THERMOGRAPH_LAKE_URL: http://beta-lake:8141 + volumes: + - appdata:/state + - applogs:/app/logs + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + networks: + - internal + - data + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_WORKER_CPUS:-1}" + restart_policy: + condition: on-failure + + beta-lake: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_ROLE: lake + PORT: 8141 + THERMOGRAPH_SERVICE_ROLE: backend + WORKERS: "1" + THERMOGRAPH_LAKE_CACHE: /state/lake-cache + volumes: + # Beta's own cache volume. Deliberately not prod's: they are read caches + # of the same bucket, but sharing a volume across two stacks would couple + # their lifecycles for no gain. + - lakecache:/state + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + # No `data` network: the lake reads object storage, never Postgres. + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_LAKE_CPUS:-1}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + + beta-daemon: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + # Same reasoning as prod's daemon: NOT env-entrypoint.sh, because that shim + # execs the image's own entrypoint (Alembic + uvicorn) and migrations belong + # to the one-shot task. Source the host-rendered env and exec the binary. + # + # Note the ordering consequence, which is load-bearing here: this sources + # the env file AFTER the `environment:` block is applied, so a key present + # in /etc/thermograph/beta-stack.env WINS over one set below. That is why + # THERMOGRAPH_DISCORD_BOT=0 lives in beta's vault and not in this file — a + # value set here would be silently overridden if the vault ever set one. + entrypoint: + - /bin/bash + - -c + - 'set -a; [ -f /host/thermograph.env ] && . /host/thermograph.env; set +a; exec /usr/local/bin/thermograph-daemon' + environment: + THERMOGRAPH_API_BASE_INTERNAL: http://beta-web:8137 + volumes: + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + # The image HEALTHCHECK curls /healthz on ${PORT}; the daemon serves + # nothing, so without this override Swarm restarts it forever. + healthcheck: + disable: true + networks: + - internal + deploy: + # EXACTLY 1, and in beta's case the Discord gateway is off entirely + # (vault: THERMOGRAPH_DISCORD_BOT=0) because prod's daemon holds the only + # permitted gateway connection for that bot token. + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "0.5" + memory: 128m + restart_policy: + condition: on-failure + update_config: + order: stop-first + failure_action: rollback + + beta-frontend: + image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + # REQUIRED: overriding `entrypoint:` with no `command:` drops the image's + # CMD entirely, and env-entrypoint.sh's fallback (`exec uvicorn app:app`) + # does not exist in this Go image — the task would exit 127 every deploy. + command: ["/usr/local/bin/thermograph-frontend"] + environment: + THERMOGRAPH_BASE: / + PORT: 8080 + THERMOGRAPH_SERVICE_ROLE: frontend + THERMOGRAPH_API_BASE_INTERNAL: http://beta-web:8137 + volumes: + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_FRONTEND_CPUS:-1}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + +networks: + # Beta's own east-west network: beta-web <-> beta-frontend <-> beta-lake, and + # the loopback LB bridge joins it (hence attachable). Keeping this separate + # from prod's overlay means beta's ordinary traffic never touches it. + internal: + driver: overlay + attachable: true + + # Prod's overlay, joined ONLY to reach the shared `db` service. Declared + # external because prod's stack owns it: `docker stack deploy` of THIS file + # must never create, modify or (on `docker stack rm thermograph-beta`) remove + # the network prod's database is on. + # + # Consequence worth knowing before you tear anything down: `docker stack rm + # thermograph` would take this network with it and beta would lose its + # database link until prod is redeployed. + data: + external: true + name: thermograph_internal + +volumes: + # Beta's own, created by this stack under the thermograph-beta_ prefix. Unlike + # prod's (which are `external` because they were inherited from the compose + # era and hold live data), these can be recreated: beta's appdata is a parquet + # cache plus derived files, and its DATABASE — the part that matters — lives + # on the shared instance, not here. + appdata: {} + applogs: {} + lakecache: {} diff --git a/infra/deploy/swarm/README.md b/infra/deploy/swarm/README.md index 056d609..386cd0f 100644 --- a/infra/deploy/swarm/README.md +++ b/infra/deploy/swarm/README.md @@ -1,36 +1,41 @@ -# 3-node Docker Swarm (prod + beta + desktop), for hosting Forgejo +# 3-node Docker Swarm (vps2 + vps1 + desktop) -This Swarm cluster's only job is to run Forgejo (`deploy/forgejo/`) — it does -**not** orchestrate the Thermograph app itself, which stays on the -Terraform-managed `docker compose` deploys on prod/beta independently (see -`terraform/README.md`). Keeping those separate means nothing here can strand -or interfere with the app's already-working, single-writer Postgres/TimescaleDB -deploys. +This Swarm mesh's only **Swarm-scheduled** workload is Forgejo +(`deploy/forgejo/`), pinned to vps1. It does **not** orchestrate the +Thermograph app the same way — prod and beta each run as their own `docker +stack deploy` (`deploy/stack/thermograph-stack.yml` / +`thermograph-beta-stack.yml`) that happens to land on this same manager node +(vps2) because their volumes are local to it today. Keeping Forgejo's stack +and the app stacks conceptually separate means nothing here can strand or +interfere with the app's single-writer Postgres/TimescaleDB. This is the canonical topology from `thermograph-docs/runbooks/implementation-handoff.md` (Track B steps 2-3) — three nodes, -not two. An earlier revision of this doc/scripts covered just prod+beta; -the desktop (this LAN dev machine) joins too. +not two. The desktop (formerly the LAN dev machine) joins as a worker for flex +capacity and AI-model hosting; it hosts no Thermograph environment. **Nodes:** -- **manager** — prod, the new 48 GB / 12-core box (more headroom). -- **worker** — beta, the old VPS (`75.119.132.91`). -- **worker** — desktop, this LAN dev machine (also runs the Forgejo Actions - runner as a plain systemd service — see `deploy/forgejo/README.md` — not as - a Swarm-scheduled container). +- **manager** — vps2 (`169.58.46.181`), the box with headroom for prod's and + beta's Swarm stacks and their local volumes. +- **worker** — vps1 (`75.119.132.91`), pinned to run Forgejo + (`node.labels.role == forge`). Also runs Grafana/Loki/Alloy and the dev + environment, both outside this Swarm cluster (plain Docker/compose on the + same host's daemon, not Swarm-scheduled). +- **worker** — desktop (this machine), flex capacity plus AI-model hosting. No + Thermograph environment runs here. One manager, not more: Raft needs 3 nodes for real quorum-based HA, and this cluster only has 3 nodes total, so making even one more of them a manager would still fall short of real HA while adding split-brain risk. If the -manager (prod) goes down, the workers keep running whatever was already -scheduled on them (Forgejo, pinned to beta) but the cluster can't reschedule -anything until prod's back — acceptable for a small cluster whose only job is -CI/CD. +manager (vps2) goes down, the workers keep running whatever was already +scheduled on them (Forgejo, pinned to vps1) but the cluster can't reschedule +anything until vps2's back — acceptable for a small cluster whose only +Swarm-scheduled job is CI/CD. ## Order of operations -1. **Agent access first** (`deploy/provision-agent-access.sh`) on prod and - beta — everything below on those two boxes is run through that access. The +1. **Agent access first** (`deploy/provision-agent-access.sh`) on vps1 and + vps2 — everything below on those two boxes is run through that access. The desktop is wherever you're already working from; no separate access step needed there. 2. **WireGuard mesh** (`setup-wireguard.sh `) — run on @@ -38,19 +43,21 @@ CI/CD. the two-pass key-exchange dance (pubkeys aren't known until every node has run it once). Verify with `ping ` to each of the other two before continuing. -3. **Swarm init** (`init-swarm.sh `) on the manager (prod) only. +3. **Swarm init** (`init-swarm.sh `) on the manager (vps2) only. 4. **Swarm join** (`join-swarm.sh `) on **each** of the - two workers (beta, desktop) — same token for both. + two workers (vps1, desktop) — same token for both. 5. **Firewall lockdown** (`firewall-swarm.sh`) on **all three** nodes — closes 2377/7946/4789 to everything except the WireGuard interface. Do this *after* joining is confirmed working on all three, not before (locking the ports first would make the join itself fail). -6. **Label beta** (`label-forge-node.sh `) on the manager — - `docker node ls` shows each node's name/ID. Only beta gets `role=forge`; - the desktop and prod don't need a Swarm label for anything in this setup. +6. **Label vps1** (`label-forge-node.sh `) on the manager — + `docker node ls` shows each node's name/ID. Only vps1 gets `role=forge`; + the desktop and vps2 don't need a Swarm label for anything in this setup. 7. Deploy Forgejo: see `deploy/forgejo/README.md`. -8. Register the Actions runner **on the desktop** (not through Swarm): - `deploy/forgejo/register-lan-runner.sh`. +8. Register the Actions runner: `deploy/forgejo/register-lan-runner.sh`. See + that document (and `DEPLOY-DEV.md`) for where it actually runs today — its + own header comment predates the vps1/vps2 rename and still describes "the + desktop" as the canonical placement. ## Why WireGuard instead of relying on Swarm's built-in TLS alone @@ -65,19 +72,19 @@ than trusting the public internet (or the desktop's home network) directly. ## Verifying ```bash -# On the manager: +# On the manager (vps2): docker node ls # all three nodes Ready -docker node inspect --format '{{.Spec.Labels}}' # role:forge +docker node inspect --format '{{.Spec.Labels}}' # role:forge # From a FOURTH machine outside the mesh entirely, confirm the Swarm ports # are NOT reachable on either VPS's public IP (the desktop has no public IP # to check this way): -nc -zv -w2 2377 # should fail/timeout -nc -zvu -w2 4789 # should fail/timeout +nc -zv -w2 2377 # should fail/timeout +nc -zvu -w2 4789 # should fail/timeout ``` ## Adding a node label back out (undo) ```bash -docker node update --label-rm role +docker node update --label-rm role ``` diff --git a/infra/deploy/swarm/firewall-swarm.sh b/infra/deploy/swarm/firewall-swarm.sh index 0e0ee65..d5eb74f 100755 --- a/infra/deploy/swarm/firewall-swarm.sh +++ b/infra/deploy/swarm/firewall-swarm.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Locks the Swarm ports (2377 control, 7946 gossip, 4789 overlay VXLAN) to the # WireGuard interface only — they must never be reachable from the public -# internet. Run on BOTH boxes after joining the swarm. Existing app-facing -# rules (80/443, SSH, etc.) are untouched. +# internet. Run on ALL THREE nodes (vps2, vps1, desktop) after joining the +# swarm. Existing app-facing rules (80/443, SSH, etc.) are untouched. set -euo pipefail WG_IFACE="${WG_IFACE:-wg0}" @@ -34,8 +34,10 @@ if ! ufw status | grep -q "^Status: active"; then echo "ufw is currently INACTIVE on this node — 'ufw enable' switches its" echo "default policy to deny-incoming for EVERYTHING, not just the Swarm" echo "ports above. Before enabling, explicitly allow every port this node" - echo "already serves publicly (SSH at minimum; on beta specifically, also" - echo "80/tcp and 443/tcp for the live thermograph.org Caddy) — check" - echo "'ss -tlnp' for what's actually listening first. Enabling ufw without" - echo "doing this WILL drop live traffic the moment it activates." + echo "already serves publicly (SSH at minimum; on vps2 specifically, also" + echo "80/tcp and 443/tcp for the live thermograph.org/beta.thermograph.org" + echo "Caddy; on vps1, 80/tcp and 443/tcp for git.thermograph.org," + echo "dashboard.thermograph.org and emigriffith.dev) — check 'ss -tlnp' for" + echo "what's actually listening first. Enabling ufw without doing this WILL" + echo "drop live traffic the moment it activates." fi diff --git a/infra/deploy/swarm/init-swarm.sh b/infra/deploy/swarm/init-swarm.sh index 4d41861..2501678 100755 --- a/infra/deploy/swarm/init-swarm.sh +++ b/infra/deploy/swarm/init-swarm.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash -# Run ONCE, on the manager node only (prod — the new 48 GB box). Initializes -# the Swarm advertising the WireGuard address, so cluster traffic never -# touches the public interface. Run setup-wireguard.sh on all THREE nodes -# first (prod, beta, and the desktop — see +# Run ONCE, on the manager node only (vps2 — the box with headroom for prod's +# and beta's Swarm stacks and their local volumes). Initializes the Swarm +# advertising the WireGuard address, so cluster traffic never touches the +# public interface. Run setup-wireguard.sh on all THREE nodes first (vps2, +# vps1, and the desktop — see # docs/runbooks/implementation-handoff.md Track B steps 2-3). set -euo pipefail @@ -21,7 +22,7 @@ echo "==> docker swarm init, advertising ${MY_WG_IP} (the WireGuard address, not docker swarm init --advertise-addr "$MY_WG_IP" --listen-addr "${MY_WG_IP}:2377" echo -echo "==> Worker join command (run this on EACH of the two workers — beta and" +echo "==> Worker join command (run this on EACH of the two workers — vps1 and" echo " the desktop; the same token works for both):" TOKEN="$(docker swarm join-token -q worker)" echo " docker swarm join --token ${TOKEN} ${MY_WG_IP}:2377" diff --git a/infra/deploy/swarm/join-swarm.sh b/infra/deploy/swarm/join-swarm.sh index 2baf94e..5177028 100755 --- a/infra/deploy/swarm/join-swarm.sh +++ b/infra/deploy/swarm/join-swarm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run ONCE on EACH worker node (beta, and the desktop — not the manager, -# prod). Joins the Swarm initialized by init-swarm.sh, over the WireGuard +# Run ONCE on EACH worker node (vps1, and the desktop — not the manager, +# vps2). Joins the Swarm initialized by init-swarm.sh, over the WireGuard # tunnel. The join token is the same for both workers. set -euo pipefail diff --git a/infra/deploy/swarm/label-forge-node.sh b/infra/deploy/swarm/label-forge-node.sh index cf07fcf..1533820 100755 --- a/infra/deploy/swarm/label-forge-node.sh +++ b/infra/deploy/swarm/label-forge-node.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run ONCE, on the manager node, after both boxes have joined the swarm. -# Labels the worker (beta) so the Forgejo stack's placement constraint +# Run ONCE, on the manager node (vps2), after both boxes have joined the +# swarm. Labels the worker (vps1) so the Forgejo stack's placement constraint # (node.labels.role == forge) schedules there and nowhere else. set -euo pipefail diff --git a/infra/deploy/swarm/setup-wireguard.sh b/infra/deploy/swarm/setup-wireguard.sh index f81525c..29f2d4c 100755 --- a/infra/deploy/swarm/setup-wireguard.sh +++ b/infra/deploy/swarm/setup-wireguard.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Sets up this node's side of a full-mesh WireGuard tunnel across all of -# prod, beta, and the desktop (three nodes, not two — see +# vps2, vps1, and the desktop (three nodes, not two — see # docs/runbooks/implementation-handoff.md Track B step 2). Docker Swarm's # control plane (2377/tcp) is TLS-encrypted by default, but the overlay data # plane (VXLAN, 4789/udp) is NOT — and it should never face the public @@ -14,8 +14,8 @@ # --- Peer list format -------------------------------------------------- # A text file, one line per OTHER node (not including the one you're running # on), each line: -# 10.10.0.1 169.58.46.181 -# 10.10.0.2 75.119.132.91 +# 10.10.0.1 169.58.46.181 +# 10.10.0.2 75.119.132.91 # 10.10.0.3 # # First pass on any node: peer pubkeys you don't have yet are "-". Run this diff --git a/infra/deploy/thermograph.env.example b/infra/deploy/thermograph.env.example index 5c423fd..f35537f 100644 --- a/infra/deploy/thermograph.env.example +++ b/infra/deploy/thermograph.env.example @@ -46,8 +46,9 @@ POSTGRES_PASSWORD=change-me # an empty value is honored as-is and would break the fallback to the public API. #THERMOGRAPH_ARCHIVE_URL=http://open-meteo-api:8080/v1/archive -# Mark the session cookie Secure — required behind Caddy's HTTPS. Set to 1 in prod; -# leave unset only for plain-HTTP LAN dev (a Secure cookie is never sent over HTTP). +# Mark the session cookie Secure — required behind Caddy's HTTPS. Set to 1 for +# prod and beta (both TLS-fronted); leave unset only for dev's plain-HTTP +# mesh-only URL (a Secure cookie is never sent over HTTP). THERMOGRAPH_COOKIE_SECURE=1 # Pin these too (see their own sections below), so container restarts don't rotate @@ -58,11 +59,14 @@ THERMOGRAPH_COOKIE_SECURE=1 # Number of uvicorn worker processes. More than 1 stops a single slow upstream fetch # (e.g. a cache-miss weather lookup) from blocking every other request — the cause of -# past brief outages. Prod runs 4 (the compose app service also defaults to WORKERS=4); -# leave unset (defaults to 1) on a small box. Workers elect one leader for the -# subscription notifier via a lockfile (THERMOGRAPH_SINGLETON_LOCK, set by the compose -# app service to /app/data/notifier.lock) so its timer-driven upstream sweep runs once, -# not once per worker. ~200 MB RAM per worker. +# past brief outages. Prod's Swarm `web` service defaults to 4 (WEB_WORKERS, also the +# autoscaler's per-replica count); beta's defaults to 2 (BETA_WEB_WORKERS) — a +# rehearsal environment, not a decision that beta needs less headroom per se. The +# base compose file (dev) also defaults to 4. Leave unset (defaults to 1) on a small +# box. Workers elect one leader for the subscription notifier via a lockfile +# (THERMOGRAPH_SINGLETON_LOCK, set by the compose/stack app service to +# /app/data/notifier.lock) so its timer-driven upstream sweep runs once, not once per +# worker. ~200 MB RAM per worker. WORKERS=4 # THERMOGRAPH_SINGLETON_LOCK arbitrates workers on ONE host. Under multi-host Swarm, diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index 86f9c53..cfebe28 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -13,11 +13,21 @@ # by deploy-dev.sh. This overlay must NOT reintroduce `build:`; it only relaxes # resource caps and LAN-exposes a port, same as the monorepo overlay did. # -# Differences from the prod stack (unchanged intent from the monorepo overlay): -# 1. backend is published on ALL interfaces (0.0.0.0:8137), not loopback, so -# phones and other devices on the Wi-Fi can reach the dev server directly -- -# dev has no Caddy in front (prod does, which is why the base file binds -# 127.0.0.1 only). +# Differences from the prod stack: +# 1. backend is published on ${DEV_BIND_ADDR}, defaulting to LOOPBACK. +# +# This used to be a flat 0.0.0.0:8137, from when dev ran on the operator's +# LAN box and the point was for phones on the Wi-Fi to reach it. Dev now +# runs on vps1, a public VPS, where 0.0.0.0 would publish whatever +# unreviewed branch is in flight to the entire internet — with no Caddy, no +# TLS and no auth in front of it. +# +# So the default is 127.0.0.1 (safe anywhere, including a laptop running +# `make dev-up`), and deploy.sh sets DEV_BIND_ADDR to the environment's mesh +# address from deploy/env-topology.sh — 10.10.0.2 on vps1. Dev is then +# reachable at http://10.10.0.2:8137 from anything on the WireGuard mesh and +# from nowhere else. There is deliberately no dev DNS record and no Caddy +# site block for it. # 2. frontend's port publish is dropped entirely -- dev has no Caddy to reach it # directly, so it stays compose-internal-only, reached solely through # backend's own reverse-proxy fallback (THERMOGRAPH_FRONTEND_BASE_INTERNAL, @@ -27,18 +37,21 @@ # prod's backend=4 / frontend=2 / db=2 allocation. `!reset` drops the base # value (both the top-level `cpus:` and the Swarm-style `deploy.resources` # block the base file carries for parity). -# 4. backend/daemon/lake get a second env_file entry pointing at a copy of the -# render under $APP_DIR (deploy-dev.sh sets THERMOGRAPH_SECRETS_ENV_FILE_MIRROR -# to produce it). This box's Docker is the snap package, confined to $HOME -- -# it can't see /etc/thermograph.env at all (not a permissions error, it just -# doesn't exist as far as snap-confined Docker is concerned), so the base -# file's env_file entry silently loads nothing here. compose appends env_file -# lists across overlays (last-wins on duplicate keys), so this is additive: -# prod/beta never set the mirror var, so they only ever get the base entry. +# 4. backend/daemon/lake get a second env_file entry pointing at an OPTIONAL +# copy of the render under $APP_DIR (`required: false`, so it is a no-op +# when absent). It exists for snap-packaged Docker, which is confined to +# $HOME and cannot see /etc/thermograph.env at all — not a permissions +# error; the file simply does not exist as far as snap-confined Docker is +# concerned, so the base file's env_file entry silently loads nothing. +# deploy-dev.sh now produces the mirror ONLY when it detects snap Docker, +# which vps1 (ordinary apt Docker) is not — there, dev reads +# /etc/thermograph.env like beta and prod do, and no plaintext copy of the +# render is written into the checkout. compose appends env_file lists across +# overlays (last-wins on duplicate keys), so this stays additive. services: backend: ports: !override - - "8137:8137" + - "${DEV_BIND_ADDR:-127.0.0.1}:8137:8137" cpus: !reset null deploy: !reset null env_file: diff --git a/infra/ops/ICEBERG-HANDOFF.md b/infra/ops/ICEBERG-HANDOFF.md index d27810b..9b015cc 100644 --- a/infra/ops/ICEBERG-HANDOFF.md +++ b/infra/ops/ICEBERG-HANDOFF.md @@ -5,6 +5,17 @@ already shipped (`infra/ops/dbq.sh`) — **mirror it**. This document is the contract, the environment facts you'll trip over, and how the result gets verified. +**Historical note:** `infra/ops/iceberg.sh` now exists, originally built +against the environment facts below as they stood before the vps1/vps2 split +— `beta` at `75.119.132.91` and `prod` at `169.58.46.181`, each its own box. +`iceberg.sh` (and `dbq.sh`) have since been updated to resolve their SSH +target and host facts from `deploy/env-topology.sh` instead of a hardcoded +table, so they already reflect the current split (vps1 = `75.119.132.91` = +dev + Forgejo + Grafana; vps2 = `169.58.46.181` = prod AND beta). This +document is left as a historical record of the original design contract, not +a live description of where each environment's engine runs — see +`infra/ops/README.md` for the current behavior. + ## Definition of done `infra/ops/iceberg.sh` exists and behaves like `dbq.sh`: @@ -36,6 +47,9 @@ new network exposure and no interactive steps. ## Environment facts you need +**As originally written** (pre vps1/vps2 split; kept for history — see the +note at the top of this document): + - Monorepo `emi/thermograph` (domain dirs: `backend/ frontend/ infra/ observability/`). Ops tooling lives in `infra/ops/`. - Environments: **dev** = LAN compose stack on the dev machine; **beta** = @@ -53,6 +67,24 @@ new network exposure and no interactive steps. services. Untested; verify before relying on it. - WireGuard mesh: prod `10.10.0.1`, beta `10.10.0.2`, dev `10.10.0.3`. - `rclone` and `age` are already installed on prod and beta. + +**Current facts, post vps1/vps2 split** (the boxes and mesh IPs above did not +move — only which environment runs where, and what the boxes are called): + +- **vps2** = `169.58.46.181` = mesh `10.10.0.1` = the box called "prod" above. + Runs **both** prod and beta now, as two separate Docker Swarm stacks — beta + is a Swarm service too today, not a compose exception, so the tunnelling + constraint above applies equally to it: `ssh -L` is impossible for beta now + as well, not just prod. +- **vps1** = `75.119.132.91` = mesh `10.10.0.2` = the box called "beta" above. + Runs Forgejo, Grafana/Loki/Alloy, and **dev** (its own Postgres container, + reached over SSH like any fleet host — not the LAN-box-local case the + original facts describe). +- **desktop** = mesh `10.10.0.3` = "the dev machine" above. Hosts no + Thermograph environment any more; it's a Swarm worker for flex capacity and + AI-model hosting. The "dev machine is a Swarm worker in the prod cluster" + fact still holds structurally (the desktop is still a worker node), but + nothing about Iceberg or the app database routes through it any more. - Reference implementation to copy: **`infra/ops/dbq.sh`** + `infra/ops/README.md`. ## Storage — almost certainly your warehouse diff --git a/infra/ops/README.md b/infra/ops/README.md index f7aeb7d..0c19dd0 100644 --- a/infra/ops/README.md +++ b/infra/ops/README.md @@ -11,34 +11,56 @@ infra/ops/dbq.sh beta -c '\dt' echo "select 1" | infra/ops/dbq.sh prod -f - ``` -Environments: `dev` (LAN compose stack on the dev machine), `beta` -(75.119.132.91), `prod` (169.58.46.181). Extra args pass straight through to +Environments: `dev` (own compose stack + own Postgres container, on vps1, +`75.119.132.91`), `beta` and `prod` (both on vps2, `169.58.46.181` — separate +roles/databases on the ONE shared TimescaleDB instance there). Host, SSH +target and container filter all come from `deploy/env-topology.sh` now, never +a hardcoded table — `dbq.sh` SSHes to vps1 for `dev` and to vps2 for +`beta`/`prod`, and beta (which has no db container of its own) resolves +prod's shared `thermograph_db` task. Extra args pass straight through to `psql` (`-tA`, `-x`, `--csv`, `-f`, …); stdin is forwarded. +**Known gap:** beta has no read-only role of its own yet — `deploy/db/provision-env-db.sh` +only provisions the environment's *owning* role (`thermograph_beta`), so a +`dbq.sh beta` query connects as `thermograph_beta` (which can write) rather +than a read-only role, unlike `dev`/`prod`. See `dbq.sh`'s own header for the +tracked follow-up (a `thermograph_beta_ro` role). + ### Why it execs into the container instead of using a connection string None of the databases are exposed over TCP — each listens only on its private -docker network. Prod's is a Swarm **overlay** (`10.0.2.0/24`) that the host -cannot route to, so `ssh -L` works for beta but is *impossible* for prod; -publishing 5432 would mean new ufw rules plus a Swarm endpoint change on -production. Running `psql` inside the db container works identically in all -three environments with no ports, no tunnels, and no infra changes. +docker network. Prod's and beta's is a Swarm **overlay** (`thermograph_internal`) +that the vps2 host itself cannot route to, so `ssh -L` is impossible for +either of them — treat beta exactly like prod here, now that it's a Swarm +service too, not the compose-network exception it used to be. Publishing 5432 +would mean new ufw rules plus a Swarm endpoint change on a live environment. +Running `psql` inside the db container works identically in all three +environments with no ports, no tunnels, and no infra changes. The prod container name is a Swarm task name that changes on every redeploy, so it is resolved at call time via `docker ps --filter name=…`, never hardcoded. ### Safety -Queries connect as **`thermograph_ro`** — a `NOSUPERUSER` role granted only -`pg_read_all_data`. Read-only is enforced by Postgres, not by convention: +`dev` and `prod` connect as **`thermograph_ro`** — a `NOSUPERUSER` role granted +only `pg_read_all_data`. Read-only is enforced by Postgres, not by convention: ``` $ infra/ops/dbq.sh prod -c "create table t(i int)" ERROR: permission denied for schema public ``` +**`beta` is the exception, today:** `deploy/db/provision-env-db.sh` only ever +provisions the environment's *owning* role (`thermograph_beta` — +`NOSUPERUSER`/`NOCREATEDB`/`NOCREATEROLE`, but it owns its own database and +schema) and revokes `CONNECT` from `PUBLIC` on `thermograph_beta`, so +`thermograph_ro` was never granted `CONNECT` there. A `dbq.sh beta` query +therefore connects as `thermograph_beta` until a beta-scoped read-only role +exists (tracked in `dbq.sh`'s own header as a TODO) — it can write to beta's +database, though never to prod's. + The app's own `thermograph` role is a superuser and is deliberately **not** used -by this tool. If the role is ever missing (fresh database), recreate it with: +by this tool. If `thermograph_ro` is ever missing (fresh instance), recreate it with: ```sql CREATE ROLE thermograph_ro LOGIN; @@ -68,15 +90,18 @@ The lake is **one shared warehouse** in Contabo object storage same credentials — so unlike Postgres there is no per-env database to reach, and the environments differ only in *where the engine runs*: a throwaway `docker run` of a small DuckDB image (duckdb CLI + `httpfs`/`iceberg` -extensions pre-installed) on the target box — locally for LAN dev, over SSH -for beta/prod. The image is built on the host the first time it's needed from -a Dockerfile embedded in the script; its tag is a hash of that Dockerfile, so -editing the script rolls every host forward on the next call. +extensions pre-installed) on the target box — vps1 for `dev`, vps2 for +`beta`/`prod`, resolved from `deploy/env-topology.sh` rather than a hardcoded +table (beta and prod are two Swarm stacks on the same vps2 box now). The +image is built on the host the first time it's needed from a Dockerfile +embedded in the script; its tag is a hash of that Dockerfile, so editing the +script rolls every host forward on the next call. -A container per query means no daemon, no listening port, no tunnel (prod's -overlay cannot be tunnelled), no firewall or Swarm changes — `sudo ss -ltnp` -is identical before and after a query. It also never execs into an app -container and resolves no container names, so prod redeploys can't break it. +A container per query means no daemon, no listening port, no tunnel (beta's +and prod's Swarm overlay cannot be tunnelled), no firewall or Swarm changes — +`sudo ss -ltnp` is identical before and after a query. It also never execs +into an app container and resolves no container names, so a redeploy can't +break it. There is no catalog service to run or expose: an Iceberg table's current state is fully described by the newest metadata JSON under diff --git a/infra/ops/dbq.sh b/infra/ops/dbq.sh index c2c0cac..0837c4f 100755 --- a/infra/ops/dbq.sh +++ b/infra/ops/dbq.sh @@ -7,26 +7,43 @@ # echo "select 1" | infra/ops/dbq.sh prod -f - # # Why exec-into-the-container instead of a connection string: -# none of the databases are exposed over TCP. Each listens only on its private -# docker network -- and prod's is a Swarm *overlay* (10.0.2.0/24) that the host -# itself cannot route to, so `ssh -L` works for beta but is impossible for prod. -# Publishing 5432 would mean new firewall + Swarm endpoint changes on production. -# Running psql *inside* the db container works identically in all three -# environments with no ports, no tunnels and no infra changes -- locally for LAN -# dev, over SSH for beta/prod. +# none of the databases are exposed over TCP. Dev's listens only on its own +# private docker network on vps1; prod and beta share ONE instance on prod's +# Swarm *overlay* (10.0.2.0/24) on vps2, which the host itself cannot route +# to, so `ssh -L` is impossible for either. Publishing 5432 would mean new +# firewall + Swarm endpoint changes on production. Running psql *inside* the +# db container works identically in all three environments with no ports, no +# tunnels and no infra changes -- over SSH to vps1 for dev, to vps2 for +# beta/prod. # -# Safety: always connects as `thermograph_ro`, a NOSUPERUSER role granted only -# pg_read_all_data. Read-only is enforced by Postgres itself, not by convention, -# so a stray INSERT/DDL fails with "permission denied" even against prod. (The -# app's own `thermograph` role is a superuser -- deliberately not used here.) +# Host, SSH target and container filter all come from env-topology.sh, never +# a hardcoded table -- beta and prod are two Swarm stacks on the SAME box +# (vps2) now, and beta has no db container of its own: it reaches prod's +# `thermograph_db` task, the one shared TimescaleDB instance. +# +# Safety: every environment connects as a READ-ONLY role, never as the role the +# app itself uses -- a stray INSERT/DDL fails with "permission denied" even +# against prod. The role is the environment's own owning role with an `_ro` +# suffix, which resolves to the long-standing `thermograph_ro` for prod and dev +# and to `thermograph_beta_ro` for beta. +# +# That suffix convention is why beta needed no special case. It would have been +# easy to give beta one: `CONNECT` is revoked from `PUBLIC` on +# `thermograph_beta`, so prod's `thermograph_ro` cannot reach it, and the +# tempting shortcut was to let beta queries connect as `thermograph_beta` -- +# the role that OWNS beta's database. That would have quietly made ad-hoc +# queries read-write on beta alone. deploy/db/provision-env-db.sh provisions the +# `_ro` role for every environment instead. # # Any extra arguments are passed straight through to psql, so -tA, -c, -f, -x, # --csv etc. all work. Stdin is forwarded, so `-f -` reads piped SQL. set -euo pipefail +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../deploy/env-topology.sh" + KEYFILE="${THERMOGRAPH_AGENT_KEY:-$HOME/.ssh/thermograph_agent_ed25519}" -DB_USER="${THERMOGRAPH_DB_QUERY_USER:-thermograph_ro}" -DB_NAME="${THERMOGRAPH_DB_NAME:-thermograph}" usage() { sed -n '2,9p' "$0" | sed 's/^# \{0,1\}//' >&2 @@ -39,13 +56,33 @@ env_name="${1:-}" shift case "$env_name" in - dev) filter=thermograph-dev-db ; ssh_target= ;; - beta) filter=thermograph-db-1 ; ssh_target=agent@75.119.132.91 ;; - prod) filter=thermograph_db ; ssh_target=agent@169.58.46.181 ;; + dev|beta|prod) ;; -h|--help) usage ;; *) echo "dbq: unknown environment '$env_name' (want: dev|beta|prod)" >&2; exit 2 ;; esac +thermograph_topology "$env_name" + +ssh_target="$TG_SSH_TARGET" + +# Container filter: a Swarm task name (shared across prod and beta -- beta has +# no db task of its own) or a compose container name, per the environment's +# deploy mode. +if [ "$TG_DEPLOY_MODE" = stack ]; then + filter="$TG_DB_SERVICE" +else + filter="${TG_COMPOSE_PROJECT}-${TG_DB_SERVICE}" +fi + +DB_NAME="${THERMOGRAPH_DB_NAME:-$TG_DB_NAME}" +if [ -n "${THERMOGRAPH_DB_QUERY_USER:-}" ]; then + DB_USER="$THERMOGRAPH_DB_QUERY_USER" +else + # thermograph_ro for prod/dev, thermograph_beta_ro for beta -- provisioned by + # deploy/db/provision-env-db.sh. Never the owning role. + DB_USER="${TG_DB_USER}_ro" +fi + [ $# -gt 0 ] || usage # Quote the psql arguments so they survive the remote shell intact. diff --git a/infra/ops/iceberg.sh b/infra/ops/iceberg.sh index 693e54d..cfabf75 100755 --- a/infra/ops/iceberg.sh +++ b/infra/ops/iceberg.sh @@ -9,9 +9,12 @@ # Why an ephemeral container instead of a query service: the lake is one shared # warehouse in Contabo object storage (s3://era5-thermograph/iceberg), readable # from every environment, so the environments differ only in WHERE the engine -# runs -- a throwaway `docker run` of a small DuckDB image on the target box. -# No daemon, no listening port, no tunnel (prod's overlay cannot be tunnelled), -# and no container names to resolve, so prod redeploys cannot break it. +# runs -- a throwaway `docker run` of a small DuckDB image on the target box +# (vps1 for dev, vps2 for beta/prod -- the SSH target comes from +# env-topology.sh, never a hardcoded table; beta and prod are two Swarm stacks +# on the SAME vps2 box now). No daemon, no listening port, no tunnel (prod's +# overlay cannot be tunnelled), and no container names to resolve, so prod +# redeploys cannot break it. # # There is no catalog service: a table's current state is its newest metadata # JSON under //metadata/, resolved at call time and exposed @@ -34,6 +37,10 @@ # read as piped SQL. A single bare argument is treated as `-c` SQL. set -euo pipefail +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../deploy/env-topology.sh" + KEYFILE="${THERMOGRAPH_AGENT_KEY:-$HOME/.ssh/thermograph_agent_ed25519}" usage() { @@ -47,13 +54,17 @@ env_name="${1:-}" shift case "$env_name" in - dev) ssh_target= ;; - beta) ssh_target=agent@75.119.132.91 ;; - prod) ssh_target=agent@169.58.46.181 ;; + dev|beta|prod) ;; -h|--help) usage ;; *) echo "iceberg: unknown environment '$env_name' (want: dev|beta|prod)" >&2; exit 2 ;; esac +# SSH target comes from env-topology.sh, not a hardcoded table: beta and prod +# are two Swarm stacks on the SAME box (vps2) now, so a stale beta entry here +# would silently point a beta query at the wrong host. +thermograph_topology "$env_name" +ssh_target="$TG_SSH_TARGET" + [ $# -gt 0 ] || usage # The engine image: duckdb CLI with the httpfs + iceberg extensions installed diff --git a/infra/terraform/README.md b/infra/terraform/README.md index 77a4df5..47717e5 100644 --- a/infra/terraform/README.md +++ b/infra/terraform/README.md @@ -10,27 +10,46 @@ scaffold" below) — scaffold-only today, no live resources until you opt in. ## What it manages -One reusable module (`modules/thermograph-host`) is instantiated per host via -`for_each` (`main.tf`'s `local.all_hosts`, merging `var.hosts` — SSH-managed, -already-existing boxes — with any `var.gcp_hosts` Terraform created itself). -This config manages **two VPS hosts** today: +One reusable module (`modules/thermograph-host`) is instantiated per +**(host, environment)** pair via `for_each` (`main.tf`'s `local.all_hosts`, +merging `var.hosts` — SSH-managed, already-existing boxes — with any +`var.gcp_hosts` Terraform created itself). `var.hosts` is keyed by HOST, not by +environment, because **vps2 alone runs two environments** (prod and beta) as +separate Docker Swarm stacks on the same box — see the `hosts` variable's file +header in `variables.tf` for why that stopped being "one entry = one +environment" and what still needs to change in `main.tf` to consume the new +shape (flagged there as a `TODO(cutover)`). This config manages **two VPS +hosts**, three environments, today: -| key | role | VPS | branch | domain | notes | -|--------|--------|-------------------------|-----------|-------------------|------------------------------------| -| `prod` | prod | NEW 48 GB / 12-core box (`169.58.46.181`) | `release` (of the APP repos — see `backend_image_tag` / `frontend_image_tag`) | `thermograph.org` | Caddy TLS; sized up (8/8/4/16g) | -| `beta` | beta | old box `75.119.132.91` | `main` (of the APP repo) | `beta.thermograph.org` | Caddy TLS; also hosts Forgejo | +| host | environment | VPS | branch | domain | notes | +|--------|-------------|--------------------------------------------|--------|------------------------|----------------------------------------------| +| `vps1` | `dev` | `75.119.132.91` | `dev` (of the APP repos — see `backend_image_tag` / `frontend_image_tag`) | none (mesh-only) | Also hosts Forgejo, Grafana/Loki/Alloy, emigriffith.dev — not Terraform-managed | +| `vps2` | `prod` | 48 GB / 12-core box (`169.58.46.181`) | `main` | `thermograph.org` | Caddy TLS (this module owns `/etc/caddy/Caddyfile`); sized up (8/8/4/16g) | +| `vps2` | `beta` | the SAME box as prod (`169.58.46.181`) | `main` | none (see below) | Separate Swarm stack + checkout, shares prod's TimescaleDB instance on a separate database/role | -Each host's own checkout on disk (`app_dir`, `git_branch`) is **this infra -repo**, not the app repo — the "branch" column above is which app-repo tag a -host is meant to track conceptually; the actual pinned versions are -`var.hosts[*].backend_image_tag` / `frontend_image_tag` (e.g. `"sha-<12 hex>"` each — -the app is two separately-published images, `emi/thermograph-backend/app` and -`emi/thermograph-frontend/app`), since the host has no app checkout to derive a tag -from. The LAN dev server is **out of scope here** — it -builds from source via the app repo's `deploy/deploy-dev.sh` (a self-hosted -Forgejo Actions runner), not Terraform. +Only ONE environment per host may set a `domain`: the module installs a full +`/etc/caddy/Caddyfile`, and a second `terraform apply` with its own `domain` set +would silently overwrite the first's site instead of adding to it (there is no +merge). Prod claims that slot on vps2 today, so beta's public reverse-proxy (if +and when `beta.thermograph.org` is exposed) has to be a site block in vps2's +shared, hand-maintained Caddy config instead — the same pattern +`deploy/forgejo/docker-stack.yml` already uses to put `git.thermograph.org` in +front of Forgejo's loopback port on vps1, alongside that box's own Caddy-fronted +`emigriffith.dev`. -Per host, over SSH provisioners, Terraform: +Each environment's own checkout on disk (`app_dir`, `git_branch`) is **this +infra repo**, not the app repo — the "branch" column above is which app-repo +tag an environment is meant to track conceptually; the actual pinned versions +are `var.hosts[*].environments[*].backend_image_tag` / `frontend_image_tag` +(e.g. `"sha-<12 hex>"` each — the app is two separately-published images, +`emi/thermograph-backend/app` and `emi/thermograph-frontend/app`), since the +host has no app checkout to derive a tag from. `app_dir` is required with no +default specifically so vps2's two environments can never collide on the same +checkout path (prod: `/opt/thermograph`, beta: `/opt/thermograph-beta`). The +LAN-laptop dev rehearsal (`make dev-up`) is **out of scope here** — it's a +local-only compose overlay, not this Terraform-managed `dev` environment. + +Per (host, environment), over SSH provisioners, Terraform: - installs Docker + the compose plugin if missing; - configures a `ufw` firewall (22/80/443 always; on a host with **no** domain it also @@ -104,7 +123,11 @@ terraform plan terraform apply ``` -Target one host with `-target='module.host["beta"]'` if you want to apply to just one. +Target one host with `-target='module.host["prod"]'` if you want to apply to just one +(the module's `for_each` key changes once `main.tf` is updated to flatten `var.hosts`'s +new per-host `environments` map — see the `TODO(cutover)` on the `hosts` variable in +`variables.tf` — at which point the key for beta becomes something like +`module.host["vps2-beta"]`, not `module.host["beta"]`). ## Secrets @@ -150,11 +173,15 @@ Terraform brings the stack up — don't let Terraform recreate containers mid-mi ## Assumptions / notes -- **beta has no public domain by default.** With `compose_files = ["docker-compose.yml"]` - the app binds `127.0.0.1:8137` (loopback), so opening the port in `ufw` alone does not - expose it. Reach beta via an SSH tunnel, or set `domain = "beta.thermograph.org"` (adds - Caddy TLS) — or add the `0.0.0.0`-publishing dev overlay to `compose_files` — to make - it reachable. `COOKIE_SECURE` is auto-set to `0` when there's no domain (a Secure +- **beta has no public domain here, and can't without a different Caddy strategy.** + With `compose_files = ["docker-compose.yml"]` the app binds `127.0.0.1:8137` + (loopback), so opening the port in `ufw` alone does not expose it — and setting + `domain` on beta the way prod does would make this module overwrite the SAME + `/etc/caddy/Caddyfile` prod's apply just installed (see the table above). Reach + beta via an SSH tunnel, or front it with a site block in vps2's own + hand-maintained Caddy config (outside Terraform), the same way Forgejo's + `git.thermograph.org` reaches Forgejo's loopback port on vps1. + `COOKIE_SECURE` is auto-set to `0` when there's no domain (a Secure cookie is never sent over plain HTTP) and `1` behind Caddy TLS. - The rendered Caddyfile only reverse-proxies the app. The repo's `deploy/Caddyfile` additionally serves the `emigriffith.dev` portfolio and legacy redirects; those are diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf index b5a2171..68e2ac2 100644 --- a/infra/terraform/main.tf +++ b/infra/terraform/main.tf @@ -44,13 +44,39 @@ module "gcp_vm" { } locals { - # Every host this config manages, SSH-only (var.hosts) plus GCP-created (whose - # `host` is filled in from the VM Terraform just created) — one unified map so a - # single `module.host` for_each below handles both without duplicating any - # provisioning logic. GCP hosts always use a named size tier (simpler than + # ONE MODULE INSTANCE PER (HOST, ENVIRONMENT) PAIR, not per host. + # + # var.hosts is keyed by machine (vps1, vps2) and each machine carries an + # `environments` map, because vps2 runs prod AND beta. The module below + # provisions an ENVIRONMENT — a checkout, a branch, image tags, a Caddy site, + # container sizing — so it needs one instance per environment, with the two + # instances on vps2 sharing that machine's SSH identity. + # + # Flattened to keys like "vps2-prod" and "vps2-beta". Keeping them distinct + # module instances is what makes `app_dir` (required, no default in + # variables.tf) do its job: two environments on one box get two explicitly + # different checkouts, so neither apply can `git reset --hard` the other's + # tree. + ssh_environments = merge([ + for hname, h in var.hosts : { + for ename, e in h.environments : + "${hname}-${ename}" => merge(e, { + host = h.host + ssh_user = h.ssh_user + ssh_private_key_path = h.ssh_private_key_path + }) + } + ]...) + + # Every environment this config manages, SSH-only (flattened above) plus + # GCP-created (whose `host` is filled in from the VM Terraform just created) — + # one unified map so a single `module.host` for_each below handles both without + # duplicating any provisioning logic. GCP hosts stay one-entry-per-machine: + # nothing creates two environments on a created VM today, and the variable + # keeps its flat shape. They always use a named size tier (simpler than # exposing the four raw sizing fields on that variable too). all_hosts = merge( - var.hosts, + local.ssh_environments, { for name, h in var.gcp_hosts : name => merge(h, { host = module.gcp_vm[name].external_ip diff --git a/infra/terraform/modules/thermograph-host/variables.tf b/infra/terraform/modules/thermograph-host/variables.tf index c41f85b..ceac2a6 100644 --- a/infra/terraform/modules/thermograph-host/variables.tf +++ b/infra/terraform/modules/thermograph-host/variables.tf @@ -21,7 +21,7 @@ variable "ssh_private_key_path" { } variable "role" { - description = "\"prod\" | \"dev\" — informational." + description = "\"prod\" | \"beta\" | \"dev\" — informational. One module instance is one (host, environment) pair, so vps2 (which runs both prod and beta) gets two instances, each with its own role/app_dir/image tags -- see the root module's `hosts` variable." type = string } diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf index 2336b73..0a5cba8 100644 --- a/infra/terraform/outputs.tf +++ b/infra/terraform/outputs.tf @@ -4,7 +4,7 @@ output "prod_url" { } output "hosts" { - description = "Per-host summary: the address managed and its role." + description = "Per-environment summary, keyed \"-\" (vps2 carries two): the machine managed and the role running on it." value = { for name, mod in module.host : name => { host = mod.host diff --git a/infra/terraform/terraform.tfvars.example b/infra/terraform/terraform.tfvars.example index a2ea7ed..7b80ef5 100644 --- a/infra/terraform/terraform.tfvars.example +++ b/infra/terraform/terraform.tfvars.example @@ -1,8 +1,8 @@ -# Copy to terraform.tfvars and fill in real IPs + credentials. +# Copy to terraform.tfvars and fill in real credentials. # cp terraform.tfvars.example terraform.tfvars # terraform.tfvars is gitignored (repo_url may carry a credential, and om_rclone_conf # always does — both land in local state). NEVER commit real values. -# + # App secrets (POSTGRES_PASSWORD, THERMOGRAPH_AUTH_SECRET, VAPID keys, # REGISTRY_TOKEN, Discord/SMTP creds, ...) are NOT here anymore -- they live in # the SOPS+age vault (../deploy/secrets/*.yaml), rendered at deploy time. See @@ -11,54 +11,99 @@ # --------------------------------------------------------------------------------- # Hosts # --------------------------------------------------------------------------------- -# Two VPS hosts. (The `dev` branch deploys to the LAN dev server via the app repo's -# deploy/deploy-dev.sh — that box is NOT managed by Terraform.) +# Two VPS boxes, keyed by HOST (vps1, vps2) — NOT by environment, because vps2 +# alone carries two (prod AND beta). Each host lists its SSH identity once, then +# an `environments` map for whatever runs on it. (The `dev` branch deploys to +# vps1 the same way in reality, but the LAN-laptop `make dev-up` rehearsal is NOT +# managed by Terraform at all.) hosts = { - # Production: the NEW 48 GB / 12-core VPS serving thermograph.org. - prod = { - host = "REPLACE_WITH_NEW_VPS_IP" # <-- the new prod VPS IP/hostname - ssh_user = "agent" - ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" - role = "prod" - git_branch = "main" # this INFRA repo's branch -- see *_image_tag for the app versions - backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" # e.g. "sha-abcdef012345" -- from thermograph-backend build-push.yml - frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" # e.g. "sha-012345abcdef" -- from thermograph-frontend build-push.yml - domain = "thermograph.org" # Caddy TLS in front, app on loopback - compose_files = ["docker-compose.yml"] - app_dir = "/opt/thermograph" - # "large" is the named size tier for this box (locals.sizes in main.tf) — same - # numbers as hand-picking workers=8/app_cpus=8/db_cpus=4/db_memory="16g" below, - # via the shortcut. The Postgres internal budget scales from db_memory - # automatically (deploy/db/init/20-tuning.sh); no separate tuning edit. - size = "large" - # Self-host the ERA5 archive here: layers docker-compose.openmeteo.yml and - # provisions the rclone mount of the object-storage bucket (om_* vars below). - openmeteo = true - om_data_dir = "/mnt/om-archive" - } - - # Beta / testing: the OLD VPS, repurposed. - beta = { + # vps1: Forgejo (git+CI+registry), Grafana/Loki/Alloy, emigriffith.dev, and the + # `dev` environment. One environment today; the shape still nests it so a + # second one (there is none planned) would never have to fight this host's + # SSH identity. + vps1 = { host = "75.119.132.91" ssh_user = "agent" ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" - role = "beta" - git_branch = "main" - backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" - frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" - # No public domain by default: no Caddy/TLS, firewall opens the app port. NOTE: - # with compose_files = ["docker-compose.yml"] the app binds 127.0.0.1 only, so - # until you either set a domain (e.g. "beta.thermograph.org", which fronts it with - # Caddy) or add the 0.0.0.0-publishing dev overlay, reach it via an SSH tunnel. - domain = "" - compose_files = ["docker-compose.yml"] - app_dir = "/opt/thermograph" - # Explicit numbers, not a size tier — both styles work on any host; a tier is - # purely an opt-in shortcut (see prod's `size = "large"` above). - workers = 4 - app_cpus = 4 - db_cpus = 2 - db_memory = "8g" + + environments = { + dev = { + role = "dev" + git_branch = "dev" # the only environment that tracks `dev`; beta/prod track `main` + backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" + frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" + # Mesh-only: no public domain, no Caddy/TLS. Reachable at + # http://10.10.0.2:8137 from the WireGuard mesh only. + domain = "" + compose_files = ["docker-compose.yml"] + app_dir = "/opt/thermograph-dev" + workers = 4 + app_cpus = 4 + db_cpus = 2 + db_memory = "8g" + } + } + } + + # vps2: prod (thermograph.org) AND beta (beta.thermograph.org) as two separate + # Swarm stacks on the SAME 48 GB / 12-core box — plus Centralis, Postfix and + # the backups (not Terraform-managed). ONE shared TimescaleDB instance serves + # both app_dirs below, on separate databases/roles (deploy/db/provision-env-db.sh); + # each environment's db_cpus/db_memory here sizes only that environment's own + # app-container caps, not a second database. + vps2 = { + host = "169.58.46.181" + ssh_user = "agent" + ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" + + environments = { + prod = { + role = "prod" + git_branch = "main" # this INFRA repo's branch -- see *_image_tag for the app versions + backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" # e.g. "sha-abcdef012345" -- from thermograph-backend build-push.yml + frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" # e.g. "sha-012345abcdef" -- from thermograph-frontend build-push.yml + domain = "thermograph.org" # Caddy TLS in front, app on loopback + compose_files = ["docker-compose.yml"] + # MUST differ from beta's app_dir below -- see variables.tf's file header. + app_dir = "/opt/thermograph" + # "large" is the named size tier for this box (locals.sizes in main.tf) — same + # numbers as hand-picking workers=8/app_cpus=8/db_cpus=4/db_memory="16g" below, + # via the shortcut. The Postgres internal budget scales from db_memory + # automatically (deploy/db/init/20-tuning.sh); no separate tuning edit. + size = "large" + # Self-host the ERA5 archive here: layers docker-compose.openmeteo.yml and + # provisions the rclone mount of the object-storage bucket (om_* vars below). + openmeteo = true + om_data_dir = "/mnt/om-archive" + } + + beta = { + role = "beta" + git_branch = "main" + backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" + frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" + # "" ON PURPOSE, unlike prod above: modules/thermograph-host installs a + # FULL /etc/caddy/Caddyfile per environment with a domain set, and only + # ONE environment on a box can own that file. Prod already claims it on + # this host, so beta.thermograph.org's public reverse-proxy has to be a + # site block in vps2's shared, hand-maintained Caddy instead (the same + # pattern deploy/forgejo/docker-stack.yml uses for git.thermograph.org + # on vps1) -- not something this module can render for a second + # environment on the same host. + domain = "" + compose_files = ["docker-compose.yml"] + # MUST differ from prod's app_dir above -- a SECOND checkout on the same + # box, so a `git reset --hard` in one deploy can never yank the tree out + # from under the other's running deploy. + app_dir = "/opt/thermograph-beta" + # Explicit numbers, not a size tier — both styles work on any environment; a + # tier is purely an opt-in shortcut (see prod's `size = "large"` above). + workers = 4 + app_cpus = 4 + db_cpus = 2 + db_memory = "8g" + } + } } # UAT: an ephemeral, single-node environment — same images/topology shape as @@ -68,20 +113,27 @@ hosts = { # host = "REPLACE_WITH_UAT_VM_IP" # ssh_user = "agent" # ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" - # role = "uat" - # git_branch = "main" - # backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" - # frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" - # domain = "" - # compose_files = ["docker-compose.yml"] - # app_dir = "/opt/thermograph" - # size = "nano" + # environments = { + # uat = { + # role = "uat" + # git_branch = "main" + # backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" + # frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" + # domain = "" + # compose_files = ["docker-compose.yml"] + # app_dir = "/opt/thermograph" + # size = "nano" + # } + # } # } } -# GCP-created hosts — SCAFFOLD ONLY, empty by default. Populate an entry to have -# Terraform actually create a GCP Compute Engine VM (see modules/gcp-host); until -# then no google_* resource is planned and no GCP credentials are needed. Example: +# GCP-created hosts — SCAFFOLD ONLY, empty by default, and still keyed one entry +# per ENVIRONMENT (see variables.tf's TODO(cutover) note on gcp_hosts — these are +# hypothetical single-purpose boxes, not vps1/vps2, so they don't need the +# host/environment nesting above). Populate an entry to have Terraform actually +# create a GCP Compute Engine VM (see modules/gcp-host); until then no google_* +# resource is planned and no GCP credentials are needed. Example: # gcp_hosts = { # gcp-uat = { # project = "REPLACE_WITH_GCP_PROJECT_ID" @@ -108,7 +160,7 @@ hosts = { # repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/emi/thermograph-infra.git" # --------------------------------------------------------------------------------- -# Self-hosted Open-Meteo archive (only used by hosts with openmeteo = true) -------- +# Self-hosted Open-Meteo archive (only used by environments with openmeteo = true) # --------------------------------------------------------------------------------- # The ERA5 .om archive lives in an object-storage bucket, rclone-mounted on the host. # om_rclone_conf holds bucket credentials (sensitive; lands in state — keep out of git). diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 8386d75..d6c187a 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -1,69 +1,109 @@ # --------------------------------------------------------------------------------- # Hosts # --------------------------------------------------------------------------------- -# One entry per VPS. The same module is instantiated for each (see main.tf's -# for_each). This config manages TWO hosts: prod (new 48 GB / 12-core VPS, branch -# `release`, thermograph.org) and beta (the old VPS 75.119.132.91, branch `main`, -# no public domain by default). The `dev` branch deploys to the LAN dev server via -# deploy/deploy-dev.sh (a self-hosted runner, in the app repo) and is NOT managed here. +# Keyed by HOST (vps1, vps2), NOT by environment: vps2 alone runs TWO environments +# (prod AND beta) as separate Swarm stacks on the SAME box, so "one entry = one +# environment" broke the moment that became true — two entries that both defaulted +# `app_dir` to "/opt/thermograph" would let a prod apply and a beta apply stomp the +# identical checkout. Each host now carries its SSH identity ONCE (host IP, +# ssh_user, key) plus an `environments` map of everything that varies per +# environment running on it — checkout path, git branch, image tags, domain, +# sizing. `app_dir` is REQUIRED with no default for exactly this reason: two +# environments on one host must be given two explicitly DIFFERENT paths (see +# terraform.tfvars.example), so their checkouts — and a `git reset --hard` in one +# of them — can never collide. # -# A host with `domain = ""` gets no Caddy/TLS: the app port is opened on the firewall -# and the app is reached directly (beta does this by default; set a domain to front -# it with Caddy TLS). +# vps1 (75.119.132.91): Forgejo (git+CI+registry), Grafana/Loki/Alloy, +# emigriffith.dev, and the `dev` environment (its own Postgres, mesh-only, no +# public DNS/Caddy site). vps2 (169.58.46.181): `prod` (thermograph.org) and +# `beta` (beta.thermograph.org) — Centralis, Postfix and the backups also live +# here — sharing the ONE TimescaleDB instance both app environments use, on +# separate databases/roles (deploy/db/provision-env-db.sh); this variable's +# per-environment db_cpus/db_memory size only that environment's app-container +# caps, not a second database (see deploy/db/init/20-tuning.sh). The LAN dev +# branch (`make dev-up`, a laptop-local rehearsal) deploys nowhere via Terraform +# and is NOT managed here. # -# Resource sizing (workers / app_cpus / db_cpus / db_memory) defaults to the historical -# 4 / 4 / 2 / 8g budget (right for beta). The prod box is 48 GB / 12 cores — raise these -# there (the example uses app_cpus 8, db_cpus 4, db_memory "16g"). The Postgres internal -# budget scales from db_memory automatically (deploy/db/init/20-tuning.sh), so no -# separate tuning edit is needed to exploit the RAM. +# An environment with `domain = ""` gets no Caddy/TLS: its app port is opened on +# the firewall and it's reached directly. +# +# Resource sizing (workers / app_cpus / db_cpus / db_memory), per environment, +# defaults to the historical 4 / 4 / 2 / 8g budget (right for beta). vps2 is 48 GB +# / 12 cores — raise these for prod (the example uses app_cpus 8, db_cpus 4, +# db_memory "16g", via the "large" size tier). The Postgres internal budget scales +# from db_memory automatically (deploy/db/init/20-tuning.sh), so no separate +# tuning edit is needed to exploit the RAM. +# +# TODO(cutover): main.tf's `module.host` for_each (via `local.all_hosts`) still +# expects ONE flat entry per key, reading `each.value.host`/`.role`/`.app_dir`/… +# directly. It needs a flattening step ahead of that for_each — one derived entry +# per (host, environment) pair, keyed e.g. "-" ("vps2-prod" and +# "vps2-beta" as two separate module instances sharing vps2's host/ssh_user/ +# ssh_private_key_path but each with its own app_dir/role/image tags/sizing). +# This variable's new shape is not yet wired into main.tf's module block. variable "hosts" { - description = "Map of hosts to manage, keyed by a short name (e.g. \"prod\", \"beta\")." + description = "Map of VPS boxes to manage, keyed by a short host name (\"vps1\", \"vps2\") — NOT by environment. SSH identity is per-host; everything environment-specific lives in that host's `environments` map." type = map(object({ host = string # IP or hostname to SSH to ssh_user = optional(string, "deploy") # SSH login user ssh_private_key_path = string # path to the private key for that user - role = string # "prod" | "beta" (informational + outputs) - git_branch = string # this INFRA repo's branch the checkout is reset to - # Which app images to run, e.g. "sha-<12 hex>" (each matching build-push.yml's tag - # for the commit that app repo built) or a semver tag on a release push. The app is - # TWO separately-published images now — emi/thermograph-backend/app and - # emi/thermograph-frontend/app — pinned independently. Required, no default: the - # host's checkout is this infra repo, not the app repos, so there's no "current - # commit" to derive a tag from; both must be explicit. Bump these (via a normal - # tfvars edit + apply) whenever an app repo ships a commit you want this host - # running; the infra repo's own git_branch is independent and rarely needs to change. - backend_image_tag = string - frontend_image_tag = string - domain = optional(string, "") # public domain; "" => no Caddy/TLS - compose_files = optional(list(string), ["docker-compose.yml"]) - app_dir = optional(string, "/opt/thermograph") # checkout path on the host - workers = optional(number, 4) # uvicorn workers (WORKERS) - app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) - db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) - db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) - # A named size tier (see locals.sizes in main.tf: nano/small/medium/large) — - # when set, overrides the four fields above with the tier's preset. Leave - # null (default) to keep hand-picking workers/app_cpus/db_cpus/db_memory, - # as prod/beta already do below. - size = optional(string, null) - # The floating tag matches today's behavior everywhere until you pin it. Pin to - # an exact minor (SELECT extversion FROM pg_extension WHERE extname='timescaledb' - # on the live DB) before any host of this stack could ever replicate with - # another — a floating tag risks mismatched extension minors, which blocks a - # physical replica (hop-1 cutover runbook hazard #7). Use the SAME tag everywhere. - timescaledb_tag = optional(string, "latest-pg18") - # Self-host the ERA5 archive (docker-compose.openmeteo.yml + a host rclone mount - # of the object-storage bucket). Only the self-hosting host (prod) sets true. - openmeteo = optional(bool, false) - om_data_dir = optional(string, "/mnt/om-archive") # host rclone mount point (OM_DATA_DIR) + # One entry per environment running on this host, keyed by environment name + # ("prod" | "beta" | "dev"). A host normally carries one; vps2 carries two. + environments = map(object({ + role = string # "prod" | "beta" | "dev" (informational + outputs; should match the map key above) + git_branch = string # this INFRA repo's branch the checkout is reset to + # Which app images to run, e.g. "sha-<12 hex>" (each matching build-push.yml's tag + # for the commit that app repo built) or a semver tag on a release push. The app is + # TWO separately-published images now — emi/thermograph-backend/app and + # emi/thermograph-frontend/app — pinned independently. Required, no default: the + # host's checkout is this infra repo, not the app repos, so there's no "current + # commit" to derive a tag from; both must be explicit. Bump these (via a normal + # tfvars edit + apply) whenever an app repo ships a commit you want this + # environment running; the infra repo's own git_branch is independent and rarely + # needs to change. + backend_image_tag = string + frontend_image_tag = string + domain = optional(string, "") # public domain for this environment; "" => no Caddy/TLS + compose_files = optional(list(string), ["docker-compose.yml"]) + # REQUIRED, no default — see the file header: two environments on one host + # must never be able to default to the same checkout path. + app_dir = string # checkout path on the host + workers = optional(number, 4) # uvicorn workers (WORKERS) + app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) + db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) + db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) + # A named size tier (see locals.sizes in main.tf: nano/small/medium/large) — + # when set, overrides the four fields above with the tier's preset. Leave + # null (default) to keep hand-picking workers/app_cpus/db_cpus/db_memory, + # as prod/beta already do below. + size = optional(string, null) + # The floating tag matches today's behavior everywhere until you pin it. Pin to + # an exact minor (SELECT extversion FROM pg_extension WHERE extname='timescaledb' + # on the live DB) before any host of this stack could ever replicate with + # another — a floating tag risks mismatched extension minors, which blocks a + # physical replica (hop-1 cutover runbook hazard #7). Use the SAME tag everywhere. + timescaledb_tag = optional(string, "latest-pg18") + # Self-host the ERA5 archive (docker-compose.openmeteo.yml + a host rclone mount + # of the object-storage bucket). Only the self-hosting environment (prod) sets true. + openmeteo = optional(bool, false) + om_data_dir = optional(string, "/mnt/om-archive") # host rclone mount point (OM_DATA_DIR) + })) })) } -# GCP-created hosts, keyed the same way as `hosts`. Default {} => zero GCP resources -# planned and the google provider is never actually invoked (see versions.tf and -# modules/gcp-host). Populate an entry to have Terraform create the VM itself; its -# output IP then feeds into the SAME thermograph-host module every SSH-managed host -# uses (main.tf), so provisioning logic is never duplicated between providers. +# GCP-created hosts, keyed the same way `hosts` used to be — one entry per +# environment, since these are hypothetical single-purpose boxes (e.g. a future +# UAT VM), not vps1/vps2, so they don't need the host/environment split above. +# Default {} => zero GCP resources planned and the google provider is never +# actually invoked (see versions.tf and modules/gcp-host). Populate an entry to +# have Terraform create the VM itself; its output IP then feeds into the SAME +# thermograph-host module every SSH-managed host uses (main.tf), so provisioning +# logic is never duplicated between providers. +# TODO(cutover): main.tf's `local.all_hosts = merge(var.hosts, ...)` currently +# merges this flat shape with `var.hosts` into one map for `module.host`'s +# for_each. Once `var.hosts` is nested (above), that merge needs to target the +# same flattened (host, environment) shape this variable already has — e.g. by +# treating each `gcp_hosts` entry as its own single-environment host. variable "gcp_hosts" { description = "Map of hosts for Terraform to CREATE on GCP (Compute Engine), keyed the same way as `hosts`. Empty by default -- no live GCP resources exist yet; this is a scaffold for future use. See modules/gcp-host." type = map(object({ diff --git a/observability/.env.example b/observability/.env.example index 73b9f35..ae61a4d 100644 --- a/observability/.env.example +++ b/observability/.env.example @@ -1,7 +1,10 @@ -# Copy to .env on the monitoring host (beta) before `docker compose up`. +# Copy to .env on the monitoring host (vps1) before `docker compose up`. # .env is gitignored — never commit real secrets. +# +# vps1 is NOT the beta environment — that runs on vps2 now, as its own Swarm +# stack. Don't confuse the two when reading older docs that say "beta" here. -# The public hostname beta's Caddy serves Grafana at (see caddy-grafana.conf). +# The public hostname vps1's Caddy serves Grafana at (see caddy-grafana.conf). GRAFANA_DOMAIN=dashboard.thermograph.org # Break-glass local admin (Google SSO below is the primary login). Keep this @@ -22,8 +25,8 @@ GOOGLE_CLIENT_SECRET= # --- Alerting (Discord) ---------------------------------------------------------- # Where every alert in grafana/provisioning/alerting/ is delivered: a webhook on # the private #ops-alerts channel (Owners category) of the Thermograph.org -# Discord. NOT email — beta's Grafana relays SMTP through prod's Postfix, so -# email alerts die exactly when prod does. +# Discord. NOT email — vps1's Grafana relays SMTP through vps2's Postfix, so +# email alerts die exactly when prod (on vps2) does. # # To mint one: Discord -> #ops-alerts -> Edit Channel -> Integrations -> # Webhooks -> New Webhook -> Copy Webhook URL. Treat it like a password; anyone diff --git a/observability/CLAUDE.md b/observability/CLAUDE.md index 1485024..4d6db89 100644 --- a/observability/CLAUDE.md +++ b/observability/CLAUDE.md @@ -1,28 +1,37 @@ # observability/ — agent instructions -The **logging stack** for the fleet: Loki + Grafana on beta, with a Grafana Alloy -agent on every node (prod, beta, dev) shipping container and app logs over the -WireGuard mesh. Grafana is fronted by beta's Caddy at -**`dashboard.thermograph.org`** (Google SSO, pre-provisioned users only) — use -that hostname everywhere, never `grafana.thermograph.org`. +The **logging stack** for the fleet: Loki + Grafana on **vps1** — the monitoring +host, mesh `10.10.0.2`, and NOT the beta *environment* (that runs on vps2 now; +any doc that still says "beta" meaning this box is stale) — with a Grafana +Alloy agent on every node shipping container and app logs over the WireGuard +mesh. vps1 also hosts Forgejo, the portfolio site, and the **dev** environment +(mesh-only, no public DNS). vps2 runs **prod and beta** as two Swarm stacks. +Grafana is fronted by vps1's Caddy at **`dashboard.thermograph.org`** (Google +SSO, pre-provisioned users only) — use that hostname everywhere, never +`grafana.thermograph.org`. This is operational config, not application code: there is **no build** and no -deploy automation. It ships by hand — `docker compose up -d` on beta for +deploy automation. It ships by hand — `docker compose up -d` on vps1 for Loki+Grafana, and `alloy/docker-compose.agent.yml` on each node. Read the root `CLAUDE.md` first; the branch model there applies here too. ## Layout -- `docker-compose.yml` — the Loki + Grafana stack (beta). +- `docker-compose.yml` — the Loki + Grafana stack (vps1). - `loki/config.yml` — mesh-only, filesystem storage. - `grafana/provisioning/` — datasource + dashboard provider, auto-loaded at startup. `grafana/dashboards/*.json` — the dashboards. - `grafana/provisioning/alerting/` — alert rules, the Discord contact point, and the notification policy. -- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` — the per-node shipper. - Node name comes from `ALLOY_NODE`, set per host. -- `caddy-grafana.conf` — the beta Caddy vhost, kept here for reference. -- `.env.example` — copy to `.env` on beta. `.env` is gitignored; never commit +- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` (+ + `alloy/docker-compose.agent.beta.yml`, a vps2-only overlay adding beta's + `applogs` mount) — the per-node shipper. `ALLOY_NODE` is the MACHINE + (`vps1`|`vps2`|`desktop`) — that's a semantic change from when the fleet had + one environment per node; `host` (`prod`|`beta`|`dev`), the ENVIRONMENT + label every dashboard/alert slices on, is now derived per source instead, + since vps2 alone carries two of them. +- `caddy-grafana.conf` — the vps1 Caddy vhost, kept here for reference. +- `.env.example` — copy to `.env` on vps1. `.env` is gitignored; never commit OAuth secrets or the admin password. ## Rules @@ -40,8 +49,8 @@ Loki+Grafana, and `alloy/docker-compose.agent.yml` on each node. Read the root literal Discord webhook URL in the repo is a hard failure. A rule pointing at a missing refId is valid YAML, provisions cleanly, and then never fires — that silent no-op is what this check exists to prevent. -- **Alerts go to Discord `#ops-alerts`, never email.** Beta's Grafana relays SMTP - through prod's Postfix, so email dies exactly when prod does. +- **Alerts go to Discord `#ops-alerts`, never email.** vps1's Grafana relays + SMTP through vps2's Postfix, so email dies exactly when prod (on vps2) does. - **Every rule is LogQL** — there is no Prometheus anywhere in the fleet. Thresholds were derived from real Loki data and the working is in the comments beside each rule; re-derive before changing a number rather than guessing. diff --git a/observability/README.md b/observability/README.md index d0f9af5..e2ac68c 100644 --- a/observability/README.md +++ b/observability/README.md @@ -7,41 +7,61 @@ over the private WireGuard mesh. This replaces the old in-repo persistent, queryable, fleet-wide log store and UI. ``` - prod (10.10.0.1) beta (10.10.0.2) desktop / LAN dev (10.10.0.3) - ┌────────────┐ ┌──────────────────┐ ┌────────────┐ - │ Alloy agent│──wg0──┐ │ Loki ◀── Alloy │ ┌──wg0│ Alloy agent│ - └────────────┘ └───▶│ Grafana (Caddy) │◀──┘ └────────────┘ - docker+caddy+app └──────────────────┘ docker+caddy+app - logs central store + UI logs + vps1 (10.10.0.2) vps2 (10.10.0.1) desktop (10.10.0.3) + ┌───────────────────────┐ ┌───────────────────────┐ ┌──────────────┐ + │ Loki ◀───── Alloy │◀──wg0───┤ Alloy agent │◀─wg0──┤ Alloy agent │ + │ Grafana (Caddy) │ │ prod + beta (Swarm) │ │ (no app │ + │ dev (compose) │◀──wg0───┤ Postfix, backups │ │ stack) │ + │ Forgejo, portfolio │ └───────────────────────┘ └──────────────┘ + └───────────────────────┘ docker+caddy+app logs ships no app logs + central store + UI + dev ``` +- **vps1** is the monitoring host: Loki, Grafana, Forgejo, the portfolio site, + and the **dev** environment (its own compose project, mesh-only, no public + DNS). It does **not** run the beta *environment* — that now lives on vps2 — so + any older doc that says "beta" meaning *this box* is stale and needs reading + as vps1. +- **vps2** runs **prod and beta** as two separate Docker Swarm stacks + (`thermograph`, `thermograph-beta`), plus Postfix and backups. One shared + TimescaleDB serves both, on separate roles/databases. +- **desktop** hosts no Thermograph environment now (AI-model hosting + flex + Swarm-worker capacity instead), so it ships no app logs. - **Loki** stores logs (filesystem, 30-day retention). Listens on the mesh IP `10.10.0.2:3100` — never public. -- **Grafana** is the UI, fronted by beta's Caddy at `dashboard.thermograph.org`, +- **Grafana** is the UI, fronted by vps1's Caddy at `dashboard.thermograph.org`, login via **Google SSO** (with a break-glass local admin); Loki is not exposed. - **Alloy** runs on each node and ships three sources to Loki: every Docker container's stdout/stderr, Caddy's host access logs, and the app's structured JSON logs (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase` - become labels). Every line is tagged with its node (`host = prod|beta|dev`). + become labels). Every line now carries two distinct labels: `host` + (`prod`|`beta`|`dev`) is the ENVIRONMENT, derived *per source* (a container's + stack prefix, the Caddy access-log filename, or which `applogs` volume a JSON + line came out of) — because vps2 alone carries two environments, and one + node-wide value would have stamped every beta container and log line as + `prod`. `node` (`vps1`|`vps2`|`desktop`) is the MACHINE, stamped externally on + everything an agent ships. See `alloy/config.alloy` for exactly how `host` is + derived per source. ## Layout | Path | What | |------|------| -| `docker-compose.yml` | the central Loki + Grafana stack (runs on beta) | +| `docker-compose.yml` | the central Loki + Grafana stack (runs on vps1) | | `loki/config.yml` | Loki config (filesystem, retention) | | `grafana/provisioning/` | auto-wired Loki datasource + dashboard provider | | `grafana/provisioning/alerting/` | contact point, notification policy and alert rules — see [Alerting](#alerting) | | `grafana/dashboards/thermograph-logs.json` | the fleet-logs dashboard | | `alloy/config.alloy` | the per-node collector config | | `alloy/docker-compose.agent.yml` | runs the Alloy agent on a node | -| `caddy-grafana.conf` | beta Caddy site block for the Grafana UI | +| `alloy/docker-compose.agent.beta.yml` | overlay adding beta's `applogs` mount — vps2 only | +| `caddy-grafana.conf` | vps1 Caddy site block for the Grafana UI | ## Deploy > ### ⚠️ Merging is not deploying > -> `/opt/observability` on beta (and prod) is a checkout of the **archived** +> `/opt/observability` on vps1 (and vps2) is a checkout of the **archived** > `emi/thermograph-observability` repo. It can never `git pull` again — the > content now lives here, in the mono repo, under `observability/`. A previous > observability PR merged and had **zero live effect** for exactly this reason. @@ -49,22 +69,23 @@ persistent, queryable, fleet-wide log store and UI. > Until someone re-points those checkouts at the mono repo, every change here > ships by hand: back up the live file with a UTC-timestamped suffix, `scp` the > new one up, `sudo cp` it into place, restart the specific container. Use -> `sudo docker restart ` — **not `docker compose`**, which on prod -> demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to interpolate from. -> (Beta's stack does have an `.env`, so compose works there — and is required +> `sudo docker restart ` — **not `docker compose`**, which on vps2 +> demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to interpolate from +> (vps2 only runs the Alloy agent — there's no Grafana there to configure). +> (vps1's stack does have an `.env`, so compose works there — and is required > when a change adds a new environment variable, since `docker restart` alone > will not pick one up.) -### 1. Central stack (on beta) +### 1. Central stack (on vps1) ```bash -# on beta (75.119.132.91): +# on vps1 (75.119.132.91): git clone thermograph-observability && cd thermograph-observability cp .env.example .env # set GF_SECURITY_ADMIN_PASSWORD docker compose up -d # Loki on 10.10.0.2:3100 + 127.0.0.1:3100; Grafana on 127.0.0.1:3000 ``` -Expose the Grafana UI: point `dashboard.thermograph.org` at beta, append +Expose the Grafana UI: point `dashboard.thermograph.org` at vps1, append `caddy-grafana.conf` to `/etc/caddy/Caddyfile`, `systemctl reload caddy`. ### Google SSO (Grafana + Forgejo) @@ -80,7 +101,7 @@ OAuth 2.0 Client (type: *Web application*) and register both redirect URIs: (`allow_sign_up=false`), so provision your email as an admin once: ```bash -# on beta, after the stack is up (uses the break-glass admin creds from .env): +# on vps1, after the stack is up (uses the break-glass admin creds from .env): curl -s -u admin:"$GF_SECURITY_ADMIN_PASSWORD" -H 'Content-Type: application/json' \ -X POST http://127.0.0.1:3000/api/admin/users \ -d '{"name":"you","login":"you@gmail.com","email":"you@gmail.com","password":"'"$(openssl rand -base64 24)"'"}' @@ -97,28 +118,50 @@ docker exec -u git forgejo admin auth add-oauth \ --auto-discover-url https://accounts.google.com/.well-known/openid-configuration ``` -### 2. Alloy agent (on every node — prod, beta, desktop) +### 2. Alloy agent (on vps1 and vps2) + +`ALLOY_NODE` names the MACHINE now, not the environment — vps2 alone runs two +environments (prod and beta), so a node name can no longer stand in for a +`host` value. `ALLOY_ENV` supplies the default environment label for anything +on that node not attributable to a specific stack (Forgejo, Grafana, the +portfolio site). `APPLOGS_VOLUME` is the node's primary `applogs` volume; vps2 +additionally needs `BETA_APPLOGS_VOLUME` and a second compose file, since it +carries both environments' structured logs side by side. + +Every node pushes to Loki over the mesh IP — a containerized agent can't reach +the host's `127.0.0.1`, but it can reach vps1's wg0 address at +`10.10.0.2:3100`. ```bash cd thermograph-observability/alloy -# Every node (beta included) pushes to Loki over the mesh IP — a containerized -# agent can't reach the host's 127.0.0.1, but it can reach beta's wg0 address: -# beta: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=beta -# prod: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=prod -# desktop: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=dev -ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + +# vps1 (dev + Forgejo + the monitoring stack itself): +ALLOY_NODE=vps1 ALLOY_ENV=dev APPLOGS_VOLUME=thermograph-dev_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ docker compose -f docker-compose.agent.yml up -d + +# vps2 (prod + beta) — note the SECOND file, adding beta's log volume: +ALLOY_NODE=vps2 ALLOY_ENV=prod APPLOGS_VOLUME=thermograph_applogs \ +BETA_APPLOGS_VOLUME=thermograph-beta_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d ``` -On the LAN dev node the app's log volume is `thermograph-dev_applogs` (not -`thermograph_applogs`) — edit the two references in `docker-compose.agent.yml` -there. The mesh must already be up (see the main repo's `deploy/swarm/` / -`INFRA.md`); Swarm is **not** required — the agents talk plain HTTP over wg0. +The mesh must already be up (see the main repo's `deploy/swarm/` / `INFRA.md`); +Swarm is **not** required for the agent itself — it talks plain HTTP over wg0. + + ## Verify ```bash -# From any mesh node, confirm Loki is receiving from every host: +# From any mesh node, confirm Loki is receiving from every environment +# (host = prod|beta|dev; a separate node/vps1|vps2|desktop label exists too): curl -s 'http://10.10.0.2:3100/loki/api/v1/label/host/values' # -> {"data":["beta","dev","prod"]} # Then open Grafana → the "Thermograph — Fleet Logs" dashboard. ``` @@ -130,6 +173,13 @@ log volume by service, app error rate by tag, upstream 429 count, Caddy 5xx count, a per-node notifier-liveness indicator (from the heartbeat log), recent app errors, and a live all-container tail. Edit it in Grafana and re-export the JSON here to version a change. + + ## Alerting Provisioned from `grafana/provisioning/alerting/`, mounted read-only into Grafana @@ -146,8 +196,8 @@ Discord contact point. Deliberately **not email**: - Grafana's factory default pointed at the literal string `` — a paging path that had never delivered a message to anyone. -- beta's Grafana relays SMTP through **prod's** Postfix at `10.10.0.1:25` (see - `docker-compose.override.yml` on beta). Email alerts therefore travel through +- vps1's Grafana relays SMTP through **vps2's** Postfix at `10.10.0.1:25` (see + `docker-compose.override.yml` on vps1). Email alerts therefore travel through the box most likely to be on fire, and vanish exactly when they matter. - Discord is off-estate, works when prod is dead, and reaches a phone. @@ -157,7 +207,7 @@ Discord contact point. Deliberately **not email**: `#notes` / `#staff-chat` are for humans. The webhook URL is a secret and is **not in this repo**: it comes from -`DISCORD_ALERT_WEBHOOK_URL` in beta's gitignored `.env`, which compose passes to +`DISCORD_ALERT_WEBHOOK_URL` in vps1's gitignored `.env`, which compose passes to Grafana and which Grafana interpolates when reading the provisioning file. Grafana refuses to start if it is unset. @@ -187,7 +237,7 @@ value is its absence**: nothing can report its own death, so if `#ops-alerts` ha been silent for more than a day, alerting is broken rather than the estate healthy. Set `isPaused: true` on it to opt out of that guarantee. -### Deploying an alerting change to beta +### Deploying an alerting change to vps1 Grafana already mounts `./grafana/provisioning` read-only, so new files under `alerting/` need no compose change — but the **first** deploy does, because it @@ -196,25 +246,26 @@ new environment variable. ```bash # run from the mono-repo root -BETA="agent@75.119.132.91" +VPS1="agent@75.119.132.91" # the monitoring host — NOT the beta environment, + # which now runs on vps2; don't repoint this at it. SSH="ssh -i ~/.ssh/thermograph_agent_ed25519" TS=$(date -u +%Y%m%dT%H%M%SZ) -# 1. put the secret in beta's .env (once), then confirm it took: -$SSH $BETA "grep -q '^DISCORD_ALERT_WEBHOOK_URL=' /opt/observability/.env && echo present || echo MISSING" +# 1. put the secret in vps1's .env (once), then confirm it took: +$SSH $VPS1 "grep -q '^DISCORD_ALERT_WEBHOOK_URL=' /opt/observability/.env && echo present || echo MISSING" # 2. back up the live compose file, stage the new files, move them into place: -$SSH $BETA "sudo cp -a /opt/observability/docker-compose.yml /opt/observability/docker-compose.yml.$TS \ +$SSH $VPS1 "sudo cp -a /opt/observability/docker-compose.yml /opt/observability/docker-compose.yml.$TS \ && rm -rf /tmp/obs-stage && mkdir -p /tmp/obs-stage/alerting" -scp -i ~/.ssh/thermograph_agent_ed25519 observability/docker-compose.yml $BETA:/tmp/obs-stage/ -scp -i ~/.ssh/thermograph_agent_ed25519 observability/grafana/provisioning/alerting/*.yml $BETA:/tmp/obs-stage/alerting/ -$SSH $BETA "sudo mkdir -p /opt/observability/grafana/provisioning/alerting \ +scp -i ~/.ssh/thermograph_agent_ed25519 observability/docker-compose.yml $VPS1:/tmp/obs-stage/ +scp -i ~/.ssh/thermograph_agent_ed25519 observability/grafana/provisioning/alerting/*.yml $VPS1:/tmp/obs-stage/alerting/ +$SSH $VPS1 "sudo mkdir -p /opt/observability/grafana/provisioning/alerting \ && sudo cp /tmp/obs-stage/docker-compose.yml /opt/observability/ \ && sudo cp /tmp/obs-stage/alerting/*.yml /opt/observability/grafana/provisioning/alerting/" # 3. recreate Grafana so it reads the new env var AND the new provisioning. # `docker restart` is NOT enough here — it will not pick up a new env var. -$SSH $BETA "cd /opt/observability && docker compose up -d grafana" +$SSH $VPS1 "cd /opt/observability && docker compose up -d grafana" ``` Subsequent rule-only changes are just step 2 plus @@ -227,13 +278,14 @@ new environment variable is involved. falls back to its default (email → ``) root policy — i.e. no alerting, which is where this started. -Alerting lives on **beta only** — prod and dev run Alloy agents, not Grafana, so -nothing in `alerting/` ever ships to them. +Alerting lives on **vps1 only** — vps2 (prod + beta) and dev only run Alloy +agents, not Grafana, so nothing in `alerting/` ever ships to them, even though +dev happens to share a box with vps1's Grafana. ### Verify alerting ```bash -# on beta, after deploying and restarting Grafana: +# on vps1, after deploying and restarting Grafana: GFP=$(grep '^GF_SECURITY_ADMIN_PASSWORD=' /opt/observability/.env | cut -d= -f2-) WH=$(grep '^DISCORD_ALERT_WEBHOOK_URL=' /opt/observability/.env | cut -d= -f2-) # 1. all 12 rules loaded and healthy (health should be "ok", never "error"): @@ -270,7 +322,7 @@ delete a receiver a policy still references). Either use the UI — Alerting → Contact points → `grafana-default-email` → Delete — or: ```bash -# on beta. Reads the running Alertmanager config, drops the one receiver, +# on vps1. Reads the running Alertmanager config, drops the one receiver, # writes it back. Verified against Grafana 11.6.1 (returns 202). GFP=$(grep '^GF_SECURITY_ADMIN_PASSWORD=' /opt/observability/.env | cut -d= -f2-) AM=localhost:3000/api/alertmanager/grafana/config/api/v1/alerts @@ -289,7 +341,7 @@ This is cosmetic — leaving it costs nothing but a confusing row in the UI. - **Loki is mesh-only** by design; only Grafana (with auth) is public. - Alloy reads the Docker socket read-only to discover containers. -- Retention is 30 days on beta's disk — bump `loki/config.yml` if you want more. +- Retention is 30 days on vps1's disk — bump `loki/config.yml` if you want more. - The app emits a `tag=heartbeat` line for the subscription notifier every ~15m; the dashboard's "Notifier alive?" panel turns red if a node misses it. Extend the same `audit.log_heartbeat()` to any future background daemon. @@ -319,6 +371,6 @@ This is cosmetic — leaving it costs nothing but a confusing row in the UI. is for, but it is a *manual* dead-man's switch: it relies on somebody noticing that the daily message stopped. A real external watchdog (an uptime pinger hitting a Grafana endpoint from off-estate) would close this properly. -- **beta runs a `docker-compose.override.yml` that is not in this repo**, wiring - Grafana's SMTP to prod's Postfix. It should be mirrored into the tracked +- **vps1 runs a `docker-compose.override.yml` that is not in this repo**, wiring + Grafana's SMTP to vps2's Postfix. It should be mirrored into the tracked compose file or deleted; alerting no longer depends on it. diff --git a/observability/alloy/config.alloy b/observability/alloy/config.alloy index 3279da2..2efa071 100644 --- a/observability/alloy/config.alloy +++ b/observability/alloy/config.alloy @@ -1,15 +1,39 @@ -// Grafana Alloy — the log collector that runs on EVERY node (prod, beta, LAN -// dev). It gathers three sources and ships them to the central Loki on beta: +// Grafana Alloy — the log collector that runs on EVERY node (vps1, vps2). It +// gathers three sources and ships them to the central Loki on vps1: // -// 1. Every Docker container's stdout/stderr (app, db, and on beta forgejo) -// via the Docker socket. +// 1. Every Docker container's stdout/stderr (app, db, and on vps1 forgejo + +// the monitoring stack itself) via the Docker socket. // 2. Caddy's host access logs (/var/log/caddy/*.log) — the reverse proxy runs // on the host, not in a container, so its logs aren't in Docker. // 3. The app's structured JSON logs (errors/access/audit *.jsonl) from the -// `applogs` Docker volume, parsed so `level`/`tag`/`phase` become labels. +// `applogs` Docker volumes, parsed so `level`/`tag`/`phase` become labels. +// +// --------------------------------------------------------------------------- +// `host` IS THE ENVIRONMENT, AND IT IS NO LONGER THE NODE +// --------------------------------------------------------------------------- +// Every query, dashboard and alert in this estate slices on host="prod" | +// "beta" | "dev". That label used to be an EXTERNAL label — one value stamped +// on everything the agent shipped — which was exact while one node ran exactly +// one environment. +// +// vps2 now runs prod AND beta. A single external label there would have marked +// every beta container, every beta access log and every beta JSON line as +// `prod`: not a cosmetic problem, but wrong data feeding the prod alert rules, +// and beta's own alerts (NotifierHeartbeatMissingBeta) firing on prod's +// traffic or never firing at all. +// +// So `host` is now derived PER SOURCE — from the container's stack prefix, from +// the access-log filename, from which log volume a JSON line came out of — and +// ALLOY_ENV supplies the fallback for anything on the node that isn't +// environment-specific (Forgejo, Grafana, Loki, the portfolio site). The node +// itself is still labelled, as `node`, because "which machine" is a real +// question too — it is just a different question from "which environment". // // Per-node settings come from the environment (see docker-compose.agent.yml): -// ALLOY_NODE — this node's name label (prod | beta | dev) +// ALLOY_NODE — the MACHINE this agent runs on (vps1 | vps2 | desktop) +// ALLOY_ENV — the default environment label for logs on this node that +// aren't attributable to a specific stack (vps1 -> dev, +// vps2 -> prod) // LOKI_URL — where to push (http://10.10.0.2:3100/loki/api/v1/push over wg0) livedebugging { enabled = false } @@ -43,9 +67,48 @@ discovery.relabel "containers" { source_labels = ["__meta_docker_container_label_com_docker_compose_service"] target_label = "service" } + + // --- environment attribution ------------------------------------------- + // Relabel rules are applied in order and a later write to the same target + // wins, so this is "default, then override with anything more specific". + // + // The regexes are fully anchored (Prometheus relabel semantics), which is + // what keeps `/thermograph_web.1.x` and `/thermograph-beta_beta-web.1.x` + // from matching each other: prod's stack prefix ends in an underscore, + // beta's in `-beta_`. + rule { + source_labels = ["__meta_docker_container_name"] + regex = ".*" + target_label = "host" + replacement = sys.env("ALLOY_ENV") + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/thermograph_.*" + target_label = "host" + replacement = "prod" + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/thermograph-beta_.*" + target_label = "host" + replacement = "beta" + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/thermograph-dev[-_].*" + target_label = "host" + replacement = "dev" + } + + // Drop noise/duplicate containers, per environment. Beta's equivalents are + // listed explicitly: its LB is `thermograph-beta-lb` (which does NOT match + // `thermograph-lb.*`) and its worker is `thermograph-beta_beta-worker`, so + // omitting them would have quietly re-admitted exactly the healthz-poll + // noise the prod entries exist to exclude. rule { source_labels = ["container"] - regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph_worker).*" + regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph-beta-lb|thermograph_worker|thermograph-beta_beta-worker).*" action = "drop" } } @@ -94,11 +157,48 @@ loki.process "caddy" { expression = "(?i)(semrushbot|claudebot|ahrefsbot|yandexbot|bytespider|mj12bot|petalbot)" drop_counter_reason = "crawler" } + + // Attribute each access log to an environment BY FILENAME. On vps2 one + // Caddy fronts both thermograph.org and beta.thermograph.org, so a single + // node-wide label would file every beta request under prod. + // + // By filename rather than by parsing `request.host` out of the JSON: the + // site block that writes the file is the same place the hostname is + // declared (see infra/deploy/Caddyfile.vps2), one label costs nothing to + // evaluate, and it keeps working for a non-JSON log format. + stage.static_labels { + values = { host = sys.env("ALLOY_ENV") } + } + stage.match { + selector = "{filename=~\".*/beta[.]log\"}" + + stage.static_labels { + values = { host = "beta" } + } + } + stage.match { + selector = "{filename=~\".*/thermograph[.]log\"}" + + stage.static_labels { + values = { host = "prod" } + } + } } // --- 3. App structured JSON logs (errors / access / audit) ----------------------- -// Mounted read-only from the app's `applogs` volume at /applogs (see the agent -// compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable. +// Mounted read-only from the app's `applogs` volumes (see the agent compose). +// Lift `level`/`tag`/`phase` out of the JSON so they're queryable. +// +// TWO MOUNTS, NOT ONE. Each environment has its own applogs volume, and on vps2 +// both exist side by side: +// +// /applogs this node's primary environment (vps2 -> prod, vps1 -> dev) +// /applogs-beta beta's, mounted on vps2 only +// +// A single mount was structurally unable to carry both: whichever volume the +// agent bound, the other environment's structured logs would never reach Loki +// at all — silently, since a missing file source is not an error. Every alert +// built on the JSON stream (the notifier heartbeats in particular) reads this. local.file_match "app_jsonl" { path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }] sync_period = "1m" @@ -122,7 +222,38 @@ loki.process "app_jsonl" { } // A record with no explicit level: an error-folder line is an error, else info. stage.static_labels { - values = { source = "app" } + values = { source = "app", host = sys.env("ALLOY_ENV") } + } + stage.labels { + values = { level = "", tag = "", phase = "" } + } +} + +// Beta's structured logs. Absent on vps1 (the mount simply isn't there), where +// this matches nothing and costs nothing. +local.file_match "app_jsonl_beta" { + path_targets = [{ __path__ = "/applogs-beta/**/*.jsonl", job = "app-json" }] + sync_period = "1m" +} + +loki.source.file "app_jsonl_beta" { + targets = local.file_match.app_jsonl_beta.targets + forward_to = [loki.process.app_jsonl_beta.receiver] + + file_watch { + min_poll_frequency = "2s" + max_poll_frequency = "10s" + } +} + +loki.process "app_jsonl_beta" { + forward_to = [loki.write.central.receiver] + + stage.json { + expressions = { level = "level", tag = "tag", phase = "phase", status = "status" } + } + stage.static_labels { + values = { source = "app", host = "beta" } } stage.labels { values = { level = "", tag = "", phase = "" } @@ -134,7 +265,9 @@ loki.write "central" { endpoint { url = sys.env("LOKI_URL") } - // Every line from this node is stamped with its node name, so one Grafana - // view can slice prod vs beta vs dev. - external_labels = { host = sys.env("ALLOY_NODE") } + // `node` — the MACHINE, stamped on everything this agent ships. `host` is + // deliberately NOT here: it is the ENVIRONMENT, set per source above, + // because vps2 carries two of them and an external label cannot vary per + // stream. Setting host here would override that work on every line. + external_labels = { node = sys.env("ALLOY_NODE") } } diff --git a/observability/alloy/docker-compose.agent.beta.yml b/observability/alloy/docker-compose.agent.beta.yml new file mode 100644 index 0000000..3308e95 --- /dev/null +++ b/observability/alloy/docker-compose.agent.beta.yml @@ -0,0 +1,24 @@ +# Overlay for vps2 ONLY — the node that runs two environments. +# +# docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d +# +# Adds beta's applogs volume at /applogs-beta, which config.alloy reads as a +# separate source and labels host="beta". Everything else — the Docker socket, +# the Caddy logs, the primary /applogs mount — comes from the base file. +# +# This is an overlay rather than another variable in the base file because the +# /applogs-beta source is labelled beta UNCONDITIONALLY. On a node where beta +# does not run, there is no correct volume to point it at: aiming it at the +# node's own logs would republish them as beta, manufacturing data for an +# environment that isn't there. "Absent" is the only honest configuration, and +# a missing overlay expresses that exactly. + +services: + alloy: + volumes: + - beta_applogs:/applogs-beta:ro + +volumes: + beta_applogs: + external: true + name: ${BETA_APPLOGS_VOLUME:?set BETA_APPLOGS_VOLUME (thermograph-beta_applogs)} diff --git a/observability/alloy/docker-compose.agent.yml b/observability/alloy/docker-compose.agent.yml index 341f9ca..9bab056 100644 --- a/observability/alloy/docker-compose.agent.yml +++ b/observability/alloy/docker-compose.agent.yml @@ -1,22 +1,41 @@ -# The Alloy log-collector agent. Runs on EACH node (prod, beta, LAN dev) — one -# per node — reading that node's Docker containers, Caddy logs, and the app's -# JSON logs, and shipping them to the central Loki on beta over the mesh. +# The Alloy log-collector agent. Runs on EACH node (vps1, vps2) — one per node +# — reading that node's Docker containers, Caddy logs, and the app's JSON logs, +# and shipping them to the central Loki on vps1 over the mesh. # # Per-node config comes from the environment (an .env file next to this, or # exported before `up`): -# ALLOY_NODE this node's label: prod | beta | dev +# ALLOY_NODE the MACHINE: vps1 | vps2 | desktop +# ALLOY_ENV the default ENVIRONMENT label for logs on this node that +# aren't attributable to a specific stack (Forgejo, Grafana, +# the portfolio site): vps1 -> dev, vps2 -> prod +# APPLOGS_VOLUME this node's primary applogs volume # LOKI_URL push endpoint. All nodes push over the mesh: # http://10.10.0.2:3100/loki/api/v1/push # -# Deploy on a node: -# ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ +# Deploy on vps1 (dev + Forgejo + the monitoring stack itself): +# ALLOY_NODE=vps1 ALLOY_ENV=dev APPLOGS_VOLUME=thermograph-dev_applogs \ +# LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ # docker compose -f docker-compose.agent.yml up -d # -# The app's JSON logs are read from its `applogs` Docker volume, named -# `_applogs`. prod and beta run the app under the compose project -# `thermograph`, so the external volume below is `thermograph_applogs`. The LAN -# dev stack uses the project `thermograph-dev` — on that node, change the two -# `thermograph_applogs` references below to `thermograph-dev_applogs`. +# Deploy on vps2 (prod + beta) — note the SECOND file, which adds beta's log +# volume: +# ALLOY_NODE=vps2 ALLOY_ENV=prod APPLOGS_VOLUME=thermograph_applogs \ +# BETA_APPLOGS_VOLUME=thermograph-beta_applogs \ +# LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ +# docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d +# +# WHY THE VOLUME NAME IS A VARIABLE NOW, AND WHY BETA IS AN OVERLAY. The name +# used to be the literal `thermograph_applogs`, with a comment telling you to +# hand-edit two lines on the dev node. That was survivable while one node ran +# one environment. vps2 runs two, each with its own applogs volume, and a single +# hardcoded mount there would ship one environment's structured logs and +# silently drop the other's. +# +# Beta's mount is a separate overlay file rather than a second variable in this +# one because the second mount is labelled host="beta" unconditionally. Pointing +# it at dev's volume on vps1 just to satisfy compose would ship every dev log +# line a second time labelled as beta — inventing an environment's worth of +# fake data on a box beta does not run on. services: alloy: @@ -27,7 +46,8 @@ services: - --storage.path=/var/lib/alloy/data - --server.http.listen-addr=0.0.0.0:12345 environment: - ALLOY_NODE: ${ALLOY_NODE:?set ALLOY_NODE (prod|beta|dev)} + ALLOY_NODE: ${ALLOY_NODE:?set ALLOY_NODE (vps1|vps2|desktop) — the MACHINE} + ALLOY_ENV: ${ALLOY_ENV:?set ALLOY_ENV (dev|beta|prod) — this node's default environment label} LOKI_URL: ${LOKI_URL:?set LOKI_URL} volumes: - ./config.alloy:/etc/alloy/config.alloy:ro @@ -36,13 +56,16 @@ services: # Caddy's host access logs (runs on the host, not a container). - /var/log/caddy:/var/log/caddy:ro # The app's structured JSON logs, straight off its named volume. - - thermograph_applogs:/applogs:ro + - applogs:/applogs:ro - alloy_data:/var/lib/alloy/data restart: unless-stopped volumes: # The app's existing log volume (created by the app's own stack). external:true - # means compose references it, never creates or deletes it. - thermograph_applogs: + # means compose references it, never creates or deletes it. The NAME varies by + # node — see the header — while the mount path stays /applogs, which is what + # lets one config.alloy serve every node. + applogs: external: true + name: ${APPLOGS_VOLUME:?set APPLOGS_VOLUME (e.g. thermograph_applogs on vps2, thermograph-dev_applogs on vps1)} alloy_data: {} diff --git a/observability/caddy-grafana.conf b/observability/caddy-grafana.conf index 787b816..420805c 100644 --- a/observability/caddy-grafana.conf +++ b/observability/caddy-grafana.conf @@ -1,10 +1,11 @@ -# Append to beta's /etc/caddy/Caddyfile (beta is the monitoring host), then -# `systemctl reload caddy`. Grafana publishes to 127.0.0.1:3000; this fronts it -# with TLS, same pattern as beta's other site blocks. Login is Google SSO (with a -# break-glass local admin); Loki (the log store, on 10.10.0.2:3100) is NOT proxied -# here and stays mesh-only. +# Append to vps1's /etc/caddy/Caddyfile (vps1 is the monitoring host — NOT the +# beta environment, which runs on vps2 now), then `systemctl reload caddy`. +# Grafana publishes to 127.0.0.1:3000; this fronts it with TLS, same pattern as +# vps1's other site blocks. Login is Google SSO (with a break-glass local +# admin); Loki (the log store, on 10.10.0.2:3100) is NOT proxied here and stays +# mesh-only. # -# dashboard.thermograph.org's A record must already point at beta (75.119.132.91). +# dashboard.thermograph.org's A record must already point at vps1 (75.119.132.91). dashboard.thermograph.org { encode zstd gzip diff --git a/observability/docker-compose.yml b/observability/docker-compose.yml index 06b3bc5..36516b7 100644 --- a/observability/docker-compose.yml +++ b/observability/docker-compose.yml @@ -1,11 +1,14 @@ # The central observability stack: Loki (log store) + Grafana (UI). Runs on ONE -# node — the monitoring host, `beta` (75.119.132.91 / mesh 10.10.0.2) — because -# beta is an always-on VPS with a public Caddy already fronting it. Every node's -# Alloy agent (see alloy/) ships logs here; agents on prod and the desktop reach -# Loki over the WireGuard mesh, so Loki must listen on the mesh interface. +# node — the monitoring host, `vps1` (75.119.132.91 / mesh 10.10.0.2) — because +# vps1 is an always-on VPS with a public Caddy already fronting it. This is NOT +# where the beta *environment* lives (that's vps2 now, alongside prod, as two +# Swarm stacks) — don't let "beta" in older docs send you looking for Grafana +# there. Every node's Alloy agent (see alloy/) ships logs here; agents on vps2 +# and desktop reach Loki over the WireGuard mesh, so Loki must listen on the +# mesh interface. # -# Deploy (on beta): docker compose up -d -# Grafana is proxied by beta's Caddy at dashboard.thermograph.org (see README); +# Deploy (on vps1): docker compose up -d +# Grafana is proxied by vps1's Caddy at dashboard.thermograph.org (see README); # Loki is NOT public — it binds the mesh IP only, reachable to agents over wg0. services: @@ -19,7 +22,7 @@ services: # Mesh-only: agents on prod (10.10.0.1) and desktop (10.10.0.3) push here # over wg0. Never published on the public interface. - "10.10.0.2:3100:3100" - # Also localhost, so the beta-local Alloy agent and curl checks can reach it. + # Also localhost, so the vps1-local Alloy agent and curl checks can reach it. - "127.0.0.1:3100:3100" restart: unless-stopped @@ -61,7 +64,7 @@ services: # --- Alerting -------------------------------------------------------------- # The Discord webhook that grafana/provisioning/alerting/contact-points.yml # interpolates as $DISCORD_ALERT_WEBHOOK_URL. It is a secret (holding it is - # enough to post in the channel), so it lives only in beta's .env. + # enough to post in the channel), so it lives only in vps1's .env. # Required, not defaulted: a Grafana that comes up with an empty webhook # looks perfectly healthy and pages nobody, which is the failure mode this # whole config exists to end. Better to refuse to start. @@ -71,7 +74,7 @@ services: - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - grafana_data:/var/lib/grafana ports: - # Host-local; beta's Caddy reverse-proxies to it. Not public directly. + # Host-local; vps1's Caddy reverse-proxies to it. Not public directly. - "127.0.0.1:3000:3000" restart: unless-stopped diff --git a/observability/grafana/provisioning/alerting/contact-points.yml b/observability/grafana/provisioning/alerting/contact-points.yml index bdddbe2..a24535d 100644 --- a/observability/grafana/provisioning/alerting/contact-points.yml +++ b/observability/grafana/provisioning/alerting/contact-points.yml @@ -3,21 +3,22 @@ # # WHY NOT EMAIL. The Grafana factory default routes to the literal string # "", i.e. nowhere. Fixing it by pointing at a real mailbox -# would still be wrong: beta's Grafana relays SMTP through prod's Postfix at -# 10.10.0.1:25 (see the live docker-compose.override.yml on beta), so email +# would still be wrong: vps1's Grafana (the monitoring host — not the beta +# environment, which runs on vps2) relays SMTP through vps2's Postfix at +# 10.10.0.1:25 (see the live docker-compose.override.yml on vps1), so email # alerts travel *through the box most likely to be on fire* and are silently # lost whenever Postfix is down — which is exactly when you need them. Discord # is off-estate: it works when prod is dead, and it works from a phone. # # THE WEBHOOK URL IS A SECRET. Anyone holding it can post into the channel, so # it is NOT in this repo. It is read from the Grafana process environment, which -# docker-compose.yml feeds from beta's gitignored .env (see .env.example). +# docker-compose.yml feeds from vps1's gitignored .env (see .env.example). # Grafana expands $VAR / $__env{VAR} when it reads provisioning files; if the # variable is unset the contact point ends up with a literal "$DISCORD_..." # string and every notification fails — see README "Verify alerting". # # To rotate: make a new webhook on the #ops-alerts channel, replace the value in -# beta's .env, `docker restart observability-grafana-1`, delete the old webhook +# vps1's .env, `docker restart observability-grafana-1`, delete the old webhook # in Discord. apiVersion: 1 diff --git a/observability/grafana/provisioning/alerting/rules-prod.yml b/observability/grafana/provisioning/alerting/rules-prod.yml index f3f2a3b..470348a 100644 --- a/observability/grafana/provisioning/alerting/rules-prod.yml +++ b/observability/grafana/provisioning/alerting/rules-prod.yml @@ -33,7 +33,11 @@ # label — only `container`, which includes a per-task suffix that changes on # every redeploy (thermograph_web.1.). So prod services are matched as # container=~"thermograph_.+" with job="docker". Do not use `service=` -# here; it only works on the compose hosts (beta, dev). +# here; it only works on dev, now the ONLY compose host left — beta is Swarm +# too (stack `thermograph-beta`, since both prod and beta run on vps2), so a +# beta-targeted rule needs its own regex against beta's prefixed container +# names (thermograph-beta_beta-web.1. etc.), not a bare `service=` +# match either. # # --------------------------------------------------------------------------- # NOT ALERTABLE, ON PURPOSE @@ -69,6 +73,10 @@ groups: # DATA: 15-minute windows over 24h ranged 35 -> 301 lines, minimum 35, # and never once reached zero — including *during* the outage (72), when # Caddy was happily logging 503s. Zero is unambiguous. + # + # The description names vps1, not "beta": the mesh path that matters here + # is the one to the MONITORING host, which is vps1. Beta is on vps2 and + # has nothing to do with whether these logs arrive. - uid: tg_prod_edge_dark title: ProdEdgeDark condition: C @@ -81,7 +89,8 @@ groups: Nothing has reached thermograph.org's reverse proxy in 15 minutes (healthy floor is 35 lines/15m, and it stayed at 72 even during the 04:20Z outage). Either prod is down, Caddy is down, prod's Alloy - agent stopped shipping, or the WireGuard mesh to beta is broken. + agent stopped shipping, or the WireGuard mesh to vps1 (where Loki + runs) is broken. This rule also fires on Loki NoData/errors, so it doubles as the "we have lost observability" alarm — if it is the only thing firing, suspect the pipeline before suspecting the app. @@ -397,9 +406,10 @@ groups: # means two consecutive empty evaluations, so roughly 25 minutes of real # silence before it pages — about one and a half missed beats. # - # dev is deliberately excluded: the LAN box had zero heartbeats for the - # first 10.5h of the backtest window because it was simply off. Alerting - # on dev would mean an alert that is firing more often than not. + # dev is deliberately excluded: the dev box (mesh-only, on vps1) had zero + # heartbeats for the first 10.5h of the backtest window because it was + # simply off. Alerting on dev would mean an alert that is firing more + # often than not. # # MUST stay filtered to daemon="subscription-notifier". The selector was # originally bare `tag="heartbeat"`, which was unambiguous while the @@ -833,10 +843,10 @@ groups: # -- 12 ---------------------------------------------------------------- # Nothing in this file can tell you that Grafana itself has stopped, or - # that beta is down, or that the Discord webhook was revoked — a monitoring - # system cannot report its own death. So this rule fires permanently and - # by design, throttled by the notification policy to exactly one Discord - # message per 24 hours. + # that vps1 (the monitoring host) is down, or that the Discord webhook was + # revoked — a monitoring system cannot report its own death. So this rule + # fires permanently and by design, throttled by the notification policy + # to exactly one Discord message per 24 hours. # # HOW TO USE IT: if #ops-alerts has been silent for more than a day, the # alerting pipeline is broken, not the estate healthy. That one daily @@ -844,6 +854,7 @@ groups: # rules are still capable of reaching you. # # To silence it (and lose that guarantee), set isPaused: true. + # - uid: tg_alerting_watchdog title: AlertingWatchdog condition: C @@ -854,13 +865,20 @@ groups: summary: 'Alerting pipeline is alive (daily heartbeat — not a fault)' description: >- This is the dead-man's switch and it is supposed to be firing. It - proves Grafana on beta is evaluating rules, Loki is answering, and + proves Grafana on vps1 is evaluating rules, Loki is answering, and the Discord webhook still works. Seeing it once a day is correct. NOT seeing it for over a day means alerting itself is down and no - other rule can reach you — check observability-grafana-1 on beta. + other rule can reach you — check observability-grafana-1 on vps1. labels: severity: watchdog - host: beta + # `node`, not `host`: every other rule's `host` names an ENVIRONMENT + # (prod/beta/dev), and this rule is about the alerting pipeline + # itself, which is a machine. It used to be filed `host: beta` back + # when the monitoring box and the beta environment were the same + # machine — they are not any more, and that label would now read as a + # claim about beta. Routing is unaffected: notification-policies.yml + # matches on `severity` only and uses `host` just for grouping. + node: vps1 class: meta isPaused: false data: diff --git a/observability/loki/config.yml b/observability/loki/config.yml index b2dfb46..95bad61 100644 --- a/observability/loki/config.yml +++ b/observability/loki/config.yml @@ -1,8 +1,9 @@ -# Loki — the single log store for the whole Thermograph fleet (prod, beta, LAN -# dev). Every node's Alloy agent pushes here over the WireGuard mesh; Grafana -# reads from here. Single-binary, filesystem-backed: no object storage, no -# clustering — right for a three-node hobby fleet, and everything lives on one -# volume you can back up or blow away. +# Loki — the single log store for the whole Thermograph fleet: prod and beta +# (both on vps2, as separate Swarm stacks) and dev (vps1, mesh-only, its own +# compose project). Every node's Alloy agent pushes here over the WireGuard +# mesh; Grafana reads from here. Single-binary, filesystem-backed: no object +# storage, no clustering — right for a three-node hobby fleet, and everything +# lives on one volume you can back up or blow away. auth_enabled: false server: From 7583258509de22b0faace75301fce509522be8e1 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 07:05:09 +0000 Subject: [PATCH 2/2] infra-sync: refuse to render dev's vault onto a host that is not dev (#107) --- .forgejo/workflows/infra-sync.yml | 25 ++++++++++++++++++++++- infra/deploy/RUNBOOK-vps1-vps2-cutover.md | 14 ++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/infra-sync.yml b/.forgejo/workflows/infra-sync.yml index 8f86697..67cd055 100644 --- a/.forgejo/workflows/infra-sync.yml +++ b/.forgejo/workflows/infra-sync.yml @@ -61,6 +61,30 @@ jobs: cd /opt/thermograph-dev git fetch --prune origin dev git reset --hard origin/dev + echo "synced to $(git log --oneline -1)" + + # RENDER ONLY IF THIS HOST IS ACTUALLY DEV. + # + # dev's env file is /etc/thermograph.env — the same path beta uses, + # and during the vps1/vps2 cutover beta is STILL ON VPS1. Rendering + # unconditionally would overwrite the live env file of the + # environment currently serving beta.thermograph.org with dev's + # credentials: beta's running containers would survive (env_file is + # read at container start) and then come up wrong on its next + # deploy, pointed at dev's database password. + # + # The host marker is the box's own statement of which environment it + # is, and it is the right authority for "is it safe to write this + # file here". It flips to `dev` in provision-dev.sh, which is run + # only after beta has vacated. Until then this skips loudly rather + # than failing: the checkout sync above is still useful and correct. + marker=$(cat /etc/thermograph/secrets-env 2>/dev/null || true) + if [ "$marker" != dev ]; then + echo "!! /etc/thermograph/secrets-env says '${marker:-}', not 'dev'." + echo "!! Skipping the secrets render — this host is not dev yet, and" + echo "!! /etc/thermograph.env here belongs to '${marker:-another environment}'." + exit 0 + fi if [ -f infra/deploy/render-secrets.sh ]; then . infra/deploy/render-secrets.sh # SKIP_COMMON is dev's standing rule, not a preference: common.yaml @@ -69,7 +93,6 @@ jobs: THERMOGRAPH_SECRETS_SKIP_COMMON=1 \ render_thermograph_secrets /opt/thermograph-dev/infra dev /etc/thermograph.env fi - echo "synced to $(git log --oneline -1)" sync-beta: if: github.ref_name == 'main' diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md index 72934a5..b1874da 100644 --- a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -278,7 +278,19 @@ sudo bash /opt/thermograph-dev/infra/deploy/provision-dev.sh # clones if absen ``` The script writes `/etc/thermograph/secrets-env` = `dev` and removes any -`deploy-mode` marker. Then deploy: +`deploy-mode` marker. + +**That marker flip is load-bearing, and it is why this step comes after step 9.** +dev's env file is `/etc/thermograph.env` — the very path beta uses while beta is +still on vps1. `infra-sync`'s `sync-dev` job refuses to render unless the marker +already says `dev`, so until you run this script it syncs the checkout and skips +the render with a loud message. Flip the marker before beta has vacated and the +next dev push overwrites beta's live env file with dev's credentials; beta's +running containers survive it and then come up wrong on their next deploy. + +(The checkout itself is safe to create at any time — only the render is gated.) + +Then deploy: ``` cd /opt/thermograph-dev