From 1a56a6380031557f0b065b9f1b8cd846de36b30f Mon Sep 17 00:00:00 2001 From: emi Date: Sat, 25 Jul 2026 17:51:42 +0000 Subject: [PATCH 01/37] alerting: base web/worker liveness on the app heartbeat, not container stdout (#94) --- .../provisioning/alerting/rules-prod.yml | 166 ++++++++++++------ 1 file changed, 113 insertions(+), 53 deletions(-) diff --git a/observability/grafana/provisioning/alerting/rules-prod.yml b/observability/grafana/provisioning/alerting/rules-prod.yml index ec2b3bb..f3f2a3b 100644 --- a/observability/grafana/provisioning/alerting/rules-prod.yml +++ b/observability/grafana/provisioning/alerting/rules-prod.yml @@ -400,6 +400,24 @@ groups: # dev is deliberately excluded: the LAN box had zero heartbeats for the # first 10.5h of the backtest window because it was simply off. Alerting # on dev would mean an alert that is firing more often than not. + # + # MUST stay filtered to daemon="subscription-notifier". The selector was + # originally bare `tag="heartbeat"`, which was unambiguous while the + # notifier thread was the only thing that ever beat. It stopped being + # unambiguous on 2026-07-25T17:24Z, when web/app.py's _start_heartbeat + # shipped: every uvicorn process in every replica now beats every 300s + # under the same tag, tagged daemon=web / daemon=worker. Measured + # immediately after that deploy, a 20m window held 16 web beats and 4 + # worker beats against 2 notifier beats — so the bare selector clears the + # `< 1` threshold on web traffic alone and can no longer observe the + # notifier at all. + # + # That matters because this rule had just caught a real 17.5h outage + # (notifier silent 2026-07-24T23:49Z -> 2026-07-25T17:25Z, zero + # subscription notifications delivered). Left bare, it would have gone + # quiet on the next occurrence instead of paging. `daemon` is a JSON field + # and not a stream label, hence the `| json |` parse rather than a label + # matcher. - uid: tg_notifier_heartbeat_prod title: NotifierHeartbeatMissingProd condition: C @@ -427,7 +445,7 @@ groups: model: refId: A datasource: { type: loki, uid: loki } - expr: 'sum(count_over_time({job="app-json", tag="heartbeat", host="prod"} [20m])) or vector(0)' + expr: 'sum(count_over_time({job="app-json", tag="heartbeat", host="prod"} | json | daemon="subscription-notifier" [20m])) or vector(0)' queryType: instant editorMode: code intervalMs: 1000 @@ -480,7 +498,7 @@ groups: model: refId: A datasource: { type: loki, uid: loki } - expr: 'sum(count_over_time({job="app-json", tag="heartbeat", host="beta"} [20m])) or vector(0)' + expr: 'sum(count_over_time({job="app-json", tag="heartbeat", host="beta"} | json | daemon="subscription-notifier" [20m])) or vector(0)' queryType: instant editorMode: code intervalMs: 1000 @@ -506,16 +524,31 @@ groups: - evaluator: { type: lt, params: [1] } # =========================================================================== - # Per-container liveness, inferred from log volume. The fleet collects no - # container metrics, so "did this service stop producing any output at all" is - # the only death certificate available — and it turns out to be a good one, - # because every service below logs on a floor that never approaches zero. + # Per-service liveness. The fleet collects no container metrics, so liveness + # has to be inferred from "is this thing still producing output at all". Two + # different sources are used here, and picking the wrong one is how this group + # generated a day of false pages: # - # Caveat worth knowing before you trust these: a container can be running and - # logging while its lines never reach Loki, if the node's Alloy agent stops - # tailing it. That is a real observed failure on prod (see README). These - # rules cannot tell the two apart — they say "no logs", which is the honest - # claim. Either way something needs looking at. + # * db and frontend infer it from CONTAINER LOG VOLUME. Valid for these two: + # both log to stdout on a floor that never approaches zero (db a metronomic + # 54-69 lines/2h, frontend ~66/30m), and prod's Alloy ships both. + # + # * web and worker infer it from the APP'S OWN HEARTBEAT (tag=heartbeat in + # the app-json file source, filtered by the daemon field). Container log + # volume is NOT a valid liveness signal for either: web's request logging + # moved off stdout onto app-json, and worker's stdout is dropped outright + # by a rule in prod's alloy config. Both containers can be perfectly + # healthy while their docker streams sit at exactly zero forever. + # + # Before adding a container-log-silence rule for a new service, confirm the + # service actually logs to stdout AND that prod's discovery.relabel "containers" + # block does not drop it. A rule against a dropped stream is unsatisfiable and + # will page forever. + # + # Caveat that still applies to the db/frontend rules: a container can be + # running and logging while its lines never reach Loki, if the node's Alloy + # agent stops tailing it. Those rules cannot tell the two apart — they say + # "no logs", which is the honest claim. Either way something needs looking at. # =========================================================================== - orgId: 1 name: prod-services @@ -524,24 +557,43 @@ groups: rules: # -- 8 ----------------------------------------------------------------- - # DATA: 15-minute windows over 24h ranged 54 -> 954 lines. The minimum, 54, - # occurred during the outage; the quietest healthy window was ~150. A - # 30-minute window has never been below ~110. Zero means the container is - # gone. This is the app itself, so it pages. + # Web-tier liveness, from the app's own heartbeat rather than container + # stdout. + # + # This rule used to count `{job="docker", container=~"thermograph_web.+"}` + # over 30m against a measured floor of ~110 lines. That floor was real when + # it was written (the backtest saw 3.8k-10.5k lines per 6h), but the app + # moved its request logging off stdout onto the app-json file source, and + # from 2026-07-25T00:02Z a web container emits ~15 uvicorn banner lines for + # its entire lifetime and nothing after. The old rule then sat firing + # permanently on a perfectly healthy tier, clearing only for the few + # minutes after each deploy or autoscale event produced fresh banners — + # 20+ pages in a day, every one of them false. See web/app.py's + # _start_heartbeat docstring, which states outright that stdout silence + # cannot distinguish a live web process from a dead one and that this + # heartbeat is the signal to alert on instead. + # + # DATA: _start_heartbeat beats every 300s from every uvicorn process in + # every replica, unguarded by the leader election, so a 4-process replica + # yields ~16 beats per 20m (measured: exactly 16). Zero over 20m means no + # web process is alive anywhere. This is the app itself, so it pages. - uid: tg_prod_web_silent - title: ProdWebContainerSilent + title: ProdWebHeartbeatMissing condition: C for: 5m noDataState: OK execErrState: OK annotations: - summary: 'prod: thermograph_web has logged nothing for 30 minutes' + summary: 'prod: no web heartbeat for 20 minutes' description: >- - The main app container has produced zero log lines in 30 minutes. - Healthy floor is ~110 lines per 30m and it never dipped below 54 even - mid-outage, so this is a container that is gone, wedged, or no longer - being tailed by prod's Alloy agent. Run `docker service ps - thermograph_web` on prod. + No thermograph_web process on prod has emitted a liveness heartbeat + in 20 minutes. Every uvicorn process beats every 5 minutes + independently of the notifier leader election, so a healthy 4-process + replica produces ~16 beats per window — zero means no web process is + alive anywhere, not merely quiet. Run `docker service ps + thermograph_web` on prod. Note this is the app's own heartbeat, not + container stdout: web's stdout is silent by design and is not a + liveness signal. labels: severity: critical host: prod @@ -549,12 +601,12 @@ groups: isPaused: false data: - refId: A - relativeTimeRange: { from: 1800, to: 0 } + relativeTimeRange: { from: 1200, to: 0 } datasourceUid: loki model: refId: A datasource: { type: loki, uid: loki } - expr: 'sum(count_over_time({job="docker", host="prod", container=~"thermograph_web.+"} [30m])) or vector(0)' + expr: 'sum(count_over_time({job="app-json", host="prod", tag="heartbeat"} | json | daemon="web" [20m])) or vector(0)' queryType: instant editorMode: code intervalMs: 1000 @@ -689,42 +741,50 @@ groups: - evaluator: { type: lt, params: [1] } # -- 11 ---------------------------------------------------------------- - # DATA: a dead-steady 121 lines/hour (its own healthcheck polling) for the - # first 14h of the backtest, then nothing at all. + # Worker liveness, from the app's own heartbeat rather than container + # stdout. # - # HEADS UP: THIS RULE WILL BE FIRING THE MOMENT YOU DEPLOY IT, and it is - # right to. As of 2026-07-24 19:45Z the prod worker container is running - # and healthy and is writing ~19 lines per 10 minutes to its own docker - # log — but not one of those lines has reached Loki since 04:43Z. Prod's - # Alloy agent stopped tailing the container when Swarm replaced the task. - # Nobody knew, because nothing was watching. See README "Known gaps". + # The previous rule counted `{job="docker", container=~"thermograph_worker.+"}` + # and was UNSATISFIABLE BY CONSTRUCTION — it could never return anything but + # zero, no matter how healthy the worker was. prod's Alloy config drops that + # container on purpose, in discovery.relabel "containers": # - # WHY THIS IS A WARNING AND NotifierHeartbeatMissingProd IS CRITICAL: the - # two draw on *different* Alloy sources — this one on loki.source.docker - # (the container's stdout), the heartbeat on loki.source.file tailing - # /applogs. That independence is the diagnostic. Right now prod's - # heartbeats are arriving in every single 20-minute window while its - # worker docker stream has been empty for 14 hours, which localises the - # fault to Alloy's docker tail rather than to the worker: the process is - # doing its job, we just cannot see it breathe. Both rules firing together - # means the worker really is down. + # regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph_worker).*" + # action = "drop" + # + # with the stated reasoning that worker stdout was 100% /healthz poll noise + # and the app's own access/*.jsonl is a strict superset of it. So the stream + # this rule queried is discarded at the agent and never reaches Loki. + # + # The old comment here read the resulting permanent zero as "Alloy stopped + # tailing the container after Swarm replaced the task" and pointed the + # runbook at restarting alloy-alloy-1. That diagnosis was wrong, and the + # fix it recommended could not have worked: a restarted Alloy re-reads the + # same config and drops the same container again. The rule fired + # continuously from 2026-07-24T21:05Z with no underlying fault. + # + # DATA: _start_heartbeat beats every 300s from the worker process + # (measured: exactly 4 per 20m). Unlike the notifier below it is not gated + # on the leader election, so it reports the process being alive even when + # this worker is not the elected notifier — which is the distinction the + # old rule was trying and failing to draw. - uid: tg_prod_worker_silent - title: ProdWorkerContainerSilent + title: ProdWorkerHeartbeatMissing condition: C for: 10m noDataState: OK execErrState: OK annotations: - summary: 'prod: thermograph_worker has logged nothing for 2 hours' + summary: 'prod: no worker heartbeat for 20 minutes' description: >- - The worker container has produced zero log lines in 2 hours against - a healthy floor of ~242 per 2h. Two different faults look identical - here: the worker is actually dead, or prod's Alloy agent has stopped - tailing it (a real, observed failure — Alloy can miss a container - after Swarm replaces the task). Check both: `docker service ps - thermograph_worker` on prod, and whether `docker logs` on the live - container is producing lines that Loki does not have. Restarting - alloy-alloy-1 on prod fixes the second case. + The thermograph_worker process on prod has not emitted a liveness + heartbeat in 20 minutes (healthy is ~4 per window, one every 5 + minutes). This beat is independent of the notifier leader election, + so it means the worker process itself is gone or wedged, not merely + that it lost the election. Run `docker service ps thermograph_worker` + on prod. Do not look at the worker's container logs — prod's Alloy + config drops that stream deliberately, so it is always empty and is + not evidence of anything. labels: severity: warning host: prod @@ -732,12 +792,12 @@ groups: isPaused: false data: - refId: A - relativeTimeRange: { from: 7200, to: 0 } + relativeTimeRange: { from: 1200, to: 0 } datasourceUid: loki model: refId: A datasource: { type: loki, uid: loki } - expr: 'sum(count_over_time({job="docker", host="prod", container=~"thermograph_worker.+"} [2h])) or vector(0)' + expr: 'sum(count_over_time({job="app-json", host="prod", tag="heartbeat"} | json | daemon="worker" [20m])) or vector(0)' queryType: instant editorMode: code intervalMs: 1000 -- 2.45.2 From b32b5841f387138278a94de0ae4ad56421fed8e6 Mon Sep 17 00:00:00 2001 From: emi Date: Sat, 25 Jul 2026 18:17:37 +0000 Subject: [PATCH 02/37] alerting: don't print a bare "value:" when ValueString is empty (#96) --- .../provisioning/alerting/contact-points.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/observability/grafana/provisioning/alerting/contact-points.yml b/observability/grafana/provisioning/alerting/contact-points.yml index 42f1f10..bdddbe2 100644 --- a/observability/grafana/provisioning/alerting/contact-points.yml +++ b/observability/grafana/provisioning/alerting/contact-points.yml @@ -37,12 +37,26 @@ contactPoints: # Deliberately plain. A template error here breaks EVERY notification # silently, so this uses only functions verified against Grafana # 11.6.1 via /api/alertmanager/grafana/config/api/v1/receivers/test. + # + # `.ValueString` is guarded because it is not always populated. It + # carries the evaluated refIds only when the notification came from an + # evaluation that produced them; an instance resolved by Grafana's + # STALENESS handling — no evaluation ever returns it again, so the + # state manager expires it — resolves with an empty ValueString. That + # happens whenever a rule's `title` changes, since the title becomes + # the alertname label and the old label set is orphaned. Renaming + # ProdWorkerContainerSilent -> ProdWorkerHeartbeatMissing on + # 2026-07-25 did exactly that and posted a resolved notice reading + # "value:" with nothing after it. An empty labelled field is worse + # than an absent one: it reads as a value that failed to compute. + # Unguarded `{{ .ValueString }}` on its own would be safe; printing + # the label unconditionally is the bug. message: |- {{ range .Alerts }}**severity:** {{ .Labels.severity }} · **host:** {{ .Labels.host }} {{ .Annotations.summary }} {{ .Annotations.description }} - `value: {{ .ValueString }}` - {{ end }} + {{ if .ValueString }}`value: {{ .ValueString }}` + {{ end }}{{ end }} # --- The factory-default contact point ------------------------------------------- -- 2.45.2 From 12441be0c10aaa9f774ff8db3af7d514876e7241 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sat, 25 Jul 2026 11:25:52 -0700 Subject: [PATCH 03/37] docs: add a developer onboarding guide for the monorepo Twelve documents under docs/onboarding/ covering orientation, local setup, the repo map, per-domain deep dives, the cross-service contracts, CI and the release flow, infra and secrets, observability, task recipes, and a list of which docs in this tree are currently stale. Every command in the setup guide was run against this checkout: the backend suite (429 passed, 8 skipped), the frontend Go suite, a venv boot of the backend, and a build+boot of the Go frontend. Two findings recorded along the way: - static/units.js's F_REGIONS is guarded by no test, despite three source comments claiming "a test asserts all three stay identical". The Go test only cross-checks the Go copy against the backend's Python. All four copies are currently identical. - backend/ and frontend/docker-compose.test.yml still default to the retired emi/thermograph-backend/app image path, and the frontend harness pins the split-era v0.0.2-split-ci tag. Claude-Session: https://claude.ai/code/session_01AfXqHrxCJLs2D7hpQkiUiJ --- README.md | 7 + docs/onboarding/01-orientation.md | 146 ++++++++++++++ docs/onboarding/02-setup.md | 217 ++++++++++++++++++++ docs/onboarding/03-repo-map.md | 189 +++++++++++++++++ docs/onboarding/04-backend.md | 292 +++++++++++++++++++++++++++ docs/onboarding/05-frontend.md | 221 ++++++++++++++++++++ docs/onboarding/06-contracts.md | 183 +++++++++++++++++ docs/onboarding/07-ci-and-release.md | 229 +++++++++++++++++++++ docs/onboarding/08-infra-secrets.md | 254 +++++++++++++++++++++++ docs/onboarding/09-observability.md | 199 ++++++++++++++++++ docs/onboarding/10-recipes.md | 256 +++++++++++++++++++++++ docs/onboarding/11-traps.md | 268 ++++++++++++++++++++++++ docs/onboarding/README.md | 70 +++++++ 13 files changed, 2531 insertions(+) create mode 100644 docs/onboarding/01-orientation.md create mode 100644 docs/onboarding/02-setup.md create mode 100644 docs/onboarding/03-repo-map.md create mode 100644 docs/onboarding/04-backend.md create mode 100644 docs/onboarding/05-frontend.md create mode 100644 docs/onboarding/06-contracts.md create mode 100644 docs/onboarding/07-ci-and-release.md create mode 100644 docs/onboarding/08-infra-secrets.md create mode 100644 docs/onboarding/09-observability.md create mode 100644 docs/onboarding/10-recipes.md create mode 100644 docs/onboarding/11-traps.md create mode 100644 docs/onboarding/README.md diff --git a/README.md b/README.md index 7db6473..a999ec5 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,13 @@ for: **per-domain images, per-domain deploys, and an async FE/BE contract**. `thermograph-docs` deliberately **stays its own repo** (ADRs + runbooks, no build artifacts, different change cadence). +## New here? + +[`docs/onboarding/`](docs/onboarding/README.md) is the developer onboarding +path: orientation, verified local-setup recipes, a per-domain deep dive, the +cross-service contracts that break silently, the release flow, and a list of +which docs in this repo are currently stale. + ## How CI stays decoupled Every workflow in `.forgejo/workflows/` is **path-filtered to its domain**: a diff --git a/docs/onboarding/01-orientation.md b/docs/onboarding/01-orientation.md new file mode 100644 index 0000000..0194611 --- /dev/null +++ b/docs/onboarding/01-orientation.md @@ -0,0 +1,146 @@ +# 1. Orientation + +## What the product actually claims + +Thermograph answers one question: **how unusual is the weather here, right now, +compared to this exact place's own history?** + +That framing is not decoration — it constrains the code, the copy and the +colour scales: + +- A grade is a **percentile against ~45 years of that grid cell's own record**, + not an absolute reading. 60 °F is *Above Normal* on a cool coast and *Below + Normal* inland on the same afternoon. +- Tier names are therefore relative words — *Near Record*, *High*, *Above + Normal*, *Normal*, *Below Normal*, *Low*, *Near Record* — never "hot", + "warm", "cold". This rule binds prose, alt text, commit messages and API + fields alike. +- Precipitation is graded on its **own** ladder, because most days are dry: a + rainy day is ranked only among the rain days in its seasonal window, and dry + days are coloured by dry-streak length instead. + +Public since **2026-07-16**. Free. Run by one operator plus automation, which +is why so much of this repo is about making irreversible things hard. + +## The domain model, in six steps + +1. **Grid** (`backend/data/grid.py`) — an arbitrary lat/lon snaps to a stable + **~4 sq mi cell**. Latitude rows are ~2 miles tall; longitude spacing scales + by `cos(latitude)` so cells stay roughly square at any latitude. The cell id + is deterministic and is the cache key for everything downstream. Worldwide + coverage, no gaps. +2. **History** (`backend/data/climate.py`) — the full **1980 → present daily + record** (max/min temp, precip, wind, gust, humidity, feels-like) for that + cell, fetched once and stored durably. Expensive to obtain, so it is treated + as source-of-truth data, not cache. +3. **Recent + forecast** — a separate bundle of recent observations plus ~8 + forward days, refreshed on its own TTL. This is what "today" is graded from. +4. **Grading** (`backend/data/grading.py`) — for a given day of year, the + reference distribution is *every historical day within ±7 days of it* + (a 15-day seasonal window that wraps the year end). The observed value is + placed on that distribution as an **empirical mid-rank percentile**, then + mapped onto `TEMP_BANDS` / `RAIN_BANDS`. +5. **Payloads** (`backend/api/payloads.py`) — pure "inputs → response dict" + assembly. One definition per payload shape, shared by the HTTP routes, the + `/cell` bundle and offline tooling. +6. **Caching** (`backend/data/store.py`) — every derived payload is stored + against a **validity token** that encodes everything the payload depends on. + A token mismatch is simply a miss. Freshness is driven by the token + advancing, *never* by clock expiry — so stale data cannot be served, and one + `PAYLOAD_VER` bump atomically orphans every pre-upgrade row. + +If you internalise one thing: **the token is the cache-correctness mechanism, +and `PAYLOAD_VER` is the lever that moves it.** See +[contracts](06-contracts.md). + +## The estate + +Four machines on a WireGuard mesh (`10.10.0.0/24`): + +| Host | Mesh IP | Public | Runs | +|---|---|---|---| +| **prod** | `10.10.0.1` | `169.58.46.181`, thermograph.org | Docker **Swarm** app stack, TimescaleDB, Postfix, backups, Centralis | +| **beta** | `10.10.0.2` | `75.119.132.91`, beta.thermograph.org | **compose** app stack, Forgejo (git + CI + registry), Grafana + Loki | +| **desktop** | `10.10.0.3` | — | LAN dev stack, the Forgejo Actions runner, your editor | +| phone | — | — | alerts | + +Two consequences you will hit within the first week: + +- **Forgejo is mesh-only.** `git.thermograph.org` resolves publicly to beta's + IP, but beta's Caddy rejects `/v2/*` (the registry API) from outside the mesh. + Any host that pulls images needs `10.10.0.2 git.thermograph.org` in + `/etc/hosts`. +- **Prod and beta run different orchestrators.** Prod is Swarm + (`infra/deploy/stack/`), beta and LAN dev are compose + (`infra/docker-compose.yml`). `deploy.sh` routes between them by reading + `/etc/thermograph/deploy-mode` on the box. Most of the time you don't care — + but when you're reading logs or naming containers, you do + (see [observability](09-observability.md)). + +## How a change travels + +``` +PR ──(required check: `gate`)──▶ dev ──▶ LAN dev box + │ + promotion PR + ▼ + main ──▶ beta.thermograph.org + │ + promotion PR ← the owner's call + ▼ + release ──▶ thermograph.org +``` + +`dev`, `main` and `release` are **protected** — no direct pushes, for humans or +agents. Promotion is by pull request, continuously and per-decision; there is +no release calendar. Full detail in [CI and release](07-ci-and-release.md). + +## The four rules that are not negotiable + +These come from the operator, via Centralis's own onboarding. They override +convenience. + +1. **Everything is a PR.** If it didn't go through the pipeline, it didn't + happen. This includes Grafana dashboards (provisioned from repo JSON — a UI + edit is silently overwritten) and secrets (SOPS vault only). +2. **Boring prod.** Anything clever proves itself on `dev` or beta first. +3. **Privacy is a feature.** No tracking cookies, no per-visitor IDs, + allowlisted anonymous counters only (`backend/core/metrics.py`, + `POST /api/v2/event`). Cheap to keep, expensive to retrofit. +4. **Grades are relative.** Never describe a percentile as hot or cold. + +### And a fifth, local to the repo + +**Commits and PRs are concise and technical, and never mention AI, assistants +or automated authorship.** Every domain `CLAUDE.md` says this; it applies to +you too. + +## Quota discipline + +Thermograph runs on free upstream data sources. Two hard rules protect them: + +- **Anything named "prefetch" or "warm" must never spend the Open-Meteo + quota.** The warming paths call `climate.load_cached_history` / + `load_cached_recent_forecast` — the cache-only loaders — and skip cells that + aren't warm. A cell self-heals on first real request instead. +- **Nominatim gets at most ~1 request/second.** All reverse geocoding funnels + through a single dedicated worker thread + (`climate._revgeo_worker`) that paces itself; never call it from a request + thread. + +Breaking either one gets the whole service rate-limited, which is why +`singleton.claim_leader()` exists: a background sweep running in N uvicorn +workers × M hosts multiplies quota use by N×M. That failure has already +happened once, in production, at 3 workers. + +## Where the written record lives + +| Question | Where | +|---|---| +| How does this repo work? | The domain `CLAUDE.md` files, then this set | +| Why is the architecture like this? | `thermograph-docs` (ADRs) — `docs_search` | +| How do I operate the fleet? | `thermograph-docs` runbooks, plus the `thermograph-ops` Centralis skill | +| What did someone find out last Tuesday? | `notes_search` | +| What is and isn't live yet? | [`CUTOVER-NOTES.md`](../../CUTOVER-NOTES.md) at the repo root | + +Next: [Local setup](02-setup.md). diff --git a/docs/onboarding/02-setup.md b/docs/onboarding/02-setup.md new file mode 100644 index 0000000..ba79e75 --- /dev/null +++ b/docs/onboarding/02-setup.md @@ -0,0 +1,217 @@ +# 2. Local setup + +Every command in this document was run against this checkout and produced the +output shown. If one fails for you, the difference is your machine, not the doc. + +## Toolchain + +| Tool | Why | Notes | +|---|---|---| +| **Python 3.12** | backend runtime and test suite | Must have the `_sqlite3` extension. A pyenv 3.10 without it will fail at `conftest` import. `scripts/test.sh` looks for `python3.12` (or uses `uv`) rather than whatever `python3` is on `PATH` — deliberately. | +| **Go 1.26** | frontend service and the backend daemon | Both are static `CGO_ENABLED=0` builds. | +| **Docker** | images, the compose/Swarm stacks, smoke tests | Docker CLI ≥ 27 is fine. | +| **uv** *(recommended)* | fast venv creation | `scripts/test.sh` uses it when present. | +| **shellcheck v0.11.0** | matches CI's pin exactly | Install to `~/.local/bin/shellcheck`. A *different* version can invent new findings and fail CI on an unrelated push. | +| **sops + age** | reading/editing the secrets vault | Only needed if you touch `infra/deploy/secrets/`. | +| **jq** | the repo's `.claude` hooks use it | Without it the prod-guard hook fails toward *asking*, which is safe but noisy. | + +Check what you have: + +```bash +which go python3.12 uv docker sops age jq shellcheck +go version && python3.12 --version && docker --version +``` + +## Backend + +### Test suite (hermetic, no network, no Docker) + +```bash +cd backend +make test # or: ./scripts/test.sh +make test ARGS='tests/data -q' # pass pytest args through +``` + +Verified: **429 passed, 8 skipped in 11.57s** on a cold venv build. + +The first run builds `.venv-test/` on Python 3.12 and installs +`requirements-dev.txt`. `tests/conftest.py` is what makes the suite hermetic: + +- no real Open-Meteo, Nominatim or GeoNames calls (the places index is marked + as already-loaded so no background download starts); +- a throwaway SQLite accounts DB and derived store in `/tmp`, never the repo's + `data/`; +- audit/error/access/activity/heartbeat log dirs redirected to `/tmp`; +- notifier and heartbeat threads disabled; +- `THERMOGRAPH_FRONTEND_BASE_INTERNAL` pointed at an unreachable placeholder + (the app fails loud at import without it). + +### Boot it locally, no Docker, no Postgres + +The backend falls back to SQLite when `THERMOGRAPH_DATABASE_URL` isn't a +Postgres URL, so this is enough: + +```bash +cd backend +THERMOGRAPH_FRONTEND_BASE_INTERNAL=http://127.0.0.1:8080 \ +THERMOGRAPH_BASE=/thermograph \ +THERMOGRAPH_ENABLE_NOTIFIER=0 \ +THERMOGRAPH_ENABLE_HEARTBEAT=0 \ + .venv-test/bin/python -m uvicorn app:app --host 127.0.0.1 --port 8137 +``` + +Verified responses: + +``` +GET /healthz → {"status":"ok","role":"all"} +GET /thermograph/api/version → {"backend_version":"2","min_frontend":"1","payload_ver":"p2"} +``` + +`app:app` is a one-line re-export shim for `web/app.py`. Keep it — systemd, CI +and the container entrypoint all target that name. + +Note the port: **8137** is the backend everywhere in this project (compose, +Caddy, the smoke harness at 18137, the frontend's default internal base). + +### Image smoke test + +```bash +cd backend +make smoke # builds the image, boots it + a throwaway TimescaleDB (tmpfs), + # asserts /healthz and /api/version +``` + +Uses `docker-compose.test.yml` on host port 18137 so it can't collide with a +dev server on 8137. + +## Frontend + +**The live frontend is Go.** `frontend/server/` is what builds, tests, ships +and runs. The Python files one level up (`app.py`, `content.py`, +`api_client.py`, `format.py`) are the superseded original — see +[traps](11-traps.md). + +### Test + +```bash +cd frontend/server +go build ./... && go vet ./... && go test ./... +``` + +Verified: all seven packages `ok` (`config`, `content`, `contentapi`, +`contentdata`, `format`, `handlers`, `render`). + +These same commands run inside `frontend/Dockerfile`'s builder stage — plus a +`gofmt -l` check that must come back empty — so **a failing Go test fails the +image build**, which is how CI catches it. There is no separate frontend test +step in the workflows. + +### Run it + +The process resolves `static/` and `content/` **relative to its working +directory**, so build in `server/` and run from `frontend/`: + +```bash +cd frontend/server && go build -o thermograph-frontend . +cd .. +THERMOGRAPH_API_BASE_INTERNAL=http://127.0.0.1:8137 \ +THERMOGRAPH_BASE=/thermograph \ +PORT=8080 \ + ./server/thermograph-frontend +``` + +Verified: `GET /healthz` → `{"status":"ok"}`, with a structured JSON log line +per request. + +`THERMOGRAPH_API_BASE_INTERNAL` is **required** — boot fails loudly without it, +by design. Optional: `THERMOGRAPH_BASE` (default `/thermograph`; the image sets +`/`), `THERMOGRAPH_API_VERSION` (default `v2` — only ever change it per the +[API-version contract](06-contracts.md)), `THERMOGRAPH_API_BASE_PUBLIC`, +`THERMOGRAPH_SSR_CACHE_TTL` (seconds, default 600), `THERMOGRAPH_GOOGLE_VERIFY`, +`THERMOGRAPH_BING_VERIFY`, `PORT` (default 8080). + +### Frontend against a real backend container + +```bash +cd frontend +make backend-up # pulls + runs the published backend image + throwaway db + # on 127.0.0.1:18137, waits for /healthz, prints the URL +make backend-down +``` + +`make test-integration` runs the Python integration tier against that. CI does +**not** run it — it needs a live backend container. + +## The daemon + +```bash +cd backend/daemon +go build ./... && go vet ./... && go test ./... + +THERMOGRAPH_INTERNAL_TOKEN=dev-token \ +THERMOGRAPH_API_BASE_INTERNAL=http://localhost:8137 \ + go run . +``` + +It refuses to start without `THERMOGRAPH_INTERNAL_TOKEN` — and the backend +answers `404` on the whole `/internal/*` surface when that token is unset. Both +ends fail closed. With Discord unconfigured it logs once and runs cron-only. + +## The full stack, locally + +```bash +cd infra +make dev-up # docker-compose.yml + docker-compose.dev.yml overlay: + # uncapped CPU, backend published on 0.0.0.0:8137 for the LAN +make dev-down +``` + +The dev overlay exports `COMPOSE_PROJECT_NAME=thermograph-dev` so the LAN stack +keeps volumes separate from anything else. **Do not remove either half of the +project-name pinning** — `infra/docker-compose.yml` pins `name: thermograph`, +and without it running compose from `infra/` derives the project name `infra`, +silently creating a *new* stack beside the running one with fresh volumes. + +Other `infra/Makefile` targets: `up`/`down` (pull + run the published images), +`db-up`/`db-down` (just Postgres, e.g. to run the app from a venv against it), +`om-up`/`om-down`/`om-backfill` (the self-hosted Open-Meteo overlay — the +backfill writes ~1–1.5 TB and takes hours). + +## Connectors (Centralis and friends) + +The fleet is not reachable from a laptop off the WireGuard mesh, and Forgejo is +mesh-only. **Centralis** is the control plane that fronts all of it — the app +database, the ERA5 lake, fleet logs, Grafana, Forgejo, docs and notes, and +Discord. + +```bash +claude mcp add --transport http centralis https://mcp.thermograph.org/mcp \ + --header "Authorization: Bearer $CENTRALIS_TOKEN" +``` + +Ask the operator for a token; don't share it. Verify with *"what's running on +prod right now?"* — it should call `fleet_status` and list the Swarm services. + +Also worth installing locally: **Chrome DevTools MCP** (design verification +needs a real browser on your machine) and **Figma**. Grafana's official MCP +server is optional and read-only — but remember dashboards are provisioned from +repo JSON, so a durable change is still a PR via `dashboard_write`. + +Run `mcp__centralis__onboarding` for the current, authoritative connector list; +it will be fresher than this page. + +## Repo-local guardrails + +`.claude/settings.json` wires three hooks that travel with the checkout: + +| Hook | When | What | +|---|---|---| +| `prod-guard.sh` | before Bash / live-host MCP calls | Classifies by **allowlist**: only positively-recognised read-only commands pass; everything else asks. Beta is guarded as strictly as prod — it hosts Forgejo, Grafana *and* beta.thermograph.org. | +| `secrets-guard.sh` | before Write/Edit | Denies any direct write to `infra/deploy/secrets/*.yaml`. Use `sops edit`. | +| `lint-after-edit.sh` | after Write/Edit | shellchecks an edited `*.sh` and feeds findings straight back. Exits quietly if shellcheck is missing. | + +They enforce what `CLAUDE.md` can only ask for. If you change `prod-guard.sh`'s +classifier, re-read `.claude/hooks/README.md` first — it documents a real +silent-total-bypass failure mode in the parsing loop. + +Next: [Repo map](03-repo-map.md). diff --git a/docs/onboarding/03-repo-map.md b/docs/onboarding/03-repo-map.md new file mode 100644 index 0000000..65cebc4 --- /dev/null +++ b/docs/onboarding/03-repo-map.md @@ -0,0 +1,189 @@ +# 3. Repo map + +The monorepo was reunified on **2026-07-22** by subtree-merging four split +repos with full history (380 commits). The split's *properties* were kept +deliberately: per-domain images, per-domain deploys, and an async FE/BE +contract. That's why the layout looks like four projects sharing a root. + +``` +thermograph/ +├── CLAUDE.md root agent instructions — branch model, deploy contract +├── CUTOVER-NOTES.md what is and isn't live; the reunification record +├── README.md domain table + how CI stays decoupled +├── .forgejo/workflows/ ALL CI. Nine files. Never add workflows elsewhere. +├── .claude/ settings.json + enforcement hooks +├── backend/ FastAPI API + Go daemon → emi/thermograph/backend +├── frontend/ Go SSR + static assets → emi/thermograph/frontend +├── infra/ compose, Swarm, deploy, secrets, terraform, ops +├── observability/ Loki + Grafana + Alloy +└── docs/onboarding/ you are here +``` + +> **CI lives only in root `.forgejo/workflows/`**, path-filtered per domain. +> Forgejo only reads root workflows — a copy under a domain's own `.forgejo/` +> is inert, and becomes a trap when someone edits it expecting an effect. + +## `backend/` — the API service + +Python 3.12 / FastAPI. Owns climate data, grading, the API, accounts, +notifications, and the SSR-content JSON API the frontend renders from. Serves +**no** HTML/CSS/JS. + +``` +app.py one-line re-export shim for web/app.py — keeps `app:app` + stable for systemd/CI/entrypoint. Don't change the name. +paths.py every filesystem location, resolved from the repo root. + NEVER reintroduce __file__-relative paths in a module. +web/app.py (1022) the real FastAPI app: routes, CORS, ETag, middleware, + lifespan, role gating, the frontend proxy fallback +api/ + payloads.py (342) pure inputs → response dict builders. One definition per + payload shape; PAYLOAD_VER lives here. + content_payloads.py the SSR content API's builders (JSON-safe, no Markup) + content_routes.py route wiring for /content/* + the same ETag pattern + internal_routes.py /internal/* — the Go daemon's control surface. Fails + closed: no token configured ⇒ the whole surface 404s. + homepage.py (367) the precomputed "unusual right now" feed + sitemap.py +data/ + climate.py (1248) fetch + cache the daily record. The source ladder lives + here. The single biggest file in the repo. + grading.py (532) day-of-year climatology, percentiles, TEMP/RAIN_BANDS + grid.py (92) lat/lon → stable ~4 sq mi cell id + store.py (329) derived-payload store (SQLite/Postgres). Pure accelerator. + climate_store.py(353) raw climate record on TimescaleDB hypertables + era5lake.py (173) the ERA5 object-storage lake: layout, grid math, clients + scoring.py (293) climate-shift scoring (recent years vs full baseline) + places.py (355) typo-tolerant local place index for /suggest + cities.py, city_events.py, meteostat.py +core/ + metrics.py (805) since-start traffic counters, multi-worker via SQLite + audit.py (167) JSONL audit / errors / access / activity / heartbeat + singleton.py (129) leader election: flock (per host) or PG advisory lock + (per cluster). Read this before adding any timer. +accounts/ fastapi-users: models, schemas, db, users, api_accounts +notifications/ + notify.py (542) the subscription evaluation engine (background thread) + push.py, mailer.py, digest.py, discord.py, discord_link.py, + discord_interactions.py +daemon/ the Go daemon — Discord gateway + cron timers +alembic/ Postgres migrations (run by the entrypoint on boot) +deploy/entrypoint.sh container entrypoint: alembic migrate, then serve +lake_app.py (204) the prod-only `lake` service (DuckDB over the bucket) +tests/ the hermetic pytest suite +cities.json, cities_flavor.json bundled reference data (generated) +``` + +Standalone scripts at the root are tooling, not runtime: `warm_cities.py`, +`indexnow.py`, `seed_era5.py`, `gen_era5_lake.py`, `gen_cities.py`, +`gen_flavor.py`, `drift_check.py`, `migrate*.py`. + +Deep dive: [04-backend.md](04-backend.md). + +## `frontend/` — SSR + static assets + +``` +server/ ★ THE LIVE SERVICE (Go 1.26, module thermograph/frontend) + main.go config + mux + static + graceful shutdown + internal/config/ every env var, with the same names/defaults as before + internal/contentapi/ backend /content/* client: TTL cache, bounded LRU, + per-key single-flight, origin forwarding + internal/content/ page handlers, funcmap, SEO + internal/contentdata/ glossary.yaml + pages.yaml loader (fail-loud) + internal/format/ unit-aware °C/°F formatting + band names + internal/handlers/ routing, SPA shells, static + internal/render/ html/template over embed.FS + ETag helpers + internal/render/templates/*.tmpl the embedded page templates +static/ every static asset — app.js, cache.js, shared.js, + account.js, calendar/day/score/compare .js + .html + shells, style.css (all design tokens), icons, sw.js +content/ glossary.yaml, pages.yaml — structured SSR copy +templates/*.html.j2 Jinja templates — used by the PYTHON service only +tests/ Python tiers: unit (hermetic, fixtures) + integration +tests/fixtures/*.json golden payloads — ALSO consumed by the Go tests +tools/shoot.py the `make shots` screenshot sweep +DESIGN.md the visual system: tokens, components, breakpoints +app.py, content.py, ⚠ the superseded Python implementation — not deployed +api_client.py, (see traps) +format.py, paths.py, +content_loader.py +``` + +Deep dive: [05-frontend.md](05-frontend.md). + +## `infra/` — how and where images run + +``` +docker-compose.yml the compose stack (beta + LAN dev): + db, backend, lake, daemon, frontend. + `name: thermograph` is PINNED — see below. +docker-compose.dev.yml LAN overlay: uncapped, backend on 0.0.0.0:8137 +docker-compose.openmeteo.yml self-hosted Open-Meteo overlay (prod-only) +Makefile compose orchestration only +deploy/ + deploy.sh ★ the single entry point for beta and prod + render-secrets.sh renders /etc/thermograph.env from the SOPS vault + secrets/ the vault: common.yaml, prod/beta/dev.yaml, + centralis.prod.yaml, example.yaml + stack/ the SWARM path (live on prod): + thermograph-stack.yml, deploy-stack.sh, + autoscale.sh, the LB + swarm/, forgejo/ mesh + Forgejo/CI-runner provisioning + Caddyfile the reverse proxy (path-splits FE vs BE) + provision-*.sh host bootstrap: agent access, dev LAN, mail, secrets + db/init/ TimescaleDB init + tuning + migrations/ hand-run SQL migrations +ops/ + dbq.sh read-only psql into any environment's db container + iceberg.sh duckdb over the ERA5 Iceberg lake +terraform/ host provisioning — NO STATE IS PERSISTED +ACCESS.md, DEPLOY.md, DEPLOY-DEV.md (partly stale — see traps) +``` + +**Why the project name is pinned.** Compose creates volumes +`thermograph_pgdata` / `_appdata` / `_applogs`, and the Swarm stack declares +those exact names as `external: true` at the same mount paths. Running compose +from `infra/` without `name: thermograph` derives project `infra`, which makes a +whole new stack with empty volumes next to the running one. `deploy-dev.sh` +exports `COMPOSE_PROJECT_NAME=thermograph-dev` (env wins over the file key) to +keep LAN dev separate on purpose. Keep both halves. + +Deep dive: [08-infra-secrets.md](08-infra-secrets.md). + +## `observability/` — the logging stack + +``` +docker-compose.yml Loki + Grafana (runs on beta) +loki/config.yml mesh-only, filesystem storage, 30-day retention +grafana/provisioning/ datasource + dashboard provider (auto-loaded) +grafana/provisioning/alerting/ rules, the Discord contact point, the policy +grafana/dashboards/*.json the fleet-logs dashboard +alloy/config.alloy the per-node shipper +alloy/docker-compose.agent.yml runs Alloy on a node (ALLOY_NODE per host) +caddy-grafana.conf beta's Grafana vhost, kept for reference +``` + +No build, no deploy automation — it ships by hand. **The repo is the only +durable path**: Grafana is provisioned from this directory at startup, so UI +edits are overwritten. Grafana is at **`dashboard.thermograph.org`** (Google +SSO, pre-provisioned users only) — never `grafana.thermograph.org`. + +Deep dive: [09-observability.md](09-observability.md). + +## The nine CI workflows + +| Workflow | Trigger | Does | +|---|---|---| +| `pr-build.yml` | PR → `dev`/`main` | The `gate` required check. Diffs the PR, builds only touched domains. Deliberately **not** path-filtered. | +| `build.yml` | `workflow_call` | Build one domain's image; run the backend suite **inside** the built image. | +| `build-push.yml` | push to `dev`/`main`/`release` touching an app domain, or a `v*.*.*` tag | Builds + pushes `sha-<12hex>` images. A version tag builds **both**. | +| `deploy.yml` | push to `main`/`release` touching an app domain | Branch selects environment; matrix covers services; SSHes and runs `deploy.sh`. | +| `infra-sync.yml` | push to `main` touching `infra/**` | Fast-forwards each host's checkout and re-renders secrets. Rolls **no** service. | +| `observability-validate.yml` | push touching `observability/**`, or `workflow_call` | Parses every artifact; validates Alloy config with the pinned binary; strict alerting checks. | +| `ops-cron.yml` | daily 03:00 UTC | **THE prod backup** (pg_dump) + IndexNow. Uses `PROD_SSH_*`. | +| `secrets-guard.yml` | every PR and push | Fails if any `infra/deploy/secrets/*.yaml` isn't SOPS-encrypted. | +| `shell-lint.yml` | every PR and push | shellcheck (pinned v0.11.0 + sha256) over every `*.sh`. | + +Deep dive: [07-ci-and-release.md](07-ci-and-release.md). + +Next: [Backend deep dive](04-backend.md). diff --git a/docs/onboarding/04-backend.md b/docs/onboarding/04-backend.md new file mode 100644 index 0000000..b6e59b4 --- /dev/null +++ b/docs/onboarding/04-backend.md @@ -0,0 +1,292 @@ +# 4. Backend deep dive + +FastAPI on Python 3.12, plus a Go daemon that ships in the same image. Owns +every piece of climate data and every bit of grading logic in the project. + +Read [`backend/CLAUDE.md`](../../backend/CLAUDE.md) alongside this — it is the +binding, terse version. + +## The one-process-many-roles model + +The backend image runs **five different jobs** depending on `THERMOGRAPH_ROLE` +and which binary compose/Swarm selects. One image means version skew between +them is impossible. + +| Service | How it's selected | Does | +|---|---|---| +| **web** | `THERMOGRAPH_ROLE=web` | Serves requests. Never starts the notifier, even if it would win the election — so the web tier scales to N replicas without also scaling background sweeps. | +| **worker** | `THERMOGRAPH_ROLE=worker` | Owns the subscription notifier. Still serves requests today. | +| **all** | default | Both — the single-process dev default. | +| **lake** | `THERMOGRAPH_ROLE=lake` | `deploy/entrypoint.sh` execs `uvicorn lake_app:app` on port 8141 instead. No database, no migrations. Prod only. | +| **daemon** | compose/Swarm sets the command to `/usr/local/bin/thermograph-daemon` | The Go binary: Discord gateway + cron timers. | + +`deploy/entrypoint.sh` runs `alembic upgrade head` (retried — a fresh Postgres +volume can still be starting) and then serves. `/healthz` and `/api/version` +are deliberately **I/O-free** so they stay cheap under tight healthcheck +intervals; they're also deliberately **not** under `THERMOGRAPH_BASE`, so a +healthcheck needn't know the base path. + +## The data pipeline + +### Where history comes from — the source ladder + +`data/climate.py::_load_history` walks this ladder on a cache miss, and the +order matters: + +1. **The ERA5 lake** (`data/era5lake.py`) — our own parquet extract of the + public ERA5 archive in Contabo object storage. True ERA5 with measured + gusts, no third-party API in the path. Unconfigured or missing-point cells + fall through at zero cost. +2. **The Open-Meteo archive** — the same ERA5 family, so a backup-sourced + record grades consistently with lake-sourced neighbours. Guarded by its own + rate-limit cooldown. +3. **Stale cache**, served without rewriting it (so `recent_stamp` stays old + and the next request retries upstream first). +4. Otherwise `WeatherUnavailable` → a clean 503. + +**NASA POWER is retired from every serving path.** Its MERRA-2 record ran +~1.3 °F off ERA5 and carried no gusts. `drift_check.py` still fetches it, and +only that. + +Every source must return a plausibly-full span (`MIN_ARCHIVE_DAYS`) before it's +accepted and cached as complete — a short response is rejected, not stored. + +### Where "today" comes from + +`_load_recent_forecast` is a separate ladder: + +1. **Open-Meteo forecast API** — recent past *and* forward days in one call, + with gusts, ERA5-consistent with the history record. Skipped while its own + cooldown is active. +2. **Fallback** — an archive recent-past range plus **MET Norway** forward days. + Costs no Open-Meteo forecast quota. +3. Stale cache, again without rewriting. + +Cached per cell for `FORECAST_TTL_HOURS`, so it tracks model updates without +hourly upstream I/O. `RECENT_PAST_DAYS = 25`, `FORECAST_DAYS = 8`. + +### Concurrency and quota guards + +- A per-cell lock serialises archive fetches, so N concurrent first-requests for + one cell cost one fetch. +- Rate limits set module-level cooldowns (`_archive_cooldown_until`, + `_forecast_cooldown_until`) rather than retrying into the wall. +- Reverse geocoding runs on **one dedicated worker thread** draining a queue at + ~1 req/s. The earlier design slept inside the caller's thread — which, in the + server, is one of Starlette's shared sync-threadpool threads. + +## Storage: three tiers with three different contracts + +| Tier | Module | Backend | Contract | +|---|---|---|---| +| **Raw climate record** | `data/climate_store.py` | TimescaleDB `climate_history` hypertable (durable), `climate_recent` (rewritten each refresh), `climate_sync` (freshness) | **Source of truth.** Expensive to refetch. Fail-soft: a miss degrades to *fetch-from-upstream*. Parquet files instead when `THERMOGRAPH_DATABASE_URL` isn't Postgres (dev, tests, offline tooling). | +| **Derived payloads** | `data/store.py` | SQLite WAL, or **UNLOGGED** Postgres tables | **Pure accelerator.** Every reader falls back to recomputing from the raw record; every helper swallows its own errors. Deleting the store is a safe full reset. | +| **Accounts** | `accounts/db.py` | Postgres (or SQLite) via SQLAlchemy | **Not regenerable.** Foreign keys enforced, errors surface rather than get swallowed, must be backed up. | + +That third row is the important distinction: accounts, subscriptions and +notifications have no source to recompute from, so they live under stricter +rules than everything else in the process. + +On Postgres, `accounts/db.py` runs **two async engines** over one primary — +read-only (`default_transaction_read_only`) for pure-GET endpoints, read-write +for everything else — plus a separate **sync** psycopg engine for the notifier +thread, which has no event loop. + +### The validity token + +Derived rows are keyed `(kind, cell_id, request_key)` and guarded by a `token` +that encodes everything the payload depends on: + +```python +PAYLOAD_VER = "p2" # api/payloads.py +history_token(history) → f"{PAYLOAD_VER}:{hist_end(history)}" +content_token(...) → f"{PAYLOAD_VER}:{CONTENT_VER}:{max_date}" +``` + +A stored token that doesn't match the caller's current token is a **miss**. +There is no clock-based expiry anywhere. That's why one `PAYLOAD_VER` bump +atomically orphans every pre-upgrade row — and why forgetting to bump it serves +old-shaped JSON forever. `CONTENT_VER` is kept separate so a content-only shape +change doesn't invalidate the whole grading cache. + +The same token doubles as a **weak ETag**. See [contracts](06-contracts.md) for +what that means for CORS. + +## Grading + +`data/grading.py` is the heart of the product: + +- For a target day of year, the reference distribution is every historical day + within **±`HALF_WINDOW` (7) days**, wrapping the year boundary — a 15-day + seasonal window. +- The observed value is placed as an **empirical mid-rank percentile**, which + handles ties correctly (crucial: most days have zero precipitation). +- `TEMP_BANDS` is a symmetric seven-tier ladder; `RAIN_BANDS` is separate, and + dry days are coloured by dry-streak length instead of rank. +- `pct_ordinal()` floors a percentile into `1..99` — never 0, never 100. + `frontend/static/shared.js::pctOrd()` mirrors it. If they diverge, the same + reading says two different things on two surfaces. + +`data/scoring.py` is a different question: how far a cell's **recent years** +have drifted from its full multi-decade baseline, per metric and per +percentile category, expressed in unit-free percentile points. Note +`baseline_overlaps_recent` in the payload — the baseline deliberately includes +the recent years, which mildly and uniformly attenuates the divergence. + +## The HTTP surface + +`web/app.py` mounts routers so that every version stays served simultaneously: + +``` +/api → v1 (legacy unversioned alias) +/api/v1 → v1 geocode, grade +/api/v2 → v2 geocode, suggest, place, grade, calendar, day, + score, forecast, cell, metrics, event +/api/v2 → content_routes: /content/hub, /content/sitemap, + /content/indexnow-key, /content/home, + /content/city/{slug}[/month/{month}|/records] +/api/v2/auth, /users fastapi-users: login, logout, register, verify, me +/api/v2/subscriptions, /notifications, /push/* accounts/api_accounts.py +/api/v2/discord/* account linking (OAuth2) +/internal/* the Go daemon's control surface — NOT under BASE +/healthz, /api/version I/O-free +/discord/interactions Discord slash-command webhook (Ed25519-verified) +/digest the footer signup form +/{path:path} catch-all proxy to the frontend ← registered LAST +``` + +Two routes worth understanding properly: + +**`GET /api/v2/cell`** — one bundle carrying every view's payload for a cell, so +the client warms four views with one request. Each slice is the *exact* payload +its per-view endpoint returns, built by the same builders under the same +derived-store keys and tokens, paired with the ETag that endpoint would emit. +The client seeds its per-view cache from the slices and later revalidates each +view individually. `prefetch=1` is the warm-only mode with a hard guarantee: +**it never spends weather-API quota** — a cold cell answers `204` with no body. +`neighbors=1` additionally enqueues the 8 surrounding cells for background +warming, also warm-only. + +**The catch-all proxy** — the frontend owns every page and asset. In prod and +beta, Caddy path-splits directly, so this proxy is never exercised. It exists as +the fallback for environments with no Caddy in front (LAN dev, bare-metal). It +forwards `X-Forwarded-Host`/`-Proto` so the frontend can build correct absolute +URLs instead of resolving to the internal hop's own address. Because it's +registered dead last, every real backend route wins first — +`/internal/*` in particular is registered before it, deliberately. + +## Background work: who runs what, and where + +This is where the project's hardest-won lessons live. + +- **Neighbour warmer** — a per-worker daemon thread draining a request-fed + queue. Correct to run in every worker: each drains its own queue. +- **Subscription notifier** (`notifications/notify.py`) — a timer-driven sweep + that issues upstream fetches independent of any request. Running it in N + workers multiplies Open-Meteo quota use N-fold. That is not hypothetical: it + is the documented cause of overnight 429/503s after prod went to 3 workers. + So it runs behind `singleton.claim_leader()`: + - `claim()` — a non-blocking `flock` on a lockfile, arbitrating workers **on + one machine**; + - `claim_pg()` — a Postgres advisory lock, visible to every host talking to + the same database, arbitrating **cluster-wide** under Swarm; + - unset (dev, tests, single worker) ⇒ always the leader. +- **Heartbeat** — deliberately **unguarded** across every worker and replica. A + beat is one local file append (no quota, no DB, no lock), so duplicates are + free, and only the process actually being dead stops them. Container-log + silence can't distinguish a live web/worker process from a dead one, so this + is the signal observability alerts on. +- **Discord gateway + recurring jobs** — moved *out* of this process entirely + into the Go daemon. See below. + +The notifier's own quiet guards: a `UNIQUE(subscription_id, event_date, metric, +direction, kind)` row-level dedup, and a **weekly cap** — a subscription that +notified within 7 days is skipped entirely. Archive reads come from cache; a +freshly-subscribed cell is fetched **once**, budget-capped per pass. + +## The Go daemon (`backend/daemon/`) + +One binary, built into the backend image, running as exactly one replica. + +**Why it exists:** the Discord gateway tolerates one session per token, and +duplicated timers repeat every job. One replica makes that a *deployment fact* +instead of a runtime election. + +**Why it's thin:** Go owns no climate or grading logic. Grading needs polars and +the cache, and the slash-command path deliberately shares one grade builder with +the API so bot grades and API grades can never drift. So anything data-shaped is +a POST back into Python: + +| Route | Purpose | +|---|---| +| `POST /internal/discord/grade` | `{"query": "phoenix"}` → gateway-ready Discord message JSON, relayed verbatim | +| `POST /internal/jobs/warm-cities` | trigger the warm-cities job | +| `POST /internal/jobs/indexnow` | trigger IndexNow check-and-submit | + +Every request carries `X-Thermograph-Internal-Token`. **Both ends fail closed**: +with no `THERMOGRAPH_INTERNAL_TOKEN`, Python answers 404 across the whole +surface and the daemon refuses to start. Caddy never routes `/internal/*` +publicly, so the token is defence in depth, not the only control. + +The cron half runs unconditionally (`warm-cities` every 24h, `indexnow` every +6h, both configurable). A fatal gateway error logs loudly and leaves cron +running — a Discord misconfiguration must not become a crash loop that also +stops the schedule. The cron scheduler defers the first run by one interval, +never overlaps a job with itself, and drops ticks that fire mid-run. + +## The lake (`lake_app.py` + `data/era5lake.py`) + +Prod-only Swarm service, same image, `THERMOGRAPH_ROLE=lake`. + +The lake is parquet in an S3-compatible bucket (Contabo, `era5-thermograph`), +extracted once from the public Earthmover ERA5 Icechunk archive by +`gen_era5_lake.py`. Two layouts under `era5/`: + +- `daily/tile=_/year=/month=/part.parquet` — the analytical hive + table. The location grain is a 12×12-point tile (~3°×3°) aligned to the + source's chunk grid; a per-point-per-month grain would be ~170M objects. +- `points/lat=/lon=.parquet` — the serving projection: one file per grid + point with its full history, so the hot path is a single GET. +- `manifest.parquet` — an index of every point file; readers prune with it and + the extractor resumes from it. + +Endpoints: `/healthz`, `GET /history?lat=&lon=` (the hot path — nearest point's +history as parquet bytes, disk-cached per point), `POST /query` (SELECT-only +DuckDB SQL over `era5_daily` and `manifest`). + +Two operational facts: **Contabo is path-style only** with `region=default` +(virtual-host addressing fails outright), and DuckDB's `httpfs`/`iceberg` +extensions are baked into the image **as uid 10001**, because extensions install +to the invoking user's `~/.duckdb` and a root-time bake left the runtime user +unable to `LOAD` them — seen live as prod's lake `/query` returning 500. + +Filter on partition columns (`tile`/`year`/`month`). That predicate is the +difference between a pruned scan and reading the whole warehouse. + +## Observability hooks inside the app + +- `core/audit.py` writes JSONL streams: `audit` (one line per graded run, with + per-phase timings and a `full`/`partial` run type), `errors` (upstream + failures tagged `retry` or `error`), plus access, activity and heartbeat. + Alloy parses these so `level`/`tag`/`phase` become Loki labels. +- `core/metrics.py` keeps since-start counters, shared across uvicorn workers + via `THERMOGRAPH_METRICS_DB` (a SQLite file) — unset keeps the + zero-dependency in-memory store. Read over the token-gated + `GET /api/v2/metrics`. +- Every unhandled 500 gets an explicit exception handler emitting + `tag="http.error"`. Without it the exception propagates past the middleware + and the 500 is invisible in both the access log and metrics. + +## Working on it + +```bash +cd backend +make test # hermetic pytest, ~12s +make test ARGS='tests/data/test_grading.py -q' +make smoke # build the image, boot it + a throwaway db +``` + +CI runs the suite **inside the built image**, so the exact interpreter and +dependencies that ship are what gets tested. + +Next: [Frontend deep dive](05-frontend.md). diff --git a/docs/onboarding/05-frontend.md b/docs/onboarding/05-frontend.md new file mode 100644 index 0000000..8241b54 --- /dev/null +++ b/docs/onboarding/05-frontend.md @@ -0,0 +1,221 @@ +# 5. Frontend deep dive + +The **SSR + static-asset service**: server-rendered crawlable pages, the +interactive tool's SPA shells, and every static asset. No climate data, no +database, no compute — everything comes from the backend's content API over +HTTP. + +> **The live service is Go.** `frontend/server/` builds, tests, ships and runs. +> The Python files one directory up (`app.py`, `content.py`, `api_client.py`, +> `format.py`, `content_loader.py`, `paths.py`, `templates/*.html.j2`) are the +> superseded original implementation, kept in-tree as the reference the port +> was made from. `frontend/CLAUDE.md` and `frontend/README.md` still describe +> the Python service — see [traps](11-traps.md). Everything in those docs about +> **contracts, env vars and design** is still correct; only "which language and +> which file" changed. + +## Shape of the Go service + +``` +main.go config → content client → glossary/pages → mux → + listen → graceful shutdown +internal/config/ every env var the service reads, with the same names, + defaults and required/optional split as the Python +internal/contentapi/ the backend /content/* client +internal/content/ the SSR page handlers, template funcmap, SEO helpers +internal/contentdata/ glossary.yaml / pages.yaml loader +internal/format/ unit-aware °C/°F formatting + band names +internal/handlers/ SPA shells + static serving +internal/render/ html/template over an embed.FS + ETag helpers +internal/render/templates/*.tmpl the page templates (embedded in the binary) +``` + +Dependencies: **stdlib plus `gopkg.in/yaml.v3`** only. The committed SSR copy +in `frontend/content/*.yaml` is shared with the rest of the repo and uses +block/folded scalars, so a real YAML parser is genuinely required. + +**Templates are embedded in the binary; `static/` and `content/` are read from +disk relative to the working directory** (`frontend/` locally, `/app` in the +image). That's why you build in `server/` and run from `frontend/`. + +### Fail-loud boot + +`main.go` exits at startup on bad configuration *or* bad content: a missing +backend URL, a garbage TTL, malformed glossary/pages YAML. The Python raised at +import for the same cases, and the reasoning is identical — a bad content edit +should break the boot, not silently 500 on the first request. + +The one deliberate exception is the IndexNow key: boot tries the key fetch once, +catches failure, and falls back to a lazy per-request lookup. **Boot must +survive an unreachable backend** — asynchronous FE/BE deploys depend on it. +Don't make it fatal. + +## Routes + +Everything below sits under `THERMOGRAPH_BASE` (default `/thermograph`; the +image sets `/`). + +| Path | Kind | +|---|---| +| `/` | SSR homepage | +| `/climate` | SSR climate hub | +| `/climate/{slug}` | SSR per-city page | +| `/climate/{slug}/records` | SSR all-time records | +| `/climate/{slug}/{month}` | SSR per-month page | +| `/glossary`, `/glossary/{term}` | SSR | +| `/about`, `/privacy` | SSR | +| `/robots.txt`, `/sitemap.xml` | generated | +| `/{indexnow-key}.txt` | the IndexNow ownership file | +| `/calendar`, `/day`, `/score`, `/compare`, `/legend`, `/alerts` | SPA shells | +| `/healthz` | liveness, I/O-free (does **not** prove the backend is reachable) | +| everything else | static assets | + +The static mount is registered last so the explicit routes win. + +**The backend owns the page metadata, not this service.** `page_title`, +`canonical_path`, `breadcrumb` and `jsonld` come from +`backend/api/content_payloads.py`. Don't recompute them here. + +### SPA shells and `__ORIGIN__` + +The shell HTML files in `static/` carry an `__ORIGIN__` placeholder for the +link-preview/Open Graph tags — preview crawlers need absolute URLs and the host +differs between LAN and prod. The substituted HTML and its ETag are **memoized +per origin**, not recomputed per request. + +Origin resolution prefers `X-Forwarded-Host` over `Host`. That matters when the +request arrives through the backend's catch-all proxy fallback: without it, the +page would advertise the internal hop's own address as canonical. + +## The content client + +`internal/contentapi` is a careful piece of code and worth reading in full +before you change caching behaviour. It has four properties, all load-bearing: + +1. **TTL cache** (`THERMOGRAPH_SSR_CACHE_TTL`, default 600s). Climate data + changes at most hourly, so without this a page-view burst on one city costs + one backend round trip per request. This is *in addition to* the backend's + derived-store/ETag caching — this one saves the network hop, the backend's + saves the recompute. +2. **Bounded LRU** (2048 entries). `city()` and `city_records()` fold the + browser-facing `origin` into the cache key, and `origin` is client-controlled + via `Host`/`X-Forwarded-Host`. An unbounded map there is a cheap + memory-exhaustion vector. +3. **Per-key single-flight.** Without it, N concurrent requests missing the same + key (cold start, or right after TTL expiry on a hot city) each fire their own + backend call — a thundering herd. +4. **Origin forwarding.** The original browser-facing origin is passed through as + `Host` / `X-Forwarded-Proto` so the backend builds JSON-LD `url` fields + against the public origin rather than the internal `http://backend:8137` + address. + +## Static assets + +Hand-written, no build step, no bundler, no framework. + +| File | Role | +|---|---| +| `app.js` | the main tool: map, search, results, inline SVG chart | +| `calendar.js` / `day.js` / `score.js` / `compare.js` | the SPA views | +| `cache.js` | IndexedDB cache + the `/cell` bundle prefetch + `If-None-Match` revalidation | +| `account.js` | auth; **pins `API_VERSION` and the `uv(path)` helper** | +| `shared.js` | shared helpers; **`pctOrd()` mirrors the backend's `pct_ordinal()`** | +| `units.js` | `F_REGIONS` — must match the backend's `F_COUNTRIES` | +| `chart.js`, `mappicker.js`, `filtersheet.js`, `nav.js`, `push-client.js`, `ios-install.js`, `digest.js`, `climate.js`, `subscriptions.js` | the rest | +| `style.css` | **the single hand-written stylesheet — every design token lives here** | +| `sw.js`, `manifest.webmanifest`, icons | PWA | + +`cache.js` is where the ETag contract actually lands: it fetches +`/api/v2/cell` once per view-set, slices it, and later revalidates each view +with `If-None-Match`. The slice→view map must track the backend's payload +shape. See [contracts](06-contracts.md). + +Static responses are served with `Cache-Control: public, max-age=300` — short +on purpose, because asset filenames are **not** content-hashed. It's a +revalidation window, not an immutable cache. + +## The design system + +[`frontend/DESIGN.md`](../../frontend/DESIGN.md) is the source of truth and +wins over any general design guidance. The essentials: + +- **Never hardcode a colour.** Every colour is a CSS custom property in + `style.css`'s `:root`, with a `@media (prefers-color-scheme: light)` override + right below. Always `var(--token)`. +- **Grade palettes encode meaning, not decoration.** Temperature is a 9-step + diverging, colourblind-safe scale (`--rec-cold` … `--normal` green midpoint … + `--rec-hot`). Precipitation is `--dry` plus `--wet-1…9`. Keep the order and + midpoints intact. +- **Inter is not self-hosted.** It renders where the OS has it and falls back to + `system-ui`. Adding a webfont link is a deliberate decision, not a tweak — it's + a network dependency on every page. +- **Inputs are `font-size: 16px` minimum.** Smaller text makes iOS zoom on + focus. Hard rule. +- **Mobile-first.** The base stylesheet is the phone layout; wider screens layer + on via `min-width`. Primary breakpoint 640/641px. The content column widens in + real steps at 1680 / 2400 / 3400px — it does *not* stay a centred 1200px strip. +- **Metric order is always `Precip · High · Low`**, everywhere. +- **Dark is the default**, light comes from `prefers-color-scheme`. Every change + must look right in both. Honour `prefers-reduced-motion`. +- Charts are bespoke inline SVG. No canvas, no charting library. + +### Verifying a visual change + +**Validate at 390 / 800 / 1920 / 2560 / 3840px in both light and dark.** These +straddle every breakpoint above. + +```bash +cd frontend +.venv/bin/pip install -q -r tools/requirements.txt +.venv/bin/python -m playwright install chromium +.venv/bin/python tools/shoot.py # the full matrix +.venv/bin/python tools/shoot.py index --width 390 --scheme dark # while iterating +``` + +PNGs land in `.screenshots/` (gitignored) as `{page}@{width}-{scheme}.png`. +Point it elsewhere with `SHOTS_BASE`. Chrome DevTools MCP is the other good +option here — a real browser on your own machine. + +## Tests + +```bash +cd frontend/server && go build ./... && go vet ./... && go test ./... +``` + +The Go tests read two directories via a fixed relative path: the committed +golden fixtures (`frontend/tests/fixtures/*.json` — the same set the Python +golden-diff comparison used) and the SSR copy (`frontend/content/*.yaml`). +`frontend/Dockerfile`'s builder stage copies them to container-root paths to +preserve that relationship, and runs `gofmt -l` + `go vet` + `go test` **before** +building. A failing Go test therefore fails the image build, which is how CI +catches it — there is no separate frontend test step in the workflows. + +The Python tiers still exist and still run locally: + +```bash +cd frontend +make test-unit # hermetic: SSR rendering fed committed fixtures +make test-integration # pulls + runs the real backend image; local only +make capture-fixtures # refresh tests/fixtures/*.json from a live backend +``` + +Treat `make capture-fixtures` as the shared operation: those fixtures feed the +**Go** tests too. + +## The image + +`python:3.12-slim` → gone. The frontend image is: + +- a `golang:1.26` builder that runs `gofmt -l` / `go vet` / `go test`, then + builds a static `CGO_ENABLED=0 -trimpath -ldflags="-s -w"` binary; +- an `alpine:3.22` final stage with **bash** (the Swarm stack bind-mounts a + bash entrypoint shim over the image's entrypoint) and **curl** (the + healthcheck); +- **uid 10001**, matching the backend image — that's the uid infra provisions + readable secrets for. Don't change it. +- `static/` and `content/` copied to `/app`, read-only at runtime. + +There is no `WORKERS` knob any more: uvicorn needed a process count, the Go +server handles concurrency in one process. + +Next: [Cross-service contracts](06-contracts.md). diff --git a/docs/onboarding/06-contracts.md b/docs/onboarding/06-contracts.md new file mode 100644 index 0000000..dad33a4 --- /dev/null +++ b/docs/onboarding/06-contracts.md @@ -0,0 +1,183 @@ +# 6. Cross-service contracts + +**Read this before any change that touches both domains.** + +Backend and frontend build separate images and deploy independently. That +independence is the point of the architecture — and it means there is no build +step, no type checker and no linker that will catch a mismatch between them. The +items below are the seams. Each one fails *silently and in production*, which is +why they're enumerated rather than left to judgement. + +## 1. API version — `GET /api/version` + +```json +{"backend_version": "2", "min_frontend": "1", "payload_ver": "p2"} +``` + +Driven by `API_CONTRACT_VERSION` and `MIN_SUPPORTED_FRONTEND` in +`backend/web/app.py`. + +**The rule for a breaking change:** ship a new `v3` router mounted **alongside** +`v2`, re-registering only the handlers that changed, and bump the constant in +the same PR. `/api` and `/api/v1` stay mounted as aliases. Every version stays +served simultaneously — that is how FE and BE ship out of lockstep at all. + +**The frontend pins the version in exactly two places:** + +| Where | What | +|---|---| +| `frontend/server/internal/config` (Go) — `THERMOGRAPH_API_VERSION`, default `v2` | every server-side path builder reads it | +| `frontend/static/account.js` (JS) — the exported `API_VERSION` and the `uv(path)` helper | every browser-side call | + +Never hardcode `api/v2/...` anywhere else. Bump both in one PR, and **only +after** the target backend's `/api/version` confirms it serves that version and +its `min_frontend` doesn't exclude the one you're leaving. + +> The Python `frontend/api_client.py` has a third pin. It isn't deployed, but if +> you're using it locally, keep it in step or you'll debug a phantom. + +## 2. `PAYLOAD_VER` — the cache invalidation token + +`backend/api/payloads.py`, currently `"p2"`. + +This is **separate from URL versioning** and does a different job: it's the +cache/ETag invalidation token. Every derived row is stored against a validity +token built from it, and a stored token that doesn't match the caller's current +token is simply a miss. + +**Bump it whenever a response payload's shape changes.** One bump atomically +orphans every pre-upgrade cached row across every environment. Forget it and +the backend serves old-shaped JSON out of cache indefinitely, with no error +anywhere — the new code is live and the old data keeps flowing. + +`CONTENT_VER` is deliberately separate so a content-only shape change doesn't +invalidate the whole grading cache. + +## 3. ETag / `If-None-Match` — and `expose_headers` + +Every graded payload is cached against a validity token that doubles as a weak +ETag. `frontend/static/cache.js` fetches `/api/v2/cell` once per view-set, +slices it, and revalidates each view individually with `If-None-Match`. + +**`expose_headers=["ETag"]` in the CORS middleware matters as much as +`allow_origins`.** Browsers hide every response header except a fixed "simple" +allowlist from cross-origin JS. Without `ETag` exposed, `cache.js`'s +`res.headers.get("ETag")` silently returns `null`, 304 revalidation quietly +stops working, and **there is no console error**. You'd see it only as a traffic +bill. + +CORS is off by default (`THERMOGRAPH_CORS_ORIGINS` unset), which is today's real +same-origin topology. Never set it to `*` — Starlette itself refuses a +credentialed wildcard. + +Don't change the ETag derivation or that header list without checking +`cache.js`. And when the payload shape changes, the **slice → view map** in +`cache.js` has to track it. + +## 4. `pct_ordinal()` — mirrored in four places + +Floor a percentile into `1..99`. **Never 0, never 100.** + +| Copy | File | Guarded by | +|---|---|---| +| canonical | `backend/data/grading.py::pct_ordinal` | its own unit tests | +| Go SSR | `frontend/server/internal/format/format.go::PctOrdinal` | `format_test.go::TestPctOrdinal` (table-driven, not a cross-check) | +| browser | `frontend/static/shared.js::pctOrd` | nothing automated | +| Python SSR *(not deployed)* | `frontend/format.py::pct_ordinal` | the Python unit tier | + +Every percentile on every surface goes through one of these. If they diverge, +the same reading says two different things in two places. The backend copy is +canonical; the others follow it. + +`TEMP_BANDS` / `RAIN_BANDS` in `grading.py` are the source of truth for tier +names and thresholds. Changing them without the frontend produces tiers drawn +in colours that disagree with the labels the API returns. + +## 5. The Fahrenheit country set + +Fourteen codes: `US PR GU VI AS MP UM BS BZ KY PW FM MH LR`. + +| Copy | File | +|---|---| +| canonical | `backend/api/content_payloads.py::F_COUNTRIES` | +| Go SSR | `frontend/server/internal/format/format.go::FCountries` | +| browser | `frontend/static/units.js::F_REGIONS` | +| Python SSR *(not deployed)* | `frontend/format.py::F_COUNTRIES` | + +`format_test.go::TestFCountriesMatchesBackend` re-parses the backend's Python +source and asserts the Go copy matches — a genuinely good guard. + +> ⚠️ **Verified gap.** Several source comments claim "a test asserts all three +> stay identical." As of this writing that is not true: no test reads +> `static/units.js`. All four copies currently *are* identical (checked), but +> the browser copy is held in step by convention alone. If you touch this set, +> update `units.js` by hand and diff it yourself. Extending the Go test to +> parse `units.js` too would close this properly and is a good first +> contribution. + +## 6. Backend boot must survive an unreachable frontend, and vice versa + +Both directions are load-bearing for asynchronous deploys: + +- **Frontend → backend:** boot tries the IndexNow-key fetch once, catches + failure, and falls back to a lazy per-request lookup. Don't make it fatal. +- **Backend → frontend:** `THERMOGRAPH_FRONTEND_BASE_INTERNAL` is required at + import (fails loud), but it's only *used* by the catch-all proxy fallback — + an unreachable value doesn't stop the backend serving its own routes. That's + why the test suite and the smoke harness both point it at + `http://127.0.0.1:1`. + +`/healthz` on both services is liveness only. Neither proves the other is +reachable, deliberately — readiness is the ingress health-gate's job. + +## 7. `THERMOGRAPH_BASE` must match on both processes + +Default `/thermograph`; both images set `/`. Both processes must be configured +**identically** in every real deployment — it's how they agree on the shared URL +prefix. The Go frontend's content client reads it too, so a sub-path deployment +works rather than only ever working at the root. + +## 8. Image tags are keyed to the domain, not the branch tip + +Both `build-push.yml` and `deploy.yml` compute the tag as +`sha-` + the first 12 hex of `git log -1 --format=%H -- /`. + +Two consequences: + +- An infra-only commit at the branch tip cannot send a deploy chasing an image + no build produced. +- `deployed_version` can legitimately show a tag that is **not** the head of the + branch. That's correct, not drift. + +`fetch-depth: 0` in the deploy workflow exists for exactly this: in a +path-filtered monorepo the tip is often another domain's commit, and a depth-1 +clone can't see past it. + +## 9. Cookie auth is same-origin today + +The interactive tool's auth is an HttpOnly session cookie. `account.js` uses +`credentials: "include"` (not the fetch default) so the cookie still rides along +if the script is ever loaded cross-origin — but the backend must actually be +configured to accept credentialed cross-origin requests (CORS + +`SameSite`) for that to work. In the default same-origin deployment it's a +non-issue. **Don't assume a fully decoupled, independently-hosted frontend +"just works" for logged-in features** without checking that configuration. + +## Quick checklist + +Before opening a PR that touches both domains: + +- [ ] Did any response payload shape change? → bump `PAYLOAD_VER`. +- [ ] Did any route or payload break compatibility? → new `vN` router + alongside, bump `API_CONTRACT_VERSION`, check `min_frontend`. +- [ ] Did I change the API version pin? → both Go config **and** + `static/account.js`, in one PR, after checking the target backend. +- [ ] Did I touch tier names, thresholds, or percentile rendering? → check all + four `pct_ordinal` copies and the colour tokens. +- [ ] Did I touch the Fahrenheit set? → four copies, and `units.js` is + unguarded. +- [ ] Did I change the `/cell` bundle shape? → the slice→view map in + `cache.js`. +- [ ] Did I change ETag derivation or CORS headers? → re-read `cache.js`. + +Next: [CI and release](07-ci-and-release.md). diff --git a/docs/onboarding/07-ci-and-release.md b/docs/onboarding/07-ci-and-release.md new file mode 100644 index 0000000..5576594 --- /dev/null +++ b/docs/onboarding/07-ci-and-release.md @@ -0,0 +1,229 @@ +# 7. CI and release + +## The branch model + +``` +PR ──(required check: `gate`)──▶ dev ──▶ LAN dev box + │ + promotion PR + ▼ + main ──▶ beta.thermograph.org + │ + promotion PR ← the owner's call, always + ▼ + release ──▶ thermograph.org +``` + +`dev`, `main` and `release` are protected — direct pushes are blocked for +everyone. Forgejo has **no auto-merge-on-green**, so merges are explicit: either +click "Auto merge when checks succeed" on a PR, or merge via the API once CI is +green. + +Promotion is continuous and per-decision. There is no release calendar. Size a +promotion to whatever has actually soaked on the environment below. + +### Who may merge what + +| Hop | Whose call | Norm | +|---|---|---| +| anything → `dev` | yours | Merge it. An open PR into `dev` is unfinished work, not caution. | +| `dev` → `main` (beta) | yours, batched | Promote when a batch is coherent enough to be worth *testing*. Beta is the test environment; not promoting is not testing. | +| `main` → `release` (prod) | **the owner's** | Prepare it, recommend for or against, and ask. Never unilaterally, never by another route. | + +Centralis's `promote(from, to)` implements exactly this, and `forge_pr_merge` +enforces it with no back door: it merges into `dev` and `main`, refuses +`main → release` under any flag, and requires `confirm_protected_base: true` +for the separate hotfix path. + +### Two things about branch divergence + +**Commit counts are not deliverable content.** Before promoting, compare the two +tips' *trees*. Identical trees mean the same changes are already on the target +under different SHAs — promoting there merges nothing while still firing the +target's deploy workflows, including `infra-sync` re-rendering +`/etc/thermograph.env` on prod and beta. The ahead/behind numbers will look like +real work. `promote` refuses this outright. + +**Expect `dev`, `main` and `release` to be mutually divergent, and expect +`--ff-only` to fail.** Every promotion merges the source into the target, +creating a merge commit *on the target* that the source never receives — so each +promotion widens the gap it just closed. That is accumulated design, not drift. +(Reconciling merges back into `dev` do happen; `be4aa94` in the history is one.) + +## The nine workflows + +All in root `.forgejo/workflows/`. **Never add a workflow under a domain's own +`.forgejo/`** — Forgejo only reads the root ones, so a copy there is inert and +becomes a trap for whoever edits it next. + +### `pr-build.yml` — the required check + +One workflow, one stable required-check name (**`gate`**), any number of +domains: + +1. a `changes` job diffs the PR against its base (needs `fetch-depth: 0` — the + base commit isn't reachable from a depth-1 clone); +2. per-domain jobs run only for domains the PR touches; +3. `gate` reduces their results into the single status branch protection + requires. + +**Deliberately not path-filtered at the workflow level.** A path-filtered +required check that never fires leaves a PR stuck on *"expected — waiting for +status"* forever, so auto-merge never proceeds. An always-running gate whose +domain jobs skip cleanly gives the same compute savings without the deadlock. +This is the one intentionally *coupled* piece of the CI design. + +### `build.yml` — the reusable build check + +Called via `uses:` with a `domain` input. The build **context is that domain's +subdirectory**, so each Dockerfile sees exactly the tree it saw when its domain +was a repo root. + +For **backend** it then runs the hermetic pytest suite *inside the image it just +built* — the exact interpreter and dependencies that ship. Three flags there are +load-bearing: `-u 0` (the image's default user can't write to site-packages), +`--entrypoint sh` (otherwise the test command is swallowed as entrypoint args +and the container just boots the server forever), and `unset THERMOGRAPH_BASE` +(the image bakes `/`, which moves every route off the `/thermograph` prefix the +tests are written against). + +For **frontend** there's no separate test step: it's a Go binary with no +interpreter in the runtime image, and `gofmt`/`vet`/`go test` already run in the +Dockerfile's builder stage — so a failing Go test fails the Build step above. + +It is deliberately **not** a live boot+healthz check. That was tried in the split +repos and repeatedly hit the runner's own environment quirks (bridge-IP +reachability, non-unique `$$`, squatted host ports) without ever going reliably +green. Image boot is verified by hand instead — `make smoke`, or `docker run` + +alembic + `/healthz` 200 + a real `/api/v2/place` 200. + +### `build-push.yml` — images + +Triggers on pushes to `dev`/`main`/`release` touching `backend/**` or +`frontend/**`, and on `v*.*.*` tags. Publishes +`git.thermograph.org/emi/thermograph/{backend,frontend}:sha-<12hex>`. + +- **Serialised** (`max-parallel: 1`): one physical runner, and two whole image + builds racing each other buys nothing. +- **Semver exception:** a `v*.*.*` tag push carries no paths context, so a + release tag builds **both** images. That's intended — a release names the + whole set, not whichever domain moved last. +- The registry is mesh-only, so the runner host needs + `10.10.0.2 git.thermograph.org` in `/etc/hosts`. + +### `deploy.yml` — one file, two environments, two services + +Formerly six near-identical files. Branch selects the environment +(`main → beta`, `release → prod`); a matrix covers the services. + +The style is **deliberately boring**: no dynamic `fromJSON` matrix, no +`cond && secrets.A || secrets.B` ternary. Those are GitHub idioms a Forgejo/act +runner may evaluate differently, and the failure mode here is "production does +not deploy" or, worse, "deploys with an empty SSH host". So the two environments +get two explicit, mutually exclusive steps and the tag is computed in shell. + +Preserved from the originals, all load-bearing: + +- `fetch-depth: 0` — the tag is keyed to the last commit that touched *that + domain*, not the branch tip. +- 12-hex truncation matching `build-push.yml` exactly. +- Per-service, per-environment concurrency with **`cancel-in-progress: false`** — + a half-finished deploy must never be cancelled by a newer one. +- **Separate `PROD_SSH_*` credentials**, so a beta credential leak cannot reach + prod. +- `appleboy/ssh-action` referenced by full URL; it isn't mirrored in Forgejo's + default action registry. + +There's also a per-leg refinement: the workflow-level `paths` filter only says +*backend or frontend moved*, so each leg re-checks whether **this** push touched +**this** domain. Without it a backend-only push would also roll the frontend, +losing the independent-deploy property the whole split exists for. When there's +no usable base SHA (first push, force push, `workflow_dispatch`) it deploys +rather than skips — rolling onto the tag already running is a no-op, whereas +skipping silently strands a change. + +### `infra-sync.yml` — infra's own pipeline + +Push to `main` touching `infra/**` → SSH to beta **and** prod, fast-forward +`/opt/thermograph`, re-render `/etc/thermograph.env` from the vault. + +**Rolls no service.** Image tags are the app domains' axis, not infra's. A +compose change that must recreate containers takes effect on the next app +deploy, or on a by-hand `SERVICE=all … infra/deploy/deploy.sh`. + +Note the asymmetry: app code *is* environment-staged (`dev`→`main`→`release` +maps to LAN→beta→prod via image tags); **infra is not** — both hosts track infra +via `main`. Prod's *app images* are staged by `release`, but its checkout +follows `main`. + +### `observability-validate.yml` + +Parses every artifact that ships to a node: both compose files, all dashboard +JSON, the Loki and provisioning YAML, and the Alloy config — checked with the +**pinned `alloy` binary (v1.9.1, matching what the fleet runs)**, downloaded +from its GitHub release, no Docker CLI needed. + +Alerting gets a stricter third step: every rule's `condition` must name a refId +that exists, every policy must route to a receiver that exists, and a literal +Discord webhook URL in the repo is a hard failure. **A rule pointing at a +missing refId is valid YAML, provisions cleanly, and then never fires** — that +silent no-op is exactly what this check exists to prevent. + +### `ops-cron.yml` — the prod backup + +Daily at 03:00 UTC: `pg_dump` on prod, plus an IndexNow ping. Both SSH into prod +and run inside the already-running stack. + +**It uses `PROD_SSH_*`, not `SSH_*`.** An earlier revision reused `SSH_*` — so +the "prod" backup was silently dumping *beta*, and prod had no backup at all. If +you touch this file, verify a dump actually lands in +`agent@prod:~/thermograph-backups/`. + +### `secrets-guard.yml` and `shell-lint.yml` — the backstops + +Both run on **every** PR and push, deliberately un-path-filtered: *a backstop +that only runs when you expect it to isn't a backstop.* + +- `secrets-guard` fails if any `infra/deploy/secrets/*.yaml` lacks both a + `sops:` block and `ENC[AES256_GCM` values. ~1 second. +- `shell-lint` runs shellcheck over every `*.sh` found by `find` (not a list — + so a new script is covered the moment it lands). Pinned to **v0.11.0 plus a + sha256**, installed from the official static release rather than `apt-get`: + the distro version drifts with the job image, and a drifted shellcheck grows + new warnings that fail CI on an unrelated push. Bump the version and the hash + together, and expect to fix new findings in the same PR. + +Keep the tree at **zero shellcheck findings**. These scripts run as root over +SSH against live hosts with no test suite in front of them; this is the only +guard. + +## Runner facts + +- Jobs run on the `docker` label — the LAN box's Forgejo Actions runner. +- Job containers get the **host's docker.sock** automounted + (Docker-outside-of-Docker, no privileged mode). The job image + (`node:20-bookworm`) ships no docker CLI, which is why several workflows + `apt-get install docker.io` first. +- Forgejo's runner has **only the labels it was explicitly registered with**. + GitHub implicitly tags every self-hosted runner `self-hosted`; Forgejo does + not. A `runs-on: [self-hosted, thermograph-lan]` array is permanently + unschedulable there — a real bug that was found and fixed once, and worth + recognising if a job ever silently stops appearing in the Actions history. + +## Opening a PR + +```bash +git switch -c fix/short-description # branch off dev +# ... work ... +cd backend && make test # or: cd frontend/server && go test ./... +git commit # concise, technical, no AI mentions +git push -u origin fix/short-description +``` + +Then open the PR into `dev` (Centralis's `forge_pr_open`, or the Forgejo web +UI). Wait for `gate`. Merge — squash is the configured style. + +If your change touches shell, expect `shell-lint` to have an opinion. If it +touches `infra/deploy/secrets/`, `secrets-guard` will too. + +Next: [Infra and secrets](08-infra-secrets.md). diff --git a/docs/onboarding/08-infra-secrets.md b/docs/onboarding/08-infra-secrets.md new file mode 100644 index 0000000..ed8eec5 --- /dev/null +++ b/docs/onboarding/08-infra-secrets.md @@ -0,0 +1,254 @@ +# 8. Infra and secrets + +`infra/` owns **how and where the already-built images run**. It never builds +app source. Hosts' `/opt/thermograph` is a checkout of this whole monorepo. + +Read [`infra/CLAUDE.md`](../../infra/CLAUDE.md) alongside this. + +## The machines + +| Host | Public | Mesh | Orchestrator | Login | +|---|---|---|---|---| +| **prod** | `169.58.46.181` / thermograph.org | `10.10.0.1` | Docker **Swarm** | `agent`, passwordless sudo | +| **beta** | `75.119.132.91` / beta.thermograph.org | `10.10.0.2` | **compose** | `agent`, passwordless sudo | +| **desktop** | — | `10.10.0.3` | compose (LAN dev) | it's your box | + +```bash +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # prod +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # beta +``` + +Beta also hosts **Forgejo** (git + CI + registry) and **Grafana + Loki**, which +is why beta is guarded as strictly as prod: a destructive command there takes +out git, CI and the registry at once. The LAN dev box is deliberately +unguarded. + +Every root-effective command on the VPSes is logged by `auditd` +(`ausearch -k agentcmd`). That's a feature — fixes are attributable. + +Prefer Centralis for routine work (`run_on_host`, `fleet_status`, +`deployed_version`) over raw SSH. + +## The two orchestrators + +Which path a host takes is decided by **`/etc/thermograph/deploy-mode`**: the +string `stack` makes `deploy.sh` exec `deploy/stack/deploy-stack.sh`; anything +else is compose. The workflows never need to know which mode a host runs. + +| | compose (beta, LAN dev) | Swarm (prod) | +|---|---|---| +| File | `infra/docker-compose.yml` | `infra/deploy/stack/thermograph-stack.yml` | +| Services | `db`, `backend`, `lake`, `daemon`, `frontend` | `db`, `web`, `worker`, `lake`, `daemon`, `frontend`, `autoscaler`, `autoscaler-lake` | +| Rolling | `up -d --no-deps ` | start-first, health-gated, auto-rollback | +| Tag file | `deploy/.image-tags.env` | `deploy/.stack-image-tags.env` | + +Note prod splits `backend` into **`web`** and **`worker`** (the +`THERMOGRAPH_ROLE` split from [04](04-backend.md)); compose runs one `backend` +service in role `all`. + +`STACK_TEST=1` rehearses the entire Swarm deploy under stack name +`thermograph-test` on throwaway volumes and ports `18137`/`18080`. Leftover +`thermograph-test_*` containers in logs are rehearsal residue — ignore them. + +### The volume/project-name coupling + +Compose creates `thermograph_pgdata` / `_appdata` / `_applogs`; the Swarm stack +declares those exact names as `external: true` at the same mount paths. That +coupling is why `name: thermograph` is pinned in the compose file — running +compose from `infra/` without it derives project `infra`, silently creating a +whole new stack with empty volumes beside the running one. `deploy-dev.sh` +exports `COMPOSE_PROJECT_NAME=thermograph-dev` (env wins over the file key) to +keep LAN dev separate on purpose. **Keep both halves.** + +## `deploy/deploy.sh` — read this before you touch a deploy + +Single entry point for beta and prod. Contract: + +```bash +SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh +SERVICE=frontend FRONTEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh +SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- …/deploy.sh +``` + +What it does, in order, and why each step is the way it is: + +1. **`flock` on `deploy/.deploy.lock`**, with a self re-exec so the lock spans + the whole run. Backend and frontend deploys can fire for the same push + seconds apart and both SSH into one checkout; concurrent runs race the git + reset, the compose project and the tag file. `-w 600` bounds the wait. +2. **Render secrets** from the SOPS vault into `/etc/thermograph.env`, then + source it, so a by-hand run interpolates the same as the systemd unit does. + Guarded on the helper's existence so the very deploy that *introduces* + `render-secrets.sh` is safe. +3. **`git reset --hard origin/$BRANCH`** on the checkout root (`BRANCH` defaults + to `main`). ⚠️ **Uncommitted edits in `/opt/thermograph` evaporate here.** + The one exception is `deploy/.image-tags.env`, untracked on purpose. +4. **Read `.image-tags.env`** so a single-service roll re-renders compose with + *both* services' real tags and never accidentally recreates or downgrades the + sibling. +5. **Registry login** only if `REGISTRY_TOKEN` is set. The CI paths don't pass + one — the host is already `docker login`ed — and an unconditional login with + an empty token would abort under `set -e`. +6. **Pull, with a bounded ~5-minute retry.** `build-push.yml` is a *separate* + workflow triggered by the same push, and Forgejo's `needs:` only orders jobs + within one file. A deploy racing ahead of its build and failing "not found" + is confirmed-live behaviour; the retry covers a normal build and still fails + loudly on a genuine problem. +7. **Daemon capability probe.** Infra tracks `main` while image tags are + env-staged, so a host can legitimately be asked to roll a backend image older + than the compose file — one built before the daemon binary existed. The + script `docker run`s the image to test for `/usr/local/bin/thermograph-daemon` + and drops `daemon` from this run if it's absent, rather than leaving a + container crash-looping on a missing binary. +8. **Roll.** `TARGETS` are `backend → (backend, lake, daemon)`, + `frontend → (frontend)`, `all → everything`. **`daemon` and `lake` ride with + `backend` and are never targets on their own** — all three run the same image + at the same tag, and rolling one without the others is exactly the version + skew the `/internal/*` contract has no negotiation for. Single-service rolls + use `--no-deps`; a full `all` uses `--remove-orphans` (a renamed-away + service's old container would otherwise squat its port — confirmed live). +9. **Write `.image-tags.env`** *after* `up`, so a failed pull never records a + tag that isn't running. +10. **Health check** (backend 8137, frontend 8080), then detached city warming. + +### Rollback + +Rollback is redeploying the previous image tag. But **do not treat image +retention as a rollback guarantee**: both `deploy.sh` and `deploy-stack.sh` +*end* by deleting images outside the running pair. On beta that succeeds, so +beta typically holds **no local rollback target at all**; on prod it usually +fails only because Swarm's stopped task containers still hold references, which +is why prod keeps a handful. Verify the target tag is actually present before +promising a rollback — Centralis's `rollback_to(dry_run: true)` checks both host +and registry. + +`docker system prune -a` on a live box is forbidden for the same reason. +`docker image prune -f` (dangling only) is the safe form. + +## Secrets — the SOPS + age vault + +**The single source of truth is `infra/deploy/secrets/*.yaml`**, committed +encrypted (values only — keys stay readable so diffs mean something) and +rendered into `/etc/thermograph.env` at deploy time by +`deploy/render-secrets.sh`. + +**Cycling a key is: edit → commit → deploy.** No SSH, no hand-edited root file, +no per-host duplication. + +| File | Holds | +|---|---| +| `common.yaml` | the 16 values identical on prod **and** beta — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config | +| `prod.yaml` | prod's own — the three held-back credentials, sizing, `THERMOGRAPH_BASE_URL`, Discord + mail credentials that exist nowhere else | +| `beta.yaml` | beta's own — the three held-back credentials, sizing, base URL | +| `dev.yaml` | the LAN box's own, **self-contained**, 12 values, no production credential | +| `centralis.prod.yaml` | Centralis's nine variables → `/etc/centralis.env`, its own renderer | +| `example.yaml` | format reference / CI fixture (fake values) | +| `../../.sops.yaml` | which age recipients files encrypt to (plaintext config) | + +The renderer concatenates `common.yaml` then `.yaml`, so a **host value +wins** (last occurrence of a duplicate key). `` comes from +`/etc/thermograph/secrets-env` on the box. + +### Two design decisions worth understanding + +**Three credentials are deliberately *not* in `common.yaml`** — +`POSTGRES_PASSWORD`, `THERMOGRAPH_AUTH_SECRET`, `THERMOGRAPH_DATABASE_URL`. +They hold identical values on prod and beta today, so by the mechanical rule +they belong there. They're kept per-host anyway because these are the +credentials that let one environment *act as* another: with them, a foothold on +beta (the more exposed box — public Forgejo and Grafana) is a foothold on prod's +database and prod's session signing. They match because beta was seeded from +prod, not because the two are meant to be one system. Keeping them per-host +costs one extra line and buys the ability to diverge. + +**`dev` renders `dev.yaml` alone** — `deploy-dev.sh` exports +`THERMOGRAPH_SECRETS_SKIP_COMMON=1`. Eleven of `common.yaml`'s sixteen values +are live production credentials, including a **read-write** object-storage +keypair on the bucket that also holds prod's database backups, and the VAPID +private key that signs push to real subscribers. The render is plaintext +concatenation, so layering it would put all of those into every container in the +dev stack — on the operator's desktop, which is also the CI runner whose +`docker`-labelled jobs get the host socket, running unreviewed `dev` code. An +override in `dev.yaml` wouldn't help: last-wins governs *consumers*, but the +production value is still physically a line in the file. + +Dev works fine without them: no S3 keys means the `lake` service answers 503 and +history falls through to the Open-Meteo archive (an accelerator, never a +dependency); VAPID and IndexNow both self-generate and persist to the `appdata` +volume; with no metrics token `/api/v2/metrics` is direct-loopback-only, which +is right on a LAN box. One `common.yaml` value is simply *wrong* for dev — +`THERMOGRAPH_COOKIE_SECURE=1` silently breaks login over plain HTTP. + +### Working with the vault + +```bash +sops edit infra/deploy/secrets/beta.yaml # never a plain editor +infra/deploy/secrets/dry-run-render.sh # preview a render +``` + +- **Never hand-edit `/etc/thermograph.env`** — it's a rendered artifact, + overwritten on the next deploy. +- **Never paste a plaintext secret onto a box.** +- `secrets-guard` CI rejects a plaintext commit; the repo's `secrets-guard.sh` + hook blocks a direct Write/Edit to those files locally. +- `deploy/secrets/seed-from-live.sh` reads production secrets and is **not** + for an agent to run. +- The **`key-gaps`** skill audits which keys are missing or drifting across + environments. It reads key *names* only, never values. + +There's a cautionary tale in `deploy/secrets/README.md` worth reading: Centralis +was hand-configured until a hand-edit wrote `CENTRALIS_TOKENS` as bare JSON into +a file that gets `source`d, bash stripped the quotes, and Centralis — correctly +failing closed on malformed JSON — dropped to a single identity. Nothing logged +an error. It looked exactly like the file had never been written. That's why +`centralis.prod.yaml` exists: the only durable fix was *a human stops writing +the file*. + +## Terraform + +`infra/terraform/` provisions and configures hosts, SSH-driven. + +> **Never run `terraform apply` casually.** No tfstate is persisted anywhere, so +> an apply would attempt full re-provisioning of live hosts. Terraform here is +> **executable documentation** until state is bootstrapped. + +## Ops query tooling + +Read-only, uniform across environments: + +```bash +infra/ops/dbq.sh dev "select count(*) from climate_history" +infra/ops/dbq.sh prod -tA -c "select max(date) from climate_history" +infra/ops/iceberg.sh prod -c "select count(*) from era5_daily" +``` + +Both exec **into the container**. No database is exposed over TCP — each listens +only on its private docker network, and prod's is a Swarm **overlay the prod +host itself cannot route to**, so `ssh -L` works for beta and is *impossible* +for prod. Exec works identically everywhere with no ports, no tunnels, no infra +changes. The prod container name is a Swarm task name that changes on every +redeploy, so it's resolved at call time via `docker ps --filter name=`, never +hardcoded. + +Queries connect as **`thermograph_ro`** — `NOSUPERUSER`, granted only +`pg_read_all_data`. Read-only is enforced by Postgres, not by convention: + +``` +$ infra/ops/dbq.sh prod -c "create table t(i int)" +ERROR: permission denied for schema public +``` + +Centralis's `sql_query` does the same thing with the same role. `write: true` +escalates to the app's superuser and takes effect immediately with no +confirmation — on prod, know what you are doing. + +## Backups + +The nightly `ops-cron.yml` `pg_dump` into `agent@prod:~/thermograph-backups/` +is it. Known gaps, stated plainly: **a single copy on the same box as the +database, no offsite yet**, and restores must handle TimescaleDB's +`continuous_agg` circular-FK warning (`--disable-triggers`). The `backups/` +prefix in the object-storage bucket belongs to those jobs — don't write outside +the lake's own prefix. + +Next: [Observability](09-observability.md). diff --git a/docs/onboarding/09-observability.md b/docs/onboarding/09-observability.md new file mode 100644 index 0000000..8605fba --- /dev/null +++ b/docs/onboarding/09-observability.md @@ -0,0 +1,199 @@ +# 9. Observability + +Fleet-wide log aggregation: **Loki + Grafana on beta**, fed by a **Grafana +Alloy** agent on every node, all over the WireGuard mesh. + +``` + prod (10.10.0.1) beta (10.10.0.2) desktop / LAN dev (10.10.0.3) + ┌────────────┐ ┌──────────────────┐ ┌────────────┐ + │ Alloy agent│──wg0──┐ │ Loki ◀── Alloy │ ┌───│ Alloy agent│ + └────────────┘ └──▶│ Grafana (Caddy) │◀─┘ └────────────┘ + docker+caddy+app └──────────────────┘ docker+caddy+app +``` + +- **Loki** — filesystem storage, **30-day retention**, listening on the mesh IP + `10.10.0.2:3100`. Never public. +- **Grafana** — the UI, fronted by beta's Caddy at + **`dashboard.thermograph.org`** (Google SSO, pre-provisioned accounts only, + `allow_sign_up=false`, plus a break-glass local admin). Use that hostname + everywhere; **never** `grafana.thermograph.org`. +- **Alloy** — on each node, shipping three sources: every Docker container's + stdout/stderr, Caddy's host access logs, and the app's structured JSON logs + (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase` become + labels). Every line is tagged `host = prod|beta|dev`. + +There is **no build and no deploy automation** for this domain — it ships by +hand. But `observability-validate.yml` parses every artifact that ships to a +node, so a malformed dashboard or an unparseable Alloy config fails in CI +instead of on the host. + +## The rule that matters most + +**The repo is the only durable path.** Dashboards and alerting are provisioned +from `observability/grafana/provisioning/` at startup, so **edits made through +Grafana's UI or API are overwritten on the next provision**. A change goes +through a PR. Centralis's `dashboard_write` opens one for you. + +## Reading the fleet + +Start wide, then narrow: + +1. `fleet_status` — HTTP reachability plus what's running on each host. +2. `logs_overview since="1h"` — where the volume is, and what has gone quiet. +3. `logs_query service=… host=…` — the actual lines. + +### ⚠ The prod trap + +Prod runs a Swarm stack, and its `job="docker"` streams carry **no `service` +label**. A raw `{host="prod", service="backend"}` query matches nothing and +reads as *"no logs"* when it actually means *"wrong selector"*. + +Use Centralis's `logs_query` `service` argument, which rewrites it to a +container name-regex with the churning task suffix wildcarded. Prod service +names are `thermograph_web`, `_worker`, `_frontend`, `_db`, `_autoscaler`, +`_lake`. + +Related: **prod container names change on every redeploy** (the Swarm task +suffix). Never hardcode one; resolve via `docker ps --filter name=` at call +time. + +## The dashboard + +`Thermograph — Fleet Logs` (auto-provisioned), with a `Node` selector: log +volume by service, app error rate by tag, upstream 429 count, Caddy 5xx count, a +per-node notifier-liveness indicator, recent app errors, and a live +all-container tail. + +## Alerting + +Provisioned from `grafana/provisioning/alerting/`, mounted read-only into +Grafana. + +**Every rule is LogQL** — there is no Prometheus anywhere in the fleet. Loki is +the only datasource, so every signal is derived from log lines. Thresholds were +backtested against real Loki data and the working is written out in the comments +beside each rule; **re-derive before changing a number rather than guessing**. + +### Alerts go to Discord `#ops-alerts`, never email + +Three reasons, all learned the hard way: + +- Grafana's factory default pointed at the literal string + `` — a paging path that had never delivered a message to + anyone. +- Beta's Grafana relays SMTP through **prod's** Postfix. Email alerts therefore + travel through the box most likely to be on fire, and vanish exactly when they + matter. +- Discord is off-estate, works when prod is dead, and reaches a phone. + +`#ops-alerts` is dedicated. `#dev` / `#uat` / `#prod` are the *product's* alert +subscription output — real deliveries to real subscribers — so routing ops noise +there would corrupt the evidence. + +The webhook URL is a secret and lives only in beta's gitignored `.env` as +`DISCORD_ALERT_WEBHOOK_URL`. Grafana refuses to start if it's unset. A literal +webhook URL committed to the repo is a **hard CI failure**. + +### The twelve rules + +| Rule | Fires when | Severity | +|---|---|---| +| `ProdEdgeDark` | zero prod Caddy log lines in 15m — site dark, or the pipeline is broken | critical | +| `ProdHealthzFailing` | any non-200 on `/healthz` in 10m | critical | +| `ProdHTTP5xxBurst` | ≥10 prod 5xx in 10m | critical | +| `ProdHTTP5xxElevated` | ≥25 prod 5xx over 6h (the error-budget alarm) | warning | +| `ProdUnhandledExceptions` | ≥3 app `tag=http.error` in 30m | warning | +| `NotifierHeartbeatMissingProd` | no `tag=heartbeat` from prod in 20m | critical | +| `NotifierHeartbeatMissingBeta` | same, on beta | warning | +| `ProdWebContainerSilent` | `thermograph_web` logged nothing in 30m | critical | +| `ProdDbContainerSilent` | `thermograph_db` logged nothing in 2h | critical | +| `ProdFrontendContainerSilent` | `thermograph_frontend` logged nothing in 30m | warning | +| `ProdWorkerContainerSilent` | `thermograph_worker` logged nothing in 2h | warning | +| `AlertingWatchdog` | always — one Discord message per day | watchdog | + +`AlertingWatchdog` is the dead-man's switch and fires permanently by design. +**Its value is its absence:** nothing can report its own death, so if +`#ops-alerts` has been silent for more than a day, alerting is broken rather +than the estate healthy. + +### The strict CI check, and why + +A rule whose `condition` names a refId that does not exist is **perfectly valid +YAML, provisions without complaint, and then never fires** — a paging path that +looks healthy and reaches nobody. `observability-validate.yml` checks that every +condition names a real refId and every policy routes to a receiver that exists, +precisely because nothing else would catch it. + +The same asymmetry shows up in verification: `GET +/api/v1/provisioning/contact-points` **redacts** the webhook URL, so you cannot +confirm interpolation by reading it back. A contact point holding the literal +string `$DISCORD_ALERT_WEBHOOK_URL` looks completely healthy from the API and +pages nobody. **The only proof is a message actually arriving in +`#ops-alerts`** — force one with the receivers-test endpoint (the command is in +`observability/README.md`). + +## Deploying an observability change + +> ⚠️ **Merging is not deploying — and this one has already bitten.** +> `/opt/observability` on beta and prod is a checkout of the **archived** +> `emi/thermograph-observability` repo. It can never `git pull` again; the +> content now lives here under `observability/`. A previous observability PR +> merged and had **zero live effect** for exactly this reason. + +Until someone re-points those checkouts at the monorepo, every change ships by +hand: back up the live file with a UTC-timestamped suffix, `scp` the new one up, +`sudo cp` it into place, restart the specific container. + +Two gotchas in that sequence: + +- Use `sudo docker restart ` on **prod** — not `docker compose`, + which there demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to + interpolate from. Beta's stack *does* have an `.env`, so compose works there. +- **`docker restart` will not pick up a new environment variable.** Any change + that introduces one needs `docker compose up -d grafana` on beta. + +Alerting lives on **beta only** — prod and dev run Alloy agents, not Grafana. + +Full step-by-step, including rollback, is in +[`observability/README.md`](../../observability/README.md). + +## Known gaps (as documented in the repo) + +- **Alloy can silently stop tailing a container after Swarm replaces the task.** + Observed on prod 2026-07-24: `thermograph_worker` was `1/1`, healthy, writing + ~19 lines per 10 minutes to its own docker log, yet not one line reached Loki + for 13½ hours. What localises it to Alloy: prod's `tag=heartbeat` lines kept + arriving throughout, and those come from a *different* Alloy source + (`loki.source.file` on `/applogs`) than container stdout + (`loki.source.docker`). One source went deaf, the other didn't. Workaround: + `sudo docker restart alloy-alloy-1` on the node. **Not root-caused.** + `ProdWorkerContainerSilent` now catches this class of failure. +- **Some services are too quiet to alert on.** `thermograph_daemon` (23 lines / + 24h), `thermograph_autoscaler` (1) and `_autoscaler-lake` (2) have a healthy + floor indistinguishable from dead; `thermograph_lake` is bursty ETL with + legitimate idle gaps. Giving each an `audit.log_heartbeat()` line would make + all four alertable with the pattern the notifier already uses. **The single + highest-value follow-up in this domain.** +- **Grafana cannot alert on its own death.** `AlertingWatchdog` is a *manual* + dead-man's switch — it relies on somebody noticing the daily message stopped. + An external uptime pinger from off-estate would close this properly. +- **Beta runs a `docker-compose.override.yml` that is not in this repo**, wiring + Grafana's SMTP to prod's Postfix. It should be mirrored into the tracked + compose file or deleted; alerting no longer depends on it. + +## When something breaks + +1. `fleet_status` — is it up at all? +2. `logs_query host=prod contains="(?i)error|traceback"` since the incident. +3. `deployed_version env=prod` — did something ship just before it started? +4. `forge_prs` / `forge_ci` — what landed recently, and did CI actually pass? +5. **Write down what you found with `notes_write` *before* fixing it.** The + reconstruction is worth more than the fix, and it will be gone by tomorrow. + +You are authorised to fix production roadblocks directly — restart or roll a +wedged service, clear a full disk, kill a runaway process, bounce a crash-looping +container. What you must not do is make the fix *durable* on the box: writes to +code, docs and dashboards go through a PR, and `deploy.sh` hard-resets the +checkout anyway. The on-box action is stabilisation; the PR is the fix. + +Next: [Recipes](10-recipes.md). diff --git a/docs/onboarding/10-recipes.md b/docs/onboarding/10-recipes.md new file mode 100644 index 0000000..58766e7 --- /dev/null +++ b/docs/onboarding/10-recipes.md @@ -0,0 +1,256 @@ +# 10. Recipes + +Task-shaped walkthroughs. Each one names the files you'll touch and the traps +specific to that change. + +--- + +## Add a field to a graded API payload + +1. **Build it** in `backend/api/payloads.py` (or `content_payloads.py` for the + SSR content API). Payload assembly is pure `inputs → dict`; keep HTTP + semantics with the caller. +2. **Bump `PAYLOAD_VER`** in `api/payloads.py`. Non-negotiable — the derived + store is token-validated with no clock expiry, so without a bump the old + shape serves out of cache forever, silently. If the change is content-API + only, `CONTENT_VER` is the narrower lever. +3. **Decide whether it's breaking.** Additive → fine on `v2`. Breaking → a new + `v3` router mounted alongside `v2` re-registering only the changed handlers, + plus `API_CONTRACT_VERSION` bumped in the same PR. +4. **Check `/cell`.** If the field lands in a slice, the slice→view map in + `frontend/static/cache.js` has to track it. +5. **Consume it** in `frontend/server/internal/contentapi/types.go` and whichever + template or JS renders it. +6. **Test both sides**: `cd backend && make test`, then + `cd frontend/server && go test ./...`. +7. **Refresh fixtures if the shape moved**: `cd frontend && make capture-fixtures` + against a live backend. Those fixtures feed the **Go** tests too. + +--- + +## Add a new SSR page + +The backend owns the page's *data and metadata*; the frontend owns its *markup*. + +1. `backend/api/content_payloads.py` — a builder returning JSON-safe dicts (raw + floats, no `Markup`, no ContextVar unit rendering), including `page_title`, + `canonical_path`, `breadcrumb` and `jsonld`. Those live here, **not** in the + frontend. +2. `backend/api/content_routes.py` — route wiring plus the same derived-store / + ETag pattern the neighbouring routes use. +3. `frontend/server/internal/contentapi/` — a typed client method. +4. `frontend/server/internal/render/templates/.html.tmpl` — embedded in + the binary, so a template change needs a rebuild. +5. `frontend/server/internal/content/pages.go` — the handler; register the route. +6. If the page should be crawlable, add it to the sitemap + (`backend/api/sitemap.py` and the `/content/sitemap` payload). +7. Copy that isn't per-city data goes in `frontend/content/pages.yaml`, loaded + fail-loud at boot. + +--- + +## Change something visual + +1. Read [`frontend/DESIGN.md`](../../frontend/DESIGN.md) first. It wins over any + general design guidance. +2. Edit `frontend/static/style.css`. **Use `var(--token)`; never paste a hex.** + New surfaces take their colour from existing tokens so light mode keeps + working for free. +3. If it's a grade colour, keep the scale's order and midpoints intact — they + encode meaning, not decoration, and the charts pull from the same tokens as + the legend. +4. **Verify at 390 / 800 / 1920 / 2560 / 3840px in both light and dark:** + ```bash + cd frontend && .venv/bin/python tools/shoot.py + ``` + or drive a real browser with the Chrome DevTools MCP. Narrow while + iterating: `tools/shoot.py index --width 390 --scheme dark`. +5. Watch the hard rules: inputs at `font-size: 16px` minimum (iOS zoom), ~44px + touch targets, honour `prefers-reduced-motion`, metric order always + `Precip · High · Low`. + +--- + +## Add or rotate a secret + +```bash +sops edit infra/deploy/secrets/prod.yaml # never a plain editor +git commit && push && open a PR +``` + +Then it reaches the hosts by either route: an `infra/**` push to `main` fires +`infra-sync.yml` (re-renders `/etc/thermograph.env` on beta and prod, rolls +nothing), or the next app deploy renders it as step 2 of `deploy.sh`. + +Decide **which file** first: + +- identical on prod *and* beta, and not a "lets one environment act as another" + credential → `common.yaml`; +- per-host, or one of `POSTGRES_PASSWORD` / `THERMOGRAPH_AUTH_SECRET` / + `THERMOGRAPH_DATABASE_URL` → `prod.yaml` / `beta.yaml`; +- dev → `dev.yaml`, and give it **its own** value. Never a copy of prod's. + +Preview with `infra/deploy/secrets/dry-run-render.sh`. Audit cross-environment +drift with the `key-gaps` skill (names only, never values). + +**Never** hand-edit `/etc/thermograph.env`, paste a plaintext secret onto a box, +or run `seed-from-live.sh`. + +If a rotated value needs containers recreated to take effect, remember +`infra-sync` rolls nothing — you need an app deploy or a by-hand +`SERVICE=all … deploy.sh`. + +--- + +## Add a recurring background job + +Ask one question first: **does it need data?** + +- **No** (a timer, a connection, an external poke) → add a `cron.Job` in + `backend/daemon/main.go`. One replica, so no election needed. Give it an env + var for its interval, defaulting sensibly and falling back to the default on a + malformed value. +- **Yes** → add an `/internal/*` route in `backend/api/internal_routes.py` and + have the daemon POST to it. Grading depends on polars and the cache; + reimplementing it out-of-process would let results drift from the API's. + +If you genuinely need an in-process background thread in the Python app instead, +**read `core/singleton.py` first**. Anything that issues upstream fetches on a +timer must run behind `claim_leader()` — N workers × M hosts multiplies quota +use N×M, and that has already caused overnight 429s in production. Things that +drain a per-worker request-fed queue (the neighbour warmer) correctly run +everywhere; so does the heartbeat, because a beat costs one local file append. + +Then give it a heartbeat (`audit.log_heartbeat()`) so observability can alert on +its death — see the "too quiet to alert on" gap in +[observability](09-observability.md). + +--- + +## Change a Grafana alert or dashboard + +**Never through the UI.** Provisioning overwrites it on the next start. + +1. Edit `observability/grafana/provisioning/alerting/*.yml` or + `observability/grafana/dashboards/*.json`. +2. If you're changing a threshold, **re-derive it against real Loki data** — the + working for the current number is in the comments beside the rule. +3. Open a PR. `observability-validate.yml` will check that every condition names + a real refId, every policy routes to a real receiver, and no literal webhook + URL is committed. +4. **Merging is not deploying.** Ship it by hand per + `observability/README.md` — and remember `docker restart` won't pick up a new + environment variable. +5. Verify by forcing a real message into `#ops-alerts`. The API redacts the + webhook, so arrival is the only proof. + +--- + +## Debug: "the site is serving stale data" + +Work the token chain, in this order: + +1. `GET /api/version` on the affected environment — is `payload_ver` what you + expect? If a payload shape changed and this didn't move, that's your answer. +2. Is the derived-store token advancing? It's built from `PAYLOAD_VER` plus the + history end date (`history_token`), so a cell whose history stopped updating + pins its token. +3. Is the *history* updating? Check `climate_sync` freshness + (`infra/ops/dbq.sh prod …`) and whether upstream is in a rate-limit cooldown + (the errors log, tagged `retry`/`error`). +4. Is it the **frontend's** TTL cache instead? `THERMOGRAPH_SSR_CACHE_TTL` + defaults to 600s, in front of the backend's own caching. +5. Is it the **browser**? `cache.js` seeds IndexedDB from the `/cell` bundle. If + ETags stopped being visible cross-origin (`expose_headers`), revalidation + silently stops. + +--- + +## Debug: "a deploy went green but nothing changed" + +Common causes, most likely first: + +1. **The tree was already identical.** A promotion between branches whose trees + match merges nothing while still firing the deploy workflows. +2. **The domain wasn't touched.** `deploy.yml` re-checks whether *this* push + touched *this* domain, and skips the leg if not. +3. **The tag is domain-keyed.** `deployed_version` showing a tag that isn't the + branch head is correct behaviour, not drift. +4. **Someone hand-edited `/opt/thermograph`.** `deploy.sh` runs + `git reset --hard` first; the edit is gone. +5. **It's an observability change.** `/opt/observability` points at the archived + repo and cannot pull. See above. + +--- + +## Promote to beta, and then to prod + +``` +dev → main your call, batched. Beta is the test environment; + not promoting is not testing. +main → release ALWAYS the owner's call. Prepare it, state what would ship, + the CI evidence and how long it soaked on beta, then recommend — + including recommending against. +``` + +Centralis's `promote(from, to)` does the mechanics and enforces the rule; +`forge_pr_merge` refuses `main → release` under any flag. Before promoting, +compare the two tips' **trees**, not their commit counts. + +--- + +## Roll back + +Redeploying the previous image tag: + +```bash +SERVICE=backend BACKEND_IMAGE_TAG=sha- \ + /opt/thermograph/infra/deploy/deploy.sh +``` + +**Check the tag actually exists first.** Both deploy scripts *end* by deleting +images outside the running pair, so beta typically holds no local rollback +target at all. `rollback_to(dry_run: true)` checks host and registry and will +tell you when it's in neither. + +--- + +## Regenerate the bundled city data + +`backend/cities.json` and `cities_flavor.json` are generated reference data, not +runtime state: + +```bash +cd backend +python gen_cities.py # the curated city set +python gen_flavor.py # the editorial blurbs +``` + +Editorial copy has its own voice guide — the **`city-copy`** skill defines what +good reads like, and **`backfill-city-copy`** drives generation at scale. Both +enforce the relative-grades rule. + +After a deploy, `warm_cities.py` pre-warms those cells so `/climate` pages serve +from cache and a crawl never bursts the archive quota. It's idempotent, paced, +and claims a flock — deploys launch it detached with no overlap check, so two +overlapping runs would otherwise double-spend quota and race the cache writes. + +--- + +## Work on a live host + +Prefer Centralis: `run_on_host`, `fleet_status`, `deployed_version`, +`logs_query`, `sql_query`. + +You may, without asking: read anything; restart or roll a wedged service; clear +a full disk; kill a runaway process; bounce a crash-looping container; run the +blessed deploy path onto an already-published image. + +You must not: hand-edit the checkout to persist a change (`git reset --hard` +eats it); hand-edit `/etc/thermograph.env` (rendered artifact); force-push; +deploy prod from an unmerged branch; `docker system prune -a` on a live box (it +deletes your rollback image — `docker image prune -f` is the safe form); or +restart the `_db` service casually. Postgres/Timescale restarts and volume +operations get confirmed with the operator first. + +Next: [Traps and stale docs](11-traps.md). diff --git a/docs/onboarding/11-traps.md b/docs/onboarding/11-traps.md new file mode 100644 index 0000000..d3e4583 --- /dev/null +++ b/docs/onboarding/11-traps.md @@ -0,0 +1,268 @@ +# 11. Traps and stale docs + +Two kinds of thing here: **live traps** (correct code that will surprise you) +and **stale docs** (files in this repo that currently describe a system that no +longer exists). The second list matters as much as the first — this repo has a +lot of documentation, most of it excellent, and some of it describing the world +before a rewrite. + +Everything below was checked against the tree at the time of writing. + +--- + +## Stale docs in this repo + +### `frontend/CLAUDE.md` and `frontend/README.md` describe a service that is not deployed + +**The live frontend is Go** (`frontend/server/`, module `thermograph/frontend`). +Both documents describe the Python FastAPI/Jinja implementation +(`frontend/app.py`, `content.py`, `api_client.py`, `format.py`, +`templates/*.html.j2`), which was superseded by the Go port and is kept in-tree +as the reference the port was made from. + +What is still **correct** in them: every contract (API-version pinning, +`pctOrd`, the `/cell` + ETag relationship, the Fahrenheit set, boot-must-survive- +an-unreachable-backend), every environment variable name and default, and all of +`DESIGN.md`. + +What is **wrong**: the language, the file paths, and — notably — +`frontend/CLAUDE.md`'s claim that `make test-unit` is *"the tier CI runs."* It +isn't any more. CI builds the frontend image, and `frontend/Dockerfile`'s +builder stage runs `gofmt -l` + `go vet` + `go test ./...` before compiling, so +a failing **Go** test is what fails CI. The Python tiers are local-only now. + +Also note `api_client.py` carries a **third** copy of the `API_VERSION` pin. +It isn't deployed, but keep it in step or you'll debug a phantom locally. + +### `infra/DEPLOY.md` describes the pre-Swarm, pre-monorepo deploy + +Three separate layers of drift: + +- It says **prod is deployed with `terraform apply`**. Prod is deployed by a + push to `release` firing `deploy.yml`, exactly like beta's `main` path — see + [07](07-ci-and-release.md). +- It says prod runs a **compose** stack. Prod runs **Swarm** + (`infra/deploy/stack/`). +- Its file table points at `/opt/thermograph/deploy/deploy.sh` and + `/opt/thermograph/docker-compose.yml`. Both moved under `infra/` at the + monorepo cutover. + +The document's own header flags the legacy `provision.sh` / systemd / venv +walkthrough as superseded — but the drift goes further than that note admits. +The **key-management** sections (Part 1) and the `/etc/hosts` registry note are +still accurate and useful. + +### `infra/DEPLOY-DEV.md` and `infra/ACCESS.md` reference deleted workflows + +Both talk about `.forgejo/workflows/deploy-dev.yml`. It does not exist. The two +LAN-dev deploy workflows were deleted rather than ported at the CI +consolidation — they were documented as inert (they targeted a monorepo layout +at `~/thermograph-dev` on a box still holding a split-era checkout). **LAN dev +is a local `make dev-up` concern, not a CI environment.** + +`ACCESS.md` §3c also lists `{build,pr-build,deploy-dev,deploy}.yml` as the +workflow set; there are now nine files and the deploy/build ones were collapsed. + +Everything else in `ACCESS.md` — the host table, the agent-access model, the key +rotation procedure, the Swarm/Forgejo tracks — is current and worth reading. + +### `backend/README.md` is written for the split-repo era + +It describes `thermograph-backend` as its own repo with sibling repos, uses the +retired image path `git.thermograph.org/emi/thermograph-backend/app`, and says +*"there is no Makefile in this repo yet"* — there is (`make test`, `make smoke`, +`make clean-venv`). + +The **"How it works"** section (grid → data → percentiles → caching) is one of +the best explanations of the domain model anywhere in the repo. Read it; just +mentally translate the topology. + +### `CUTOVER-NOTES.md` carries a superseded block, correctly labelled + +Its per-domain workflow names (`backend-build-push.yml`, `backend-deploy.yml`, …) +no longer exist — the six deploy workflows collapsed into `deploy.yml` and the +two build-push workflows into `build-push.yml`. The file says so in an explicit +"Superseded 2026-07-25" note, and everything it describes about *behaviour* is +unchanged. + +It also references `backend/MONOREPO-CUTOVER-PLAN.md`, which is not in the tree. + +`CUTOVER-NOTES.md` remains the source of truth for what is and isn't live. + +### The domain `CLAUDE.md` files still use repo-era wording + +They say "this repo" and link to sibling repo names. Mentally map +`thermograph-backend` → `backend/`, and fix the wording as you touch files near +it — that's the standing instruction in the root `CLAUDE.md`. + +### `observability/README.md`: merging is not deploying + +`/opt/observability` on beta and prod is a checkout of the **archived** +`emi/thermograph-observability` repo. It can never `git pull` again. A previous +observability PR merged and had **zero live effect** for exactly this reason. +The README documents the hand-ship procedure; until the checkouts are re-pointed +at the monorepo, that's the only path. Flagged again in +[09](09-observability.md). + +--- + +## Live traps + +### The `docker-compose.test.yml` files still pull retired image paths + +Both `backend/docker-compose.test.yml` and `frontend/docker-compose.test.yml` +default `BACKEND_IMAGE_PATH` to **`emi/thermograph-backend/app`** — the +split-era path. The current image is `emi/thermograph/backend`. + +- `backend/make smoke` is fine: it builds `:local` and never pulls. +- `frontend/make backend-up` **does** pull, and pins + `THERMOGRAPH_BACKEND_TEST_TAG` to `v0.0.2-split-ci`. The old packages were + deliberately left in the registry for rollback and GC'd only after burn-in, so + this may work, may serve a very old backend, or may 404 depending on when you + read this. If `make backend-up` fails or behaves oddly, set the path and tag + explicitly: + ```bash + BACKEND_IMAGE_PATH=emi/thermograph/backend \ + THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex> make backend-up + ``` + +### `static/units.js`'s `F_REGIONS` is guarded by nothing + +Source comments in three files claim *"a test asserts all three stay +identical."* The only automated assertion is +`format_test.go::TestFCountriesMatchesBackend`, which re-parses the backend's +Python and compares the **Go** copy. Nothing reads `units.js`. + +All four copies currently *are* identical (14 codes, checked). But the browser +copy is held in step by convention alone. Extending the Go test to parse +`units.js` too is a good first contribution. + +### The compose project name is load-bearing + +`infra/docker-compose.yml` pins `name: thermograph`. Without it, running compose +from `infra/` derives project `infra` — a silently **new** stack with empty +volumes beside the running one. `deploy-dev.sh` exports +`COMPOSE_PROJECT_NAME=thermograph-dev` to keep LAN dev separate on purpose. +Keep both halves. + +### `deploy.sh` hard-resets the host checkout + +`git reset --hard origin/$BRANCH` runs before anything else. Uncommitted edits +in `/opt/thermograph` evaporate. `infra-sync.yml` does the same reset without +deploying. The one deliberate exception is the untracked +`deploy/.image-tags.env` — losing *that* makes the next single-service deploy +roll the sibling onto `local`. + +### The deploy scripts eat your rollback image + +Both `deploy.sh` and `deploy-stack.sh` **end** by deleting images outside the +running pair. On beta that succeeds, so beta typically holds **no local rollback +target at all**; on prod it usually fails only because Swarm's stopped task +containers still hold references. `docker system prune -a` is forbidden on a +live box for the same reason — but the deploy script is the thing that actually +eats the image, not prune. Verify a tag exists before promising a rollback. + +### `daemon` and `lake` are never deploy targets on their own + +They ride with `backend`. All three run the same image at the same tag — a +second binary and a second role inside one image — and the `/internal/*` +contract between daemon and web has no version negotiation. There's also a +mechanical reason: a single-service deploy runs `up -d --no-deps `, so +a service in compose but absent from `TARGETS` would simply never be created. + +### `docker compose config --images ` does not filter + +Confirmed live on Compose v5.3.1: it prints **every** service's image, one per +line, in file order. A `| head -1` therefore grabbed `db`'s image, and +`deploy.sh`'s daemon capability probe found no daemon binary in a Postgres image +and dropped the daemon from *every* backend deploy. The script now builds the +image reference from the same variables compose interpolates. + +### Prod's Loki streams carry no `service` label + +Prod runs Swarm; its `job="docker"` streams have no `service` label. A raw +`{host="prod", service="backend"}` query matches nothing and reads as "no logs" +when it means "wrong selector". Use `logs_query`'s `service` argument. Prod +container names also change on every redeploy (the task suffix) — resolve via +`docker ps --filter name=` at call time, never hardcode. + +### A Grafana alert rule can provision cleanly and never fire + +A rule whose `condition` names a non-existent refId is valid YAML, provisions +without complaint, and pages nobody. Likewise a contact point holding the +literal string `$DISCORD_ALERT_WEBHOOK_URL` looks completely healthy from the +API — and the API **redacts** the URL, so you cannot confirm interpolation by +reading it back. The only proof is a message arriving in `#ops-alerts`. + +### Terraform has no state anywhere + +A casual `terraform apply` would attempt full re-provisioning of live hosts. +It's executable documentation until state is bootstrapped. + +### The registry is mesh-only + +`git.thermograph.org` resolves publicly to beta's IP, but beta's Caddy rejects +`/v2/*` from outside the WireGuard mesh. Any host that pulls needs +`10.10.0.2 git.thermograph.org` in `/etc/hosts`. Beta itself doesn't — it *is* +the box. + +### Forgejo runners only have the labels they registered with + +GitHub implicitly tags every self-hosted runner `self-hosted`; Forgejo does not. +`runs-on: [self-hosted, thermograph-lan]` is permanently unschedulable there — +a real bug, found and fixed once. If a job silently stops appearing in the +Actions history, check this first. + +### Forgejo's `needs:` doesn't cross workflow files + +`build-push.yml` and `deploy.yml` fire from the same push with no ordering +guarantee, and a deploy racing ahead of its build and failing "not found" is +confirmed-live. That's why `deploy.sh`'s pull has a bounded ~5-minute retry. + +### `/api/v2/cell?prefetch=1` must never spend quota + +The hard guarantee: a cold cell answers `204` with no body. Anything named +"prefetch" or "warm" calls the cache-only loaders. Breaking this gets the whole +service rate-limited. + +### Background timers must go through leader election + +`singleton.claim_leader()` — flock per host, Postgres advisory lock per cluster. +A timer-driven sweep running in N uvicorn workers × M Swarm hosts multiplies +Open-Meteo quota use N×M. That is the documented cause of overnight 429/503s +after prod went to 3 workers. The heartbeat is the deliberate exception — a beat +is one local file append, so duplicates are free. + +### `THERMOGRAPH_DATA_DIR` exists because of a real collision + +The backend's Python package `data/` and its runtime data dir `/data` +collide at the same path once the code lives at the image root. Mounting a +volume there shadows and erases `data/*.py`, so `import data.climate` fails at +boot. Point the env var **outside the code tree** in a container. And never +reintroduce `__file__`-relative paths in a module — `paths.py` resolves +everything from the repo root for this reason. + +### Alloy can silently stop tailing a container + +Observed on prod, 13½ hours of `thermograph_worker` logs never reaching Loki +while the container was healthy and writing. Not root-caused. Workaround: +`sudo docker restart alloy-alloy-1`. `ProdWorkerContainerSilent` now catches the +class. + +### Nominatim is ~1 req/s, on one thread + +All reverse geocoding funnels through a single dedicated worker draining a +queue. The earlier design slept inside the caller's thread — which in the server +is one of Starlette's shared sync-threadpool threads. + +--- + +## If you fix one of these + +Docs are cheap to correct and the root `CLAUDE.md` already asks you to fix +repo-era wording as you touch files near it. If you're in `frontend/` anyway, +bringing `frontend/CLAUDE.md` in line with the Go service would be the single +highest-value documentation change available — it's the file an agent reads +*before* touching that domain. + +Back to the [index](README.md). diff --git a/docs/onboarding/README.md b/docs/onboarding/README.md new file mode 100644 index 0000000..9b177df --- /dev/null +++ b/docs/onboarding/README.md @@ -0,0 +1,70 @@ +# Onboarding — becoming a full contributor to Thermograph + +This is the developer onboarding path for the `emi/thermograph` monorepo: what +the product is, how the code is shaped, how to run it, what will break silently +if you get it wrong, and how a change actually travels from your editor to +`thermograph.org`. + +**Scope.** This covers *working in this repo*. Cross-cutting architecture +decision records and operator runbooks live in the separate `thermograph-docs` +repo (reachable through Centralis's `docs_search`) — this set links out rather +than duplicating them. Where a fact is owned by a `CLAUDE.md`, that file stays +the source of truth and this set explains the context around it. + +## Reading order + +| # | Doc | Read it when | +|---|-----|--------------| +| 1 | [Orientation](01-orientation.md) | First. What the product actually claims, the estate, the four non-negotiable rules. | +| 2 | [Local setup](02-setup.md) | Before you touch anything. Toolchain, verified run/test recipes for every service. | +| 3 | [Repo map](03-repo-map.md) | To find things. Every domain and directory, and which files matter. | +| 4 | [Backend deep dive](04-backend.md) | Before your first backend change. Data pipeline, grading, caching, roles, daemon. | +| 5 | [Frontend deep dive](05-frontend.md) | Before your first frontend change. The Go SSR service, static assets, design system. | +| 6 | [Cross-service contracts](06-contracts.md) | **Before any change that touches both.** These break silently and in production. | +| 7 | [CI and release](07-ci-and-release.md) | Before you open a PR. Nine workflows, three branches, three environments. | +| 8 | [Infra and secrets](08-infra-secrets.md) | Before you touch deploy, compose, or a secret. | +| 9 | [Observability](09-observability.md) | When something is wrong and you need to see it. | +| 10 | [Recipes](10-recipes.md) | Task-shaped walkthroughs for the things you'll actually do. | +| 11 | [Traps and stale docs](11-traps.md) | **Skim early, re-read often.** Which docs in this repo currently lie, and why. | + +## The short version + +Thermograph grades how unusual today's weather is at any point on Earth against +~45 years of that exact location's own history. Percentiles, never thermometer +readings. + +- **`backend/`** — Python 3.12 / FastAPI. Owns all climate data, grading, the + API, accounts, notifications, plus a Go daemon (`backend/daemon/`) that owns + the Discord gateway and recurring timers. Ships as `emi/thermograph/backend`. +- **`frontend/`** — **Go** SSR service (`frontend/server/`) plus every static + asset. No climate data, no database, no compute. Ships as + `emi/thermograph/frontend`. (The Python implementation at `frontend/*.py` is + the superseded original — see [traps](11-traps.md).) +- **`infra/`** — compose and Swarm files, deploy scripts, the SOPS secrets + vault, Terraform, ops query tooling. +- **`observability/`** — Loki + Grafana on beta, an Alloy agent per node. + +Branches stage environments: PR → `dev` (LAN dev) → `main` (beta) → `release` +(prod). The two app domains build and deploy **independently** — that +independence is the whole reason the split-then-reunify history exists, and +[contracts](06-contracts.md) is the list of things that keep it safe. + +## Your first day + +1. Read [Orientation](01-orientation.md) and [Traps](11-traps.md). +2. Work through [Local setup](02-setup.md) until `make test` is green in both + `backend/` and `frontend/server/`. +3. Read [Repo map](03-repo-map.md) with the tree open beside it. +4. Pick something small in the domain you're least afraid of, and follow + [Recipes](10-recipes.md) end to end — including opening the PR. + +## Your first week + +- Read [Contracts](06-contracts.md) properly. Every item on that list has + already cost someone a production incident somewhere in this project's + history; that's why each one is written down. +- Get Centralis wired up (see [Local setup](02-setup.md)) and ask it + `fleet_status`, `logs_overview`, `deployed_version`. You cannot debug this + estate blind, and the boxes are not reachable from a laptop off the + WireGuard mesh. +- Read the `CLAUDE.md` for each domain. They are terse, current, and binding. -- 2.45.2 From 5f20fba9f5437b9581aef957708c79649c69f64b Mon Sep 17 00:00:00 2001 From: emi Date: Sat, 25 Jul 2026 21:11:32 +0000 Subject: [PATCH 04/37] frontend: fix the three inconsistencies the onboarding guide found (#99) --- backend/CLAUDE.md | 15 +- backend/docker-compose.test.yml | 6 +- docs/onboarding/02-setup.md | 10 + docs/onboarding/05-frontend.md | 10 +- docs/onboarding/06-contracts.md | 25 ++- docs/onboarding/11-traps.md | 103 +++++----- frontend/CLAUDE.md | 108 ++++++++--- frontend/Dockerfile | 13 ++ frontend/README.md | 182 +++++++++--------- frontend/docker-compose.test.yml | 13 +- frontend/scripts/backend-for-tests.sh | 38 +++- .../server/internal/format/format_test.go | 83 ++++++-- 12 files changed, 404 insertions(+), 202 deletions(-) diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index ccc1b68..02b479a 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -38,13 +38,20 @@ for systemd/CI callers. Entry target is `app:app`. `cache.js` reads `null` cross-origin and never revalidates. Don't change the ETag derivation or that list without checking `frontend/static/cache.js`. - **`data/grading.py::pct_ordinal()`** is mirrored by `frontend`'s - `shared.js::pctOrd()` — floor a percentile into `1..99`, never 0 or 100. + `server/internal/format::PctOrdinal` (SSR) and `static/shared.js::pctOrd()` + (browser) — floor a percentile into `1..99`, never 0 or 100. `TEMP_BANDS`/`RAIN_BANDS` are the source of truth for tier names and thresholds; changing them without the frontend produces tiers drawn in colours that disagree with the labels the API returns. -- **Fahrenheit country set** — `api/content_payloads.py`'s `F_COUNTRIES` must stay - identical to `frontend`'s `format.py::F_COUNTRIES` and `static/units.js`'s - `F_REGIONS`. There is a test asserting this. +- **Fahrenheit country set** — `api/content_payloads.py`'s `F_COUNTRIES` is + canonical; `frontend`'s `server/internal/format::FCountries` and + `static/units.js`'s `F_REGIONS` must stay identical to it. Both are asserted + by tests in `frontend/server/internal/format/format_test.go` — but note the + backend cross-check **skips in CI** (the frontend image's build context is + `frontend/`, so this file is unreachable from the builder stage, which is the + only place CI runs those tests). It fires on a full checkout. The `units.js` + check does run in the image build. `frontend/format.py`'s copy is the + superseded Python service's and is not deployed. - **`/healthz` and `/api/version` are deliberately I/O-free** so they stay cheap under tight healthcheck intervals. diff --git a/backend/docker-compose.test.yml b/backend/docker-compose.test.yml index 2b1addb..d5ff99e 100644 --- a/backend/docker-compose.test.yml +++ b/backend/docker-compose.test.yml @@ -22,7 +22,11 @@ services: backend: # Defaults to a locally-built `:local` image (scripts/smoke.sh builds it). In CI, # set BACKEND_IMAGE_TAG=sha-<12hex> to smoke the exact published image. - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:-local} + # The path must match what build-push.yml publishes -- emi/thermograph/backend. + # It read emi/thermograph-backend/app (the retired split-era path) until this + # was corrected; harmless for the default `:local` build, wrong the moment + # BACKEND_IMAGE_TAG names a real published tag. + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} depends_on: db: condition: service_healthy diff --git a/docs/onboarding/02-setup.md b/docs/onboarding/02-setup.md index ba79e75..52aa099 100644 --- a/docs/onboarding/02-setup.md +++ b/docs/onboarding/02-setup.md @@ -139,9 +139,19 @@ make backend-up # pulls + runs the published backend image + throwaway db make backend-down ``` +The image tag is **derived from your checkout** — `sha-<12hex of git log -1 -- +backend/>`, the same domain-keyed rule `build-push.yml` and `deploy.yml` use — +so the harness follows the tree. If those backend commits are still local-only, +no image exists yet and the script says so; pin a published build with +`THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex>`. + `make test-integration` runs the Python integration tier against that. CI does **not** run it — it needs a live backend container. +> Heads-up: against a freshly-booted throwaway backend, that tier currently +> fails 7 of 16 with `503` — the database is empty, so nothing is warm. +> Pre-existing, and unrelated to which image tag you use. + ## The daemon ```bash diff --git a/docs/onboarding/05-frontend.md b/docs/onboarding/05-frontend.md index 8241b54..c1dd17e 100644 --- a/docs/onboarding/05-frontend.md +++ b/docs/onboarding/05-frontend.md @@ -8,11 +8,11 @@ HTTP. > **The live service is Go.** `frontend/server/` builds, tests, ships and runs. > The Python files one directory up (`app.py`, `content.py`, `api_client.py`, > `format.py`, `content_loader.py`, `paths.py`, `templates/*.html.j2`) are the -> superseded original implementation, kept in-tree as the reference the port -> was made from. `frontend/CLAUDE.md` and `frontend/README.md` still describe -> the Python service — see [traps](11-traps.md). Everything in those docs about -> **contracts, env vars and design** is still correct; only "which language and -> which file" changed. +> superseded original implementation, kept in-tree as the reference the port was +> made from. Nothing deploys them, and they hold duplicate copies of a couple of +> contracts (`api_client.py`'s `API_VERSION` pin, `format.py`'s `F_COUNTRIES`) — +> keep those in step if you run them locally. Whether to delete them is an open +> question; raise it rather than doing it in passing. ## Shape of the Go service diff --git a/docs/onboarding/06-contracts.md b/docs/onboarding/06-contracts.md index dad33a4..a0020b3 100644 --- a/docs/onboarding/06-contracts.md +++ b/docs/onboarding/06-contracts.md @@ -104,16 +104,23 @@ Fourteen codes: `US PR GU VI AS MP UM BS BZ KY PW FM MH LR`. | browser | `frontend/static/units.js::F_REGIONS` | | Python SSR *(not deployed)* | `frontend/format.py::F_COUNTRIES` | -`format_test.go::TestFCountriesMatchesBackend` re-parses the backend's Python -source and asserts the Go copy matches — a genuinely good guard. +Two tests in `frontend/server/internal/format/format_test.go` guard this: -> ⚠️ **Verified gap.** Several source comments claim "a test asserts all three -> stay identical." As of this writing that is not true: no test reads -> `static/units.js`. All four copies currently *are* identical (checked), but -> the browser copy is held in step by convention alone. If you touch this set, -> update `units.js` by hand and diff it yourself. Extending the Go test to -> parse `units.js` too would close this properly and is a good first -> contribution. +- `TestFCountriesMatchesBackend` re-parses the backend's Python and asserts the + Go copy matches. **This one skips in CI** — the frontend image's build context + is `frontend/`, so `backend/` is unreachable from the builder stage, and the + builder stage is the only place CI runs these tests. Treat it as a + checkout-only guard: it fires on your machine and in a full-checkout run, not + during the image build. +- `TestFCountriesMatchesUnitsJS` parses `static/units.js` and diffs both + directions. This one *does* run in the image build — `frontend/Dockerfile` + copies that single file into the builder stage for exactly that reason — so a + divergence fails the build. + +The browser copy was unguarded until that second test was added; several source +comments had long claimed "a test asserts all three stay identical" when only +the Go↔backend pair was checked. The Python `frontend/format.py` copy remains +guarded only by the local Python tier, which is fine — it isn't deployed. ## 6. Backend boot must survive an unreachable frontend, and vice versa diff --git a/docs/onboarding/11-traps.md b/docs/onboarding/11-traps.md index d3e4583..4604b5d 100644 --- a/docs/onboarding/11-traps.md +++ b/docs/onboarding/11-traps.md @@ -12,27 +12,23 @@ Everything below was checked against the tree at the time of writing. ## Stale docs in this repo -### `frontend/CLAUDE.md` and `frontend/README.md` describe a service that is not deployed +### ~~`frontend/CLAUDE.md` and `frontend/README.md` describe a service that is not deployed~~ — fixed -**The live frontend is Go** (`frontend/server/`, module `thermograph/frontend`). -Both documents describe the Python FastAPI/Jinja implementation -(`frontend/app.py`, `content.py`, `api_client.py`, `format.py`, -`templates/*.html.j2`), which was superseded by the Go port and is kept in-tree -as the reference the port was made from. +**Fixed.** Both now describe the Go service (`frontend/server/`), state plainly +that the Python files at that level are the superseded original, and correct +`CLAUDE.md`'s claim that `make test-unit` is *"the tier CI runs"* — it isn't. +CI builds the frontend image, and `frontend/Dockerfile`'s builder stage runs +`gofmt -l` + `go vet` + `go test ./...` before compiling, so a failing **Go** +test is what fails CI. The Python tiers are local-only. -What is still **correct** in them: every contract (API-version pinning, -`pctOrd`, the `/cell` + ETag relationship, the Fahrenheit set, boot-must-survive- -an-unreachable-backend), every environment variable name and default, and all of -`DESIGN.md`. +Still worth knowing: the superseded Python files carry **duplicate copies of +some contracts** — `api_client.py` has a third `API_VERSION` pin and +`format.py` a fourth `F_COUNTRIES`. Nothing deploys them, but if you run them +locally, keep them in step or you'll debug a phantom. -What is **wrong**: the language, the file paths, and — notably — -`frontend/CLAUDE.md`'s claim that `make test-unit` is *"the tier CI runs."* It -isn't any more. CI builds the frontend image, and `frontend/Dockerfile`'s -builder stage runs `gofmt -l` + `go vet` + `go test ./...` before compiling, so -a failing **Go** test is what fails CI. The Python tiers are local-only now. - -Also note `api_client.py` carries a **third** copy of the `API_VERSION` pin. -It isn't deployed, but keep it in step or you'll debug a phantom locally. +Whether to delete them outright is an open question, not a settled one: the +Python test tiers still run, against fixtures the Go tests share. Raise it +rather than doing it in passing. ### `infra/DEPLOY.md` describes the pre-Swarm, pre-monorepo deploy @@ -108,34 +104,49 @@ at the monorepo, that's the only path. Flagged again in ## Live traps -### The `docker-compose.test.yml` files still pull retired image paths +### ~~The `docker-compose.test.yml` files pull retired image paths~~ — fixed -Both `backend/docker-compose.test.yml` and `frontend/docker-compose.test.yml` -default `BACKEND_IMAGE_PATH` to **`emi/thermograph-backend/app`** — the -split-era path. The current image is `emi/thermograph/backend`. +**Fixed.** Both now use `emi/thermograph/backend`, the path `build-push.yml` +actually publishes. -- `backend/make smoke` is fine: it builds `:local` and never pulls. -- `frontend/make backend-up` **does** pull, and pins - `THERMOGRAPH_BACKEND_TEST_TAG` to `v0.0.2-split-ci`. The old packages were - deliberately left in the registry for rollback and GC'd only after burn-in, so - this may work, may serve a very old backend, or may 404 depending on when you - read this. If `make backend-up` fails or behaves oddly, set the path and tag - explicitly: - ```bash - BACKEND_IMAGE_PATH=emi/thermograph/backend \ - THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex> make backend-up - ``` +The frontend harness had the worse half of this: it also pinned +`THERMOGRAPH_BACKEND_TEST_TAG` to the split-era `v0.0.2-split-ci`. Rather than +swap one hardcoded pin for another, `scripts/backend-for-tests.sh` now +**derives** the tag from the checkout — `sha-<12hex of git log -1 -- backend/>`, +the same domain-keyed rule `build-push.yml` and `deploy.yml` both use — so the +harness follows the tree instead of drifting behind a pin. `docker-compose.test.yml` +requires the variable (`:?`) rather than defaulting it, so a stale pin can't +creep back in silently. -### `static/units.js`'s `F_REGIONS` is guarded by nothing +If the derived tag has no published image (backend commits that are still +local-only), the script says so and tells you to pin one: -Source comments in three files claim *"a test asserts all three stay -identical."* The only automated assertion is -`format_test.go::TestFCountriesMatchesBackend`, which re-parses the backend's -Python and compares the **Go** copy. Nothing reads `units.js`. +```bash +THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex> make backend-up +``` -All four copies currently *are* identical (14 codes, checked). But the browser -copy is held in step by convention alone. Extending the Go test to parse -`units.js` too is a good first contribution. +> Unrelated but adjacent: `make test-integration` against a freshly-booted +> throwaway backend currently fails 7 of 16 tests with `503`, because nothing +> in that empty database is warm. Verified identical on the old image and the +> new one, so it predates these changes and is **not** caused by them. Not +> investigated further here. + +### ~~`static/units.js`'s `F_REGIONS` is guarded by nothing~~ — fixed + +**Fixed.** `format_test.go::TestFCountriesMatchesUnitsJS` now parses +`static/units.js` and diffs it against the Go set in both directions. + +The subtlety worth knowing: the *existing* backend cross-check +(`TestFCountriesMatchesBackend`) **skips in CI**. The frontend image's build +context is `frontend/`, so `backend/` is structurally unreachable from the +builder stage — and the builder stage is the only place CI runs these tests. It +is a checkout-only guard and always was. + +The new `units.js` check does not have that problem, because `static/` *is* +inside the context: `frontend/Dockerfile` copies that one file into the builder +stage specifically so the assertion runs during the image build. Verified by +mutating `units.js` and confirming the **image build fails**, not just the local +test run. ### The compose project name is load-bearing @@ -259,10 +270,10 @@ is one of Starlette's shared sync-threadpool threads. ## If you fix one of these -Docs are cheap to correct and the root `CLAUDE.md` already asks you to fix -repo-era wording as you touch files near it. If you're in `frontend/` anyway, -bringing `frontend/CLAUDE.md` in line with the Go service would be the single -highest-value documentation change available — it's the file an agent reads -*before* touching that domain. +Docs are cheap to correct, and the root `CLAUDE.md` already asks you to fix +repo-era wording as you touch files near it. The three entries struck through +above were fixed in the same change that added this guide; the rest are still +open, and `infra/DEPLOY.md` is probably the highest-value one left — it +describes a deploy model that has since changed twice. Back to the [index](README.md). diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 5542457..4804185 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -7,65 +7,117 @@ no compute — everything comes from `backend/`'s content API over HTTP. Read the root `CLAUDE.md` first for the branch model, deploy contract and image names. +## The live service is Go — `server/` + +`server/` (module `thermograph/frontend`, Go 1.26) is what builds, tests, ships +and runs. It is the only thing in the `emi/thermograph/frontend` image. + +The Python files at this level — `app.py`, `content.py`, `api_client.py`, +`format.py`, `content_loader.py`, `paths.py`, `templates/*.html.j2` — are the +**superseded original implementation**, kept as the reference the port was made +from. Nothing deploys them. Don't add features there; don't take their file +paths as current. (They still carry duplicate copies of some contracts below — +notably `api_client.py`'s `API_VERSION` pin and `format.py`'s `F_COUNTRIES`. If +you use them locally, keep them in step or you will debug a phantom.) + +Deleting them is a live question, not a settled one — the Python test tiers +still run against the fixtures the Go tests share. Raise it rather than doing it +in passing. + ## Build & verify -- `make test` — whole suite (`scripts/test.sh`). -- `make test-unit` — hermetic unit tier: SSR rendering fed committed fixtures, - no Docker. **This is the tier CI runs.** -- `make test-integration` — pulls and runs the real backend image and tests the - live contract. Local only; CI does not run it. +- `cd server && go build ./... && go vet ./... && go test ./...` — **the suite + that matters.** +- **CI runs it inside `Dockerfile`'s builder stage**, which does `gofmt -l` + + `go vet` + `go test ./...` before compiling. A failing Go test therefore fails + the image build — that is the whole frontend check. There is no separate + frontend test step in any workflow. +- `make test-unit` / `make test-integration` — the **Python** tiers. Local only; + CI runs neither. The unit tier is hermetic (SSR rendering fed committed + fixtures); the integration tier pulls and runs a real backend image. - `make backend-up` / `make backend-down` — a local backend container for dev. + The image tag is derived from this checkout (last commit touching `backend/`, + the same domain-keyed rule build-push and deploy use); override with + `THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex>`. - `make capture-fixtures` — refresh `tests/fixtures/*.json` from a live backend. + **These fixtures feed the Go tests too**, not just the Python ones. ## Running it -`THERMOGRAPH_API_BASE_INTERNAL` is **required** — `api_client.py` raises at -import if unset. +`THERMOGRAPH_API_BASE_INTERNAL` is **required** — the boot fails loudly without +it, as the Python raised at import. + +Build in `server/`, run from `frontend/`: `static/` and `content/` resolve +relative to the working directory, while templates are embedded in the binary. ```bash +cd server && go build -o thermograph-frontend . +cd .. THERMOGRAPH_API_BASE_INTERNAL=http://127.0.0.1:8137 THERMOGRAPH_BASE=/thermograph \ - .venv/bin/uvicorn app:app --host 0.0.0.0 --port 8080 + ./server/thermograph-frontend ``` Other env: `THERMOGRAPH_BASE` (default `/thermograph`; the Dockerfile sets `/`), `THERMOGRAPH_API_VERSION` (default `v2`), `THERMOGRAPH_API_BASE_PUBLIC` (browser-facing backend origin when cross-origin; empty = same-origin, today's -default), `THERMOGRAPH_SSR_CACHE_TTL` (default 600s). +default), `THERMOGRAPH_SSR_CACHE_TTL` (default 600s), +`THERMOGRAPH_GOOGLE_VERIFY` / `THERMOGRAPH_BING_VERIFY`, `PORT` (default 8080). + +Boot is fail-loud on bad **config or content**: a missing backend URL, a garbage +TTL, or malformed `content/*.yaml` all stop the process rather than 500 on the +first request. ## Layout -- **`content.py`** — SSR routes (climate hub, city, month, records, glossary, - about, privacy) + `robots.txt` + `sitemap.xml`. Each fetches from the backend - via `api_client.py`; the backend's `content_payloads.py` owns `page_title` / - `canonical_path` / `breadcrumb` / `jsonld`, not this domain. +- **`server/internal/content/`** — the SSR page handlers (climate hub, city, + month, records, glossary, about, privacy) + `robots.txt` + `sitemap.xml`, + plus the template funcmap and SEO helpers. The backend's + `content_payloads.py` owns `page_title` / `canonical_path` / `breadcrumb` / + `jsonld`, **not this domain**. +- **`server/internal/contentapi/`** — the backend `/content/*` client: TTL + cache, bounded LRU, per-key single-flight, origin forwarding. All four + properties are load-bearing; read the file before changing caching. +- **`server/internal/render/`** + `render/templates/*.tmpl` — `html/template` + over an `embed.FS`, plus the ETag helpers. Templates are **embedded**, so a + template change needs a rebuild. +- **`server/internal/{config,format,contentdata,handlers}/`** — env parsing, + unit-aware °C/°F formatting and band names, the glossary/pages loader, and + the SPA-shell + static handlers. - **`static/*.js`** — `app.js` (map/search/results + inline SVG chart), `calendar.js`/`day.js`/`score.js`/`compare.js` (SPA shells), `account.js` (auth), `cache.js` (IndexedDB cache + `/cell` bundle prefetch), `shared.js`. - **`static/style.css`** — the single hand-written stylesheet; design tokens live here, documented in `DESIGN.md`. -- **`templates/*.html.j2`**, **`content/*.yaml`** (structured SSR copy, loaded by - `content_loader.py`). +- **`content/*.yaml`** — structured SSR copy, loaded by + `server/internal/contentdata`. ## Contracts with the backend -- **API version is pinned in exactly two places** — `api_client.py`'s - `API_VERSION` (Python) and `static/account.js`'s exported `API_VERSION` plus the - `uv(path)` helper (JS). Never hardcode `api/v2/...` anywhere else. Bump both in - one PR, and only after the target backend's `/api/version` confirms it serves - that version and its `min_frontend` doesn't exclude the one you're leaving. +- **API version is pinned in exactly two places** — `server/internal/config`'s + `THERMOGRAPH_API_VERSION` (Go) and `static/account.js`'s exported + `API_VERSION` plus the `uv(path)` helper (JS). Never hardcode `api/v2/...` + anywhere else. Bump both in one PR, and only after the target backend's + `/api/version` confirms it serves that version and its `min_frontend` doesn't + exclude the one you're leaving. - **`shared.js::pctOrd()` must mirror `backend`'s `data/grading.py::pct_ordinal()`** - in behaviour — floor into `1..99`, never 0 or 100. Every percentile on every - surface goes through one of the two; if they diverge, the same reading says two + in behaviour — floor into `1..99`, never 0 or 100 — and so must + `server/internal/format`'s `PctOrdinal`. Every percentile on every surface + goes through one of the three; if they diverge, the same reading says two different things in two places. - **`/cell` bundle + ETag** — `cache.js` fetches `/api/v2/cell` once per view-set and slices it, revalidating with `If-None-Match`. The slice→view map must track the backend's payload shape. -- **Fahrenheit country set** — `format.py::F_COUNTRIES` and `static/units.js`'s - `F_REGIONS` must stay identical to the backend's `F_COUNTRIES`. -- **Boot must survive an unreachable backend.** `content.py`'s `register()` tries - the IndexNow-key fetch once, catches failure, and falls back to a lazy - per-request lookup. Asynchronous FE/BE deploys depend on this — don't make it - fatal. +- **Fahrenheit country set** — `server/internal/format`'s `FCountries` and + `static/units.js`'s `F_REGIONS` must stay identical to the backend's + `F_COUNTRIES`. Both are now asserted by tests in + `server/internal/format/format_test.go`: the backend cross-check is a + checkout-only guard (the image build context is `frontend/`, so `backend/` is + unreachable there), while the `units.js` check runs in the image build too — + `Dockerfile` copies that one file into the builder stage for exactly that + reason. +- **Boot must survive an unreachable backend.** The IndexNow-key fetch is tried + once at boot, caught, and falls back to a lazy per-request lookup. + Asynchronous FE/BE deploys depend on this — don't make it fatal. ## Commits & PRs diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 692ecb9..920c545 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -35,6 +35,19 @@ COPY server/ ./ COPY tests/fixtures /tests/fixtures COPY content /content +# static/units.js is copied for ONE test: internal/format's +# TestFCountriesMatchesUnitsJS, which asserts the browser's F_REGIONS still +# matches the Go/backend Fahrenheit country set. The builder stage is the only +# place CI ever executes these tests, so without this the check would skip in CI +# and only ever run on a developer's checkout. Same three-levels-up relationship +# the test expects (/src/internal/format -> /static/units.js), anchored the same +# way the two directories above are. +# +# The sibling backend cross-check cannot be wired up this way: the build context +# is frontend/, so backend/ is structurally unreachable and that test stays a +# checkout-only guard. +COPY static/units.js /static/units.js + RUN test -z "$(gofmt -l .)" && go vet ./... && go test ./... # Static binary: CGO off (no libc dependency on Alpine), -trimpath for diff --git a/frontend/README.md b/frontend/README.md index ccc40d5..c22d8e4 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -2,107 +2,92 @@ The server-rendered content pages, the interactive-tool SPA shells, and every static asset for [Thermograph](https://thermograph.org) — grades recent local -weather against ~45 years of climate history. This repo holds **no climate -data and does no compute**; it renders pages and serves the JS/CSS/image -assets that call the backend's API from the browser. It was split out of the -`emi/thermograph` monorepo (`frontend_ssr/` + `frontend/` there) so it can -build its own image and deploy independently of the backend. See -[`CLAUDE.md`](CLAUDE.md) for the full agent-facing detail on the split -topology and deploy contract, and -[`thermograph-docs`](../thermograph-docs) (sibling repo) for the -architecture decision record behind the split. +weather against ~45 years of climate history. This domain holds **no climate +data and does no compute**; it renders pages and serves the JS/CSS/image assets +that call the backend's API from the browser. It builds its own image +(`emi/thermograph/frontend`) and deploys independently of the backend. -## Layout +See [`CLAUDE.md`](CLAUDE.md) for the agent-facing detail on the deploy contract +and the cross-service contracts, and [`DESIGN.md`](DESIGN.md) for the visual +system. + +## The service is Go + +`server/` is the live implementation. The Python files at this level (`app.py`, +`content.py`, `api_client.py`, `format.py`, `content_loader.py`, `paths.py`, +`templates/*.html.j2`) are the **superseded original**, kept as the reference +the port was made from — nothing deploys them. See `server/README.md` for the +port's own notes. ``` -api_client.py HTTP client for the backend's content API (api/content_routes.py - on the backend). Fails loud at import if - THERMOGRAPH_API_BASE_INTERNAL is unset. TTL-cached (10 min - default) in front of every call. -app.py FastAPI app: /healthz, the interactive tool's SPA-shell - routes (calendar/day/score/compare/legend/alerts), and the - StaticFiles mount for everything else. Calls - content.register(app) for the SSR routes below. -content.py Server-rendered pages: climate hub, per-city, month, - records, glossary, about, privacy, robots.txt, sitemap.xml. -content_loader.py Loads content/glossary.yaml + content/pages.yaml (SSR copy), - validated fail-loud at load time. -format.py Unit-aware (°C/°F) formatting for the SSR templates, - ContextVar-scoped active unit. -paths.py Canonical filesystem locations (STATIC_DIR, TEMPLATES_DIR, - CONTENT_DIR), resolved from the repo root. -templates/ Jinja templates content.py renders (base, home, city, month, - hub, records, glossary, glossary_term, about, privacy). -static/ Every static asset: the interactive tool's JS - (app.js, calendar.js, day.js, score.js, compare.js, - account.js, cache.js, shared.js, units.js, mappicker.js, …), - the SPA-shell HTML pages, style.css (all design tokens — - see DESIGN.md), icons/favicons, manifest.webmanifest, sw.js. -content/ Structured SSR copy: glossary.yaml, pages.yaml. -tests/ pytest suite — see the KNOWN GAP note below before relying on it. -tools/ shoot.py — the make-shots visual-verify screenshot tool - (see DESIGN.md). -Dockerfile Builds this repo's own image (python:3.12-slim, - uvicorn app:app), independent of the backend's. +server/ the service (module thermograph/frontend, Go 1.26) + main.go config + mux + static + graceful shutdown + internal/config/ every env var it reads + internal/contentapi/ backend /content/* client: TTL cache, bounded LRU, + per-key single-flight, origin forwarding + internal/content/ SSR page handlers, template funcmap, SEO + internal/contentdata/ glossary.yaml / pages.yaml loader (fail-loud) + internal/format/ unit-aware °C/°F formatting + band names + internal/handlers/ SPA shells + static serving + internal/render/ html/template over embed.FS + ETag helpers + internal/render/templates/ the page templates (embedded, *.tmpl) +static/ every static asset: the interactive tool's JS + (app.js, calendar.js, day.js, score.js, compare.js, + account.js, cache.js, shared.js, units.js, + mappicker.js, …), the SPA-shell HTML pages, + style.css (all design tokens — see DESIGN.md), + icons/favicons, manifest.webmanifest, sw.js +content/ structured SSR copy: glossary.yaml, pages.yaml +tests/fixtures/ golden payloads — consumed by the GO tests as well + as the Python tiers +tests/{unit,integration}/ the Python tiers (local only; CI runs neither) +tools/shoot.py the screenshot sweep for visual verification +Dockerfile golang builder (gofmt + vet + test) → alpine runtime ``` -## How it fits the split - -Four sibling repos replace the old monorepo: - -- **`thermograph-backend`** — the FastAPI API, grading/scoring/grid compute, - and DB. This repo's only runtime dependency. -- **`thermograph-frontend`** (this repo). -- **`thermograph-infra`** — Terraform, compose files, `deploy/deploy.sh` (the - script both repos' deploy workflows SSH into), the secrets vault. -- **`thermograph-docs`** — architecture/decision docs and runbooks. - -Each app repo now builds and publishes its **own** image -(`git.thermograph.org/emi/thermograph-frontend/app`, tagged by git SHA and -semver) instead of the old shared `emi/thermograph/app` image, and deploys via -its own `.forgejo/workflows/deploy.yml` (main → beta) and `deploy-prod.yml` -(release → prod), each rolling only the `frontend` compose service on the -target VPS. +Dependencies are **stdlib plus `gopkg.in/yaml.v3`** — the committed SSR copy in +`content/*.yaml` uses block/folded scalars, so a real YAML parser is required. ## Talking to the backend -All data comes from the backend's content API over HTTP -(`api_client.py`) — this process holds nothing in-process. Two env vars -control the topology: +All data comes from the backend's content API over HTTP — this process holds +nothing. Two env vars control the topology: - **`THERMOGRAPH_API_BASE_INTERNAL`** (required) — where this process reaches the backend server-side, e.g. `http://backend:8137` in compose, or - `http://127.0.0.1:8137` for a local backend checkout. `api_client.py` raises - at import if this is unset — a missing backend URL should break boot, not - silently 500 the first request. -- **`THERMOGRAPH_API_BASE_PUBLIC`** (optional) — the backend's - browser-facing origin, used only when frontend and backend are served - cross-origin (e.g. a genuinely separate LAN-dev topology). Left empty - (the default), asset/API references stay relative and same-origin, which is - today's real deployed topology. + `http://127.0.0.1:8137` for a local backend. Boot **fails loudly** if unset: + a missing backend URL should break the boot, not silently 500 on the first + request. +- **`THERMOGRAPH_API_BASE_PUBLIC`** (optional) — the backend's browser-facing + origin, used only when frontend and backend are served cross-origin. Left + empty (the default), asset/API references stay relative and same-origin, + which is today's real deployed topology. + +`THERMOGRAPH_BASE` (default `/thermograph`) must be set identically on both +frontend and backend in every real deployment — it's how both processes agree on +the shared URL prefix (or the deployed clean-root `/`, which the Dockerfile +sets). **Same-origin cookie-auth caveat:** the interactive tool's auth (`static/account.js`) is an HttpOnly session cookie. It uses `credentials: "include"` (not the fetch default) specifically so the cookie -still rides along if this script is ever loaded cross-origin, but the backend -must actually be configured to accept credentialed cross-origin requests -(CORS + `SameSite`) for that case to work — in the default same-origin -deployment this is a non-issue. Don't assume a fully decoupled, independently- -hosted frontend "just works" for logged-in features without checking that -cookie/CORS configuration first. - -`THERMOGRAPH_BASE` (default `/thermograph`) must be set identically on both -frontend and backend in every real deployment — it's how both processes agree -on the shared URL prefix (or the deployed clean-root `/`, which the Dockerfile -sets by default). +still rides along if this script is ever loaded cross-origin — but the backend +must actually be configured to accept credentialed cross-origin requests (CORS + +`SameSite`) for that case to work. In the default same-origin deployment this is +a non-issue. Don't assume a fully decoupled, independently-hosted frontend "just +works" for logged-in features without checking that configuration first. ## Build & run +Build in `server/`, run from here — `static/` and `content/` resolve relative to +the working directory, while templates are embedded in the binary: + ```bash -python3 -m venv .venv && .venv/bin/pip install -r requirements.txt +cd server && go build -o thermograph-frontend . +cd .. THERMOGRAPH_API_BASE_INTERNAL=http://127.0.0.1:8137 \ THERMOGRAPH_BASE=/thermograph \ - .venv/bin/uvicorn app:app --host 0.0.0.0 --port 8080 + ./server/thermograph-frontend ``` Or build the image directly: @@ -115,12 +100,37 @@ docker run -p 8080:8080 -e THERMOGRAPH_API_BASE_INTERNAL=http://backend:8137 the The image healthchecks `GET /healthz` (liveness only — it does no I/O, so it stays cheap; it does not prove the backend is reachable). +## Test + +```bash +cd server && go build ./... && go vet ./... && go test ./... +``` + +`Dockerfile`'s builder stage runs `gofmt -l` + `go vet` + `go test ./...` before +compiling, so **a failing Go test fails the image build** — and that is the +whole frontend check in CI. There is no separate frontend test step in any +workflow. + +The Python tiers are local-only: + +```bash +make test-unit # hermetic: SSR rendering fed committed fixtures +make test-integration # pulls + runs a real backend image +make backend-up # just the backend container, for dev +make capture-fixtures # refresh tests/fixtures/*.json from a live backend +``` + +`make backend-up` derives the backend image tag from this checkout (the last +commit touching `backend/` — the same domain-keyed rule build-push and deploy +use); pin one with `THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex>`. + ## More detail -- [`CLAUDE.md`](CLAUDE.md) — agent-facing instructions: deploy contract, - API-version pinning, the cross-repo `/cell`/ETag and `pctOrd()` contracts, - and the current known test-suite gap. +- [`CLAUDE.md`](CLAUDE.md) — agent-facing instructions: build/verify, layout, + API-version pinning, and the `/cell`/ETag, `pctOrd()` and Fahrenheit-set + contracts. +- [`server/README.md`](server/README.md) — the Go port's own notes. - [`DESIGN.md`](DESIGN.md) — the visual system (tokens, components, - breakpoints) and `make shots` visual verification. -- [`thermograph-docs`](../thermograph-docs) — the repo-topology decision - record this split implements. + breakpoints) and `tools/shoot.py` visual verification. +- [`docs/onboarding/`](../docs/onboarding/README.md) — the developer onboarding + path for the whole monorepo. diff --git a/frontend/docker-compose.test.yml b/frontend/docker-compose.test.yml index 83b5d33..234fa2d 100644 --- a/frontend/docker-compose.test.yml +++ b/frontend/docker-compose.test.yml @@ -20,9 +20,16 @@ services: retries: 20 backend: - # The published backend image is pulled (no local build). Default a published tag; - # override THERMOGRAPH_BACKEND_TEST_TAG to pin a specific backend build (e.g. a sha). - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${THERMOGRAPH_BACKEND_TEST_TAG:-v0.0.2-split-ci} + # The published backend image is pulled (no local build). + # + # No default tag on purpose: scripts/backend-for-tests.sh computes one from + # THIS checkout -- sha-<12hex of `git log -1 -- backend/`>, the same + # domain-keyed rule build-push.yml and deploy.yml both use -- so the harness + # tracks the tree instead of drifting behind a hardcoded pin. (It was pinned + # to the split-era v0.0.2-split-ci, under the retired + # emi/thermograph-backend/app path, long after both were superseded.) + # Override THERMOGRAPH_BACKEND_TEST_TAG to pin a specific build. + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${THERMOGRAPH_BACKEND_TEST_TAG:?set it, or run via scripts/backend-for-tests.sh which derives it} depends_on: db: condition: service_healthy diff --git a/frontend/scripts/backend-for-tests.sh b/frontend/scripts/backend-for-tests.sh index 0770a96..1f48ab4 100755 --- a/frontend/scripts/backend-for-tests.sh +++ b/frontend/scripts/backend-for-tests.sh @@ -7,8 +7,9 @@ # scripts/backend-for-tests.sh down # stop + remove (incl. the throwaway db volume) # scripts/backend-for-tests.sh url # print the base URL (no lifecycle change) # -# Which backend build: THERMOGRAPH_BACKEND_TEST_TAG (default a published tag; set a -# sha-<12hex> to pin one). Host port: BACKEND_TEST_HOST_PORT (default 18137). +# Which backend build: THERMOGRAPH_BACKEND_TEST_TAG. Unset, it is DERIVED from this +# checkout (see below) rather than defaulted to a hardcoded tag -- set a sha-<12hex> +# to pin one. Host port: BACKEND_TEST_HOST_PORT (default 18137). set -euo pipefail cd "$(dirname "$0")/.." @@ -17,10 +18,39 @@ export BACKEND_TEST_HOST_PORT="${BACKEND_TEST_HOST_PORT:-18137}" BASE_URL="http://127.0.0.1:${BACKEND_TEST_HOST_PORT}" COMPOSE=(docker compose -f docker-compose.test.yml) +# Derive the backend image tag the SAME way build-push.yml and deploy.yml do: +# keyed to the last commit that touched backend/, not the branch tip. That is the +# image actually published for this checkout's backend tree, so the harness follows +# the tree instead of drifting behind a pin (this file previously defaulted to the +# split-era v0.0.2-split-ci tag under a retired image path). +# +# Exported before ANY compose call -- docker-compose.test.yml requires the variable +# (`:?`), so `down` and `url` need it set too, not just `up`. +# +# `:/backend/` is a repo-ROOT-relative pathspec: a bare `backend/` would resolve +# against this script's cwd (frontend/) and silently match nothing, yielding an +# empty sha and a confusing error instead of the right tag. +if [ -z "${THERMOGRAPH_BACKEND_TEST_TAG:-}" ]; then + domain_sha="$(git log -1 --format=%H -- ':/backend/' 2>/dev/null || true)" + if [ -z "$domain_sha" ]; then + echo "!! could not derive a backend image tag from git (no backend/ history here?)." >&2 + echo " Set THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex> explicitly." >&2 + exit 2 + fi + THERMOGRAPH_BACKEND_TEST_TAG="sha-${domain_sha:0:12}" +fi +export THERMOGRAPH_BACKEND_TEST_TAG + case "${1:-up}" in up) - echo "==> pulling backend image (${THERMOGRAPH_BACKEND_TEST_TAG:-v0.0.2-split-ci})" >&2 - "${COMPOSE[@]}" pull backend >&2 + echo "==> pulling backend image (${THERMOGRAPH_BACKEND_TEST_TAG})" >&2 + if ! "${COMPOSE[@]}" pull backend >&2; then + echo "!! no published backend image for ${THERMOGRAPH_BACKEND_TEST_TAG}." >&2 + echo " That tag is keyed to the last commit touching backend/ -- if those" >&2 + echo " commits are local-only, build-push.yml has not published it yet." >&2 + echo " Pin a published build: THERMOGRAPH_BACKEND_TEST_TAG=sha-<12hex> $0 up" >&2 + exit 1 + fi echo "==> starting backend + db" >&2 "${COMPOSE[@]}" up -d >&2 echo "==> waiting for backend /healthz (up to 90s)" >&2 diff --git a/frontend/server/internal/format/format_test.go b/frontend/server/internal/format/format_test.go index defa4b5..699f999 100644 --- a/frontend/server/internal/format/format_test.go +++ b/frontend/server/internal/format/format_test.go @@ -45,42 +45,93 @@ func TestUnitForCountry(t *testing.T) { } } -// TestFCountriesMatchesBackend re-parses the backend's F_COUNTRIES (the -// source of truth per both CLAUDE.md files) and asserts our set is identical. -// Skips when the monorepo backend checkout isn't present. -func TestFCountriesMatchesBackend(t *testing.T) { - path := filepath.Join("..", "..", "..", "..", "backend", "api", "content_payloads.py") +// parseCountryCodes pulls a set of ISO-3166 alpha-2 codes out of `path` by +// matching `outer` (which must capture the literal's body in group 1) and then +// scanning that body for quoted codes. Returns nil when the file isn't present, +// so a caller can skip rather than fail in a checkout/build context that does +// not carry it. +func parseCountryCodes(t *testing.T, path string, outer *regexp.Regexp) map[string]bool { + t.Helper() raw, err := os.ReadFile(path) if err != nil { - t.Skipf("backend checkout not present (%v) — parity asserted only against the committed copy", err) + return nil } - m := regexp.MustCompile(`(?s)F_COUNTRIES\s*=\s*frozenset\(\{(.*?)\}\)`).FindSubmatch(raw) + m := outer.FindSubmatch(raw) if m == nil { - t.Fatalf("could not find F_COUNTRIES in %s", path) + t.Fatalf("could not find the country-set literal in %s", path) } codes := regexp.MustCompile(`"([A-Z]{2})"`).FindAllSubmatch(m[1], -1) if len(codes) == 0 { t.Fatalf("no codes parsed from %s", path) } - backend := map[string]bool{} + set := map[string]bool{} for _, c := range codes { - backend[string(c[1])] = true + set[string(c[1])] = true } - if len(backend) != len(FCountries) { - t.Errorf("F_COUNTRIES size mismatch: backend %d vs ours %d", len(backend), len(FCountries)) + return set +} + +// assertMatchesFCountries diffs a parsed copy against ours, both directions. +func assertMatchesFCountries(t *testing.T, name string, other map[string]bool) { + t.Helper() + if len(other) != len(FCountries) { + t.Errorf("size mismatch: %s %d vs ours %d", name, len(other), len(FCountries)) } - for code := range backend { + for code := range other { if !FCountries[code] { - t.Errorf("backend has %q, ours does not", code) + t.Errorf("%s has %q, ours does not", name, code) } } for code := range FCountries { - if !backend[code] { - t.Errorf("ours has %q, backend does not", code) + if !other[code] { + t.Errorf("ours has %q, %s does not", code, name) } } } +// TestFCountriesMatchesBackend re-parses the backend's F_COUNTRIES (the +// source of truth per both CLAUDE.md files) and asserts our set is identical. +// Skips when the monorepo backend checkout isn't present — notably inside +// frontend/Dockerfile's builder stage, whose build context is frontend/ only, +// so backend/ is structurally unreachable there. This check is therefore a +// local/CI-checkout guard, not an image-build one. +func TestFCountriesMatchesBackend(t *testing.T) { + path := filepath.Join("..", "..", "..", "..", "backend", "api", "content_payloads.py") + backend := parseCountryCodes(t, path, regexp.MustCompile(`(?s)F_COUNTRIES\s*=\s*frozenset\(\{(.*?)\}\)`)) + if backend == nil { + t.Skip("backend checkout not present — parity asserted only against the committed copy") + } + assertMatchesFCountries(t, "backend", backend) +} + +// TestFCountriesMatchesUnitsJS asserts the BROWSER copy — static/units.js's +// F_REGIONS — matches too. +// +// This copy was previously guarded by nothing at all, despite comments in +// backend/api/content_payloads.py, frontend/format.py and format.go each +// claiming "a test asserts all three stay identical": the only check compared +// Go against the backend's Python, and nothing read units.js. The sets happened +// to agree, held in step by convention alone. Client-side unit selection +// disagreeing with server-rendered unit selection means the same page shows °C +// in SSR and °F after hydration — a silent, per-country split. +// +// Unlike the backend check above, static/ IS inside the frontend build context, +// and frontend/Dockerfile copies units.js into the builder stage precisely so +// this runs during the image build — the only place CI executes these tests. +func TestFCountriesMatchesUnitsJS(t *testing.T) { + for _, path := range []string{ + filepath.Join("..", "..", "..", "static", "units.js"), // repo checkout + filepath.Join("/", "static", "units.js"), // Dockerfile builder stage + } { + js := parseCountryCodes(t, path, regexp.MustCompile(`(?s)F_REGIONS\s*=\s*new Set\(\[(.*?)\]\)`)) + if js != nil { + assertMatchesFCountries(t, "static/units.js", js) + return + } + } + t.Skip("static/units.js not reachable from this working directory") +} + func TestTempSpans(t *testing.T) { // Exact rendered bytes — the golden-diff contract. if got := Temp("F", fp(71.1)); got != `71°F` { -- 2.45.2 From 5b7e30dad9c6b9218e8b6007478deee4300fb188 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 06:20:19 +0000 Subject: [PATCH 05/37] fix(frontend): Weekly view must always send an explicit date to /grade --- frontend/static/app.js | 25 +++++++-- frontend/static/cache.js | 21 +++++-- frontend/tests/unit/test_app_js_grade_date.py | 55 +++++++++++++++++++ 3 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 frontend/tests/unit/test_app_js_grade_date.py diff --git a/frontend/static/app.js b/frontend/static/app.js index 4b21646..c8a97b8 100644 --- a/frontend/static/app.js +++ b/frontend/static/app.js @@ -152,10 +152,27 @@ async function runGrade() { updateHash(); placeholder.hidden = true; results.hidden = false; - // Omit the date when it's today, so the URL (and cache key) matches the prefetch - // fired from the other views — a same-tab navigation then reuses the response. - const q = `lat=${selected.lat}&lon=${selected.lon}`; - const url = dateInput.value === todayISO() ? uv(`grade?${q}`) : uv(`grade?${q}&date=${dateInput.value}`); + // Always send the target date, even when it's "today" — the backend only + // falls back to its own UTC today (api_grade's `date` param default; see + // backend/web/app.py) when the param is omitted or empty, a day behind local + // midnight for any viewer east of UTC (reported from Kaunas, UTC+3: asking + // for 7/26 rendered 7/25). The comment this replaces claimed omitting the + // date kept this URL matching the cross-view prefetch — but that match was + // the other half of the bug: cache.js's viewFetches() seeds the grade view's + // cache entry under this exact dateless URL too, tagged fresh for the + // viewer's local day no matter which day the server actually resolved, so a + // same-tab nav that had already prefetched this cell (e.g. via the Day page) + // could serve that stale, server-resolved slice straight out of IndexedDB + // with no live request ever going out. Sending the real date makes every + // load self-consistent with the viewer's own clock instead of aliasing onto + // whatever "today" the server or an earlier prefetch resolved. + // dateInput.value can also come back empty (a cleared native date field — + // day.js guards the same input for the same reason), so fall back to the + // viewer's local today rather than ever send `date=` empty, which hits that + // same server fallback. + const date = dateInput.value || todayISO(); + const q = `lat=${selected.lat}&lon=${selected.lon}&date=${date}`; + const url = uv(`grade?${q}`); await loadView({ url, ttl: TTL.grade, seq: ++gradeSeq, current: () => gradeSeq, spinner: () => { results.innerHTML = ``; }, diff --git a/frontend/static/cache.js b/frontend/static/cache.js index 33fe40c..ee04bf5 100644 --- a/frontend/static/cache.js +++ b/frontend/static/cache.js @@ -207,7 +207,7 @@ function seedCache(url, data, etag) { // lives here next to the bundle fetch — not knowledge of any one page. function viewFetches(q, today) { return { - grade: [uv(`grade?${q}`), TTL.grade], + grade: [uv(`grade?${q}&date=${today}`), TTL.grade], forecast: [uv(`forecast?${q}`), TTL.forecast], calendar: [uv(`calendar?${q}&months=24`), TTL.calendar], day: [uv(`day?${q}&date=${today}`), TTL.day], @@ -232,6 +232,12 @@ export function prefetchViews(lat, lon, ownViews) { if (_prefetched === tag) return; _prefetched = tag; const q = `lat=${lat}&lon=${lon}`; + // The client's local "today" — sent explicitly so the server never has to guess + // it from its own (UTC) clock. Getting this wrong is exactly what poisoned the + // Weekly view's cache for UTC+ visitors between their local midnight and UTC + // midnight: the bundle used to be built against the server's date while the + // per-view request stated the client's, so the two silently disagreed. + const today = localDay(); const own = new Set(ownViews); // Yield first so the landing view finishes rendering before we fetch the rest. setTimeout(async () => { @@ -239,17 +245,20 @@ export function prefetchViews(lat, lon, ownViews) { const ek = "tg-cell-etag:" + q; let prev = null; try { prev = sessionStorage.getItem(ek); } catch (e) {} - const res = await fetch(uv(`cell?${q}&neighbors=1`), + const res = await fetch(uv(`cell?${q}&date=${today}&neighbors=1`), { credentials: "include", ...(prev ? { headers: { "If-None-Match": prev } } : {}) }); if (res.ok && res.status !== 304) { const bundle = await res.json(); try { sessionStorage.setItem(ek, res.headers.get("ETag") || ""); } catch (e) {} - const fetches = viewFetches(q, localDay()); + // Seed under bundle.today (the date the server actually resolved — normally + // just an echo of the date= we sent) rather than our own `today`, so the + // seeded key stays byte-identical to what a per-view request for that same + // resolved date will use even in the rare case the two disagree (e.g. a + // request that straddles local midnight). + const fetches = viewFetches(q, bundle.today); for (const [name, slice] of Object.entries(bundle.slices || {})) { if (own.has(name) || !slice || !slice.data) continue; - const url = name === "day" - ? uv(`day?${q}&date=${bundle.today}`) // the day slice targets today - : (fetches[name] || [])[0]; + const url = (fetches[name] || [])[0]; if (url) seedCache(url, slice.data, slice.etag); } } diff --git a/frontend/tests/unit/test_app_js_grade_date.py b/frontend/tests/unit/test_app_js_grade_date.py new file mode 100644 index 0000000..e27fd6c --- /dev/null +++ b/frontend/tests/unit/test_app_js_grade_date.py @@ -0,0 +1,55 @@ +"""Regression guard for the grade-URL "today" date bug: runGrade() in app.js used +to omit `date` from the /api/v2/grade request whenever the selected date equaled +the viewer's local today, so the backend fell back to ITS OWN UTC today +(api_grade's `date` param default in backend/web/app.py) -- a day behind local +midnight for any viewer east of UTC. Reported from Kaunas (UTC+3): asking for +the Weekly tab on 7/26 rendered 7/25. + +There is no JS/DOM test harness in this repo to drive app.js and assert on the +real fetch() URL it builds -- static/package.json declares zero dependencies, so +there is no jest/playwright/node runner wired into CI, and nothing else covers +static/*.js runtime behavior. So this can't be a behavioral test. What it CAN +do, hermetically, through the same served-asset route test_pages.py already +exercises (`GET {B}/app.js`), is treat the real shipped source as data and +assert the structural property the fix guarantees: runGrade() builds exactly one +grade URL and always includes an explicit `date=`, rather than branching on +whether the selected date is "today" and omitting it in that case. + +This fails against the pre-fix source and passes against the fixed source -- +a source guard, not a substitute for a real behavioral test. +""" +import re + +B = "/thermograph" # matches THERMOGRAPH_BASE set in tests/conftest.py + + +def _run_grade_body(client) -> str: + r = client.get(f"{B}/app.js") + assert r.status_code == 200 + src = r.text + m = re.search(r"async function runGrade\(\) \{\n(.*?)\n\}\n", src, re.DOTALL) + assert m, "runGrade() not found in app.js -- update this test if it moved/was renamed" + return m.group(1) + + +def test_run_grade_always_sends_explicit_date(client): + """The historical bug: `dateInput.value === todayISO() ? grade?q : grade?q&date=...` + omitted `date` for "today", so the backend's own UTC-today fallback rendered the + previous day for any UTC+ viewer between their local midnight and UTC midnight.""" + body = _run_grade_body(client) + assert not re.search(r"dateInput\.value\s*===\s*todayISO\(\)", body), \ + "runGrade() still branches on today to decide whether to send date=" + assert body.count("grade?") == 1, \ + "runGrade() should build exactly one grade URL, not two divergent branches" + assert re.search(r"date=\$\{[^}]+\}", body), \ + "runGrade() must always build the URL with an explicit date= param" + + +def test_run_grade_never_sends_an_empty_date(client): + """A cleared native date field reports dateInput.value === "" (day.js guards the + same input for the same reason). An empty date= hits the identical server-UTC + fallback this fix exists to avoid, so runGrade() must fall back to the viewer's + own local today rather than ever send the param empty.""" + body = _run_grade_body(client) + assert re.search(r"dateInput\.value\s*\|\|\s*todayISO\(\)", body), \ + "runGrade() should fall back to todayISO() when dateInput.value is empty" -- 2.45.2 From d138f00a2077a5e9c7e0fab9d74b437fd43e6101 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 06:56:38 +0000 Subject: [PATCH 06/37] =?UTF-8?q?infra:=20split=20the=20estate=20into=20vp?= =?UTF-8?q?s1/vps2=20=E2=80=94=20beta=20joins=20prod,=20dev=20gets=20a=20h?= =?UTF-8?q?ome=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/hooks/README.md | 16 +- .claude/hooks/prod-guard.sh | 17 +- .forgejo/workflows/deploy.yml | 100 +++-- .forgejo/workflows/infra-sync.yml | 97 +++- .forgejo/workflows/observability-validate.yml | 3 +- .forgejo/workflows/ops-cron.yml | 147 +++--- CLAUDE.md | 60 ++- CUTOVER-NOTES.md | 35 ++ README.md | 7 +- docs/onboarding/01-orientation.md | 51 ++- docs/onboarding/02-setup.md | 13 +- docs/onboarding/03-repo-map.md | 46 +- docs/onboarding/04-backend.md | 7 +- docs/onboarding/05-frontend.md | 2 +- docs/onboarding/07-ci-and-release.md | 85 ++-- docs/onboarding/08-infra-secrets.md | 209 ++++++--- docs/onboarding/09-observability.md | 71 +-- docs/onboarding/10-recipes.md | 19 +- docs/onboarding/11-traps.md | 61 ++- docs/onboarding/README.md | 11 +- infra/.claude/skills/key-gaps/SKILL.md | 25 +- infra/.env.example | 17 +- infra/ACCESS.md | 158 ++++--- infra/CLAUDE.md | 96 +++- infra/DEPLOY-DEV.md | 233 ++++++---- infra/DEPLOY.md | 188 ++++---- infra/Makefile | 10 +- infra/README.md | 87 ++-- infra/deploy/Caddyfile | 120 ----- infra/deploy/Caddyfile.vps1 | 61 +++ infra/deploy/Caddyfile.vps2 | 145 ++++++ infra/deploy/RUNBOOK-vps1-vps2-cutover.md | 420 ++++++++++++++++++ infra/deploy/backup/README.md | 41 +- infra/deploy/db/init/20-tuning.sh | 7 +- infra/deploy/db/provision-env-db.sh | 161 +++++++ infra/deploy/deploy-dev.sh | 99 +++-- infra/deploy/deploy.sh | 96 +++- infra/deploy/env-topology.sh | 237 ++++++++++ infra/deploy/forgejo/README.md | 96 ++-- infra/deploy/forgejo/caddy-git.conf | 11 +- infra/deploy/forgejo/docker-stack.yml | 31 +- infra/deploy/forgejo/register-lan-runner.sh | 27 +- infra/deploy/openmeteo/README.md | 10 +- infra/deploy/provision-dev-lan.sh | 30 -- infra/deploy/provision-dev.sh | 79 ++++ infra/deploy/provision-mail.sh | 62 ++- infra/deploy/provision-secrets.sh | 46 +- infra/deploy/render-secrets.sh | 27 +- infra/deploy/secrets/README.md | 142 ++++-- infra/deploy/secrets/beta.yaml | 30 +- infra/deploy/secrets/dev.yaml | 38 +- infra/deploy/secrets/seed-encrypt-on-host.sh | 22 +- infra/deploy/secrets/seed-from-live.sh | 22 +- infra/deploy/stack/deploy-stack.sh | 146 ++++-- infra/deploy/stack/lb/Caddyfile.beta | 34 ++ infra/deploy/stack/thermograph-beta-stack.yml | 278 ++++++++++++ infra/deploy/swarm/README.md | 71 +-- infra/deploy/swarm/firewall-swarm.sh | 14 +- infra/deploy/swarm/init-swarm.sh | 11 +- infra/deploy/swarm/join-swarm.sh | 4 +- infra/deploy/swarm/label-forge-node.sh | 4 +- infra/deploy/swarm/setup-wireguard.sh | 6 +- infra/deploy/thermograph.env.example | 18 +- infra/docker-compose.dev.yml | 41 +- infra/ops/ICEBERG-HANDOFF.md | 32 ++ infra/ops/README.md | 61 ++- infra/ops/dbq.sh | 69 ++- infra/ops/iceberg.sh | 23 +- infra/terraform/README.md | 75 +++- infra/terraform/main.tf | 36 +- .../modules/thermograph-host/variables.tf | 2 +- infra/terraform/outputs.tf | 2 +- infra/terraform/terraform.tfvars.example | 168 ++++--- infra/terraform/variables.tf | 146 +++--- observability/.env.example | 11 +- observability/CLAUDE.md | 35 +- observability/README.md | 150 +++++-- observability/alloy/config.alloy | 159 ++++++- .../alloy/docker-compose.agent.beta.yml | 24 + observability/alloy/docker-compose.agent.yml | 53 ++- observability/caddy-grafana.conf | 13 +- observability/docker-compose.yml | 21 +- .../provisioning/alerting/contact-points.yml | 9 +- .../provisioning/alerting/rules-prod.yml | 42 +- observability/loki/config.yml | 11 +- 85 files changed, 4218 insertions(+), 1482 deletions(-) delete mode 100644 infra/deploy/Caddyfile create mode 100644 infra/deploy/Caddyfile.vps1 create mode 100644 infra/deploy/Caddyfile.vps2 create mode 100644 infra/deploy/RUNBOOK-vps1-vps2-cutover.md create mode 100755 infra/deploy/db/provision-env-db.sh create mode 100644 infra/deploy/env-topology.sh delete mode 100755 infra/deploy/provision-dev-lan.sh create mode 100755 infra/deploy/provision-dev.sh create mode 100644 infra/deploy/stack/lb/Caddyfile.beta create mode 100644 infra/deploy/stack/thermograph-beta-stack.yml create mode 100644 observability/alloy/docker-compose.agent.beta.yml diff --git a/.claude/hooks/README.md b/.claude/hooks/README.md index 0075e86..afec666 100644 --- a/.claude/hooks/README.md +++ b/.claude/hooks/README.md @@ -13,7 +13,7 @@ They are wired up in `.claude/settings.json`. ## Why prod-guard exists -`agent` has passwordless sudo on prod and beta, and the operator's global settings +`agent` has passwordless sudo on vps1 and vps2, and the operator's global settings allow `Bash(ssh prod:*)` outright. Nothing about `ssh prod 'docker service rm …'` goes through a pull request, so CI cannot see it — before this hook, any session in any directory could delete production with no prompt. @@ -24,9 +24,17 @@ is wrong by construction — the first destructive verb nobody thought of sails straight through. Being wrong in the "ask" direction costs a keystroke; being wrong the other way costs production. -Beta is guarded as strictly as prod: it serves beta.thermograph.org *and* hosts -Forgejo, so a destructive command there takes out git, CI and the registry at once. -The LAN dev box is deliberately unguarded. +Beta is guarded as strictly as prod — more strictly than the names alone suggest, +now: vps2 (`169.58.46.181`) runs **both** prod and beta as separate Swarm stacks on +the same manager, sharing one TimescaleDB instance, so a "beta" command there has +prod's blast radius (see `infra/deploy/env-topology.sh`). vps1 (`75.119.132.91`) is +the other guarded host: Forgejo (git + CI + registry) and Grafana/Loki live there, +plus the `dev` environment, so a destructive command there takes out git, CI and the +registry at once. Mesh IPs did not move in the vps1/vps2 split (vps1 is still +`10.10.0.2`, vps2 is still `10.10.0.1`); what moved is which environment runs where +— `HOST_TOKEN_RE` in `prod-guard.sh` recognises `vps1`/`vps2` alongside the +environment names (`prod`/`beta`) and the raw IPs, so a command naming the host +either way still gets caught. The LAN dev box is deliberately unguarded. ## Changing the classifier diff --git a/.claude/hooks/prod-guard.sh b/.claude/hooks/prod-guard.sh index 5d81dda..319eaf3 100755 --- a/.claude/hooks/prod-guard.sh +++ b/.claude/hooks/prod-guard.sh @@ -19,9 +19,15 @@ # That is also what happens if this script errors or jq is missing, so a bug here # fails toward the prompt rather than toward silent execution. # -# Beta is guarded as strictly as prod: it serves beta.thermograph.org AND hosts -# Forgejo, so a destructive command there can take out the git server, CI and the -# container registry at once. +# Beta is guarded as strictly as prod -- more strictly than the names alone +# suggest, now: vps2 (169.58.46.181) runs prod AND beta as two Swarm stacks on +# the SAME manager, sharing the same TimescaleDB instance, so a "beta" command +# there has PROD's blast radius (shared Docker host, shared Postgres server -- +# see infra/deploy/env-topology.sh). vps1 (75.119.132.91) is the other guarded +# host: Forgejo (git + CI + registry), Grafana/Loki, and the dev environment -- +# a destructive command there can take out the git server, CI and the container +# registry at once. Mesh IPs did NOT move in the vps1/vps2 split (vps1 is still +# 10.10.0.2, vps2 is still 10.10.0.1); what moved is which environment runs where. set -uo pipefail payload=$(cat) @@ -32,7 +38,10 @@ tool=$(printf '%s' "$payload" | jq -r '.tool_name // ""') # An ssh target naming a live host, matched as a whole token after any `user@`. # Backslashes are doubled because awk -v processes escape sequences in the value # before the regex ever sees it; a single \. arrives as a bare dot (and warns). -readonly HOST_TOKEN_RE='^(prod|beta|169\\.58\\.46\\.181|75\\.119\\.132\\.91|10\\.10\\.0\\.[12])$' +# vps2 (169.58.46.181) carries BOTH prod and beta; vps1 (75.119.132.91) carries +# Forgejo/Grafana/dev -- "vps1"/"vps2" are recognised alongside the environment +# names and raw IPs so a command naming the host either way still gets caught. +readonly HOST_TOKEN_RE='^(prod|beta|vps1|vps2|169\\.58\\.46\\.181|75\\.119\\.132\\.91|10\\.10\\.0\\.[12])$' emit() { # $1=allow|ask|deny $2=reason jq -n --arg d "$1" --arg r "$2" \ diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index ceae65e..f28931f 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -10,16 +10,32 @@ name: Deploy # parameterised, so the duplication bought nothing and cost six files to keep in # step. # -# The two *-deploy-dev.yml workflows are NOT ported. They were documented as -# inert: they call the monorepo layout at ~/thermograph-dev on the LAN box, -# which is still a split-era thermograph-infra checkout, so that path does not -# exist there. LAN dev is a local `make dev-up` concern, not a CI environment. +# DEV IS A REAL DEPLOY TARGET AGAIN. It previously was not: dev meant a +# sudo-free compose stack on the operator's desktop, and the two +# *-deploy-dev.yml workflows that drove it were inert. Dev now lives on vps1 at +# /opt/thermograph-dev — a normal fleet host, reached over SSH exactly like beta +# and prod — so it gets a leg here. +# +# SECRETS ARE KEYED BY HOST, NOT BY ENVIRONMENT. `SSH_*` used to mean "beta" and +# `PROD_SSH_*` "prod", which worked only while each environment owned a box. It +# stopped being true in two directions at once: vps2 now runs beta AND prod, and +# the box `SSH_*` pointed at is now vps1, which runs dev, Forgejo and Grafana. +# The old names would have made "the beta secret" and "the Forgejo box secret" +# the same value by accident — the ops-cron file already records one incident +# caused by exactly that conflation. So: VPS1_SSH_* and VPS2_SSH_*, named for +# the machine, and the environment is passed separately as THERMOGRAPH_ENV. +# +# BETA AND PROD DEPLOYING TO ONE BOX DO NOT RACE. They have separate checkouts +# (/opt/thermograph-beta and /opt/thermograph), so the `git reset --hard` in one +# cannot pull the tree out from under the other, and deploy.sh's flock is per +# checkout. The concurrency groups below stay keyed by ref+service, which keeps +# two pushes to the SAME branch serialised — the property that actually matters. # # DELIBERATELY BORING EXPRESSIONS. No dynamic matrix (fromJSON), no # `cond && secrets.A || secrets.B` ternary. Those are GitHub idioms that a # Forgejo/act runner may evaluate differently, and the failure mode here is # "production does not deploy" or, worse, "deploys with an empty SSH host". The -# two environments therefore get two explicit, mutually exclusive steps. +# three environments therefore get three explicit, mutually exclusive steps. # # What is preserved from the originals, all of it load-bearing: # - fetch-depth: 0, because the image tag is keyed to the LAST COMMIT THAT @@ -29,14 +45,18 @@ name: Deploy # - The 12-hex truncation, matching build-push exactly. # - Per-service, per-environment concurrency with cancel-in-progress: false -- # a half-finished deploy must never be cancelled by a newer one. -# - Separate PROD_SSH_* credentials, so a beta credential leak cannot reach -# prod. +# - Separate credentials per host. Note what this does and does not buy now: +# it keeps vps1 (Forgejo, its CI, and whatever unreviewed branch dev is +# running) away from vps2 entirely. It no longer puts a host boundary +# between beta and prod, because they share vps2 by design — that boundary +# is now at the database (separate roles and databases) and the filesystem +# (separate checkouts and rendered env files). # - appleboy/ssh-action by full URL; it is not mirrored in Forgejo's default # action registry. on: push: - branches: [main, release] + branches: [dev, main, release] paths: - 'backend/**' - 'frontend/**' @@ -71,10 +91,15 @@ jobs: run: | set -euo pipefail - # Branch selects the environment. main -> beta, release -> prod. + # Branch selects the environment; the environment selects the host and + # the script path. dev -> dev (vps1), main -> beta (vps2), + # release -> prod (vps2). The paths differ because each environment + # has its own checkout — two environments on vps2 must never share + # one — and dev has its own entry point for its secrets policy. case "${{ github.ref_name }}" in - main) environment=beta ;; - release) environment=prod ;; + dev) environment=dev ; script=/opt/thermograph-dev/infra/deploy/deploy-dev.sh ;; + main) environment=beta ; script=/opt/thermograph-beta/infra/deploy/deploy.sh ;; + release) environment=prod ; script=/opt/thermograph/infra/deploy/deploy.sh ;; *) echo "::error::Deploy triggered on unexpected ref '${{ github.ref_name }}'"; exit 1 ;; esac @@ -105,6 +130,7 @@ jobs: { echo "environment=$environment" + echo "script=$script" echo "changed=$changed" echo "tag=$tag" } >> "$GITHUB_OUTPUT" @@ -126,38 +152,58 @@ jobs: echo "==> ${{ matrix.service }} -> $environment | changed=$changed | tag=$tag" - - name: Deploy to beta + - name: Deploy to dev (vps1) + if: steps.plan.outputs.changed == 'true' && steps.plan.outputs.environment == 'dev' + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} + envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG,THERMOGRAPH_ENV + script: ${{ steps.plan.outputs.script }} + env: + SERVICE: ${{ matrix.service }} + THERMOGRAPH_ENV: dev + BACKEND_IMAGE_TAG: ${{ matrix.service == 'backend' && steps.plan.outputs.tag || '' }} + FRONTEND_IMAGE_TAG: ${{ matrix.service == 'frontend' && steps.plan.outputs.tag || '' }} + + - name: Deploy to beta (vps2) if: steps.plan.outputs.changed == 'true' && steps.plan.outputs.environment == 'beta' uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} - envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG - script: /opt/thermograph/infra/deploy/deploy.sh + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG,THERMOGRAPH_ENV + script: ${{ steps.plan.outputs.script }} env: SERVICE: ${{ matrix.service }} + # THERMOGRAPH_ENV is what makes this a BETA deploy rather than a prod + # one: same host, same credentials, same script — the environment is + # the only thing that differs, and deploy.sh refuses to run if it + # disagrees with the checkout it was invoked from. + THERMOGRAPH_ENV: beta # Only the matching one is read by deploy.sh for a single-service roll; # the other stays empty and the persisted .image-tags.env supplies the # sibling's live tag, so this roll never disturbs it. BACKEND_IMAGE_TAG: ${{ matrix.service == 'backend' && steps.plan.outputs.tag || '' }} FRONTEND_IMAGE_TAG: ${{ matrix.service == 'frontend' && steps.plan.outputs.tag || '' }} - - name: Deploy to prod + - name: Deploy to prod (vps2) if: steps.plan.outputs.changed == 'true' && steps.plan.outputs.environment == 'prod' uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - # A completely separate secret set from beta's, deliberately: a beta - # credential leak must not reach prod. - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} - envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG - script: /opt/thermograph/infra/deploy/deploy.sh + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + envs: SERVICE,BACKEND_IMAGE_TAG,FRONTEND_IMAGE_TAG,THERMOGRAPH_ENV + script: ${{ steps.plan.outputs.script }} env: SERVICE: ${{ matrix.service }} + THERMOGRAPH_ENV: prod BACKEND_IMAGE_TAG: ${{ matrix.service == 'backend' && steps.plan.outputs.tag || '' }} FRONTEND_IMAGE_TAG: ${{ matrix.service == 'frontend' && steps.plan.outputs.tag || '' }} diff --git a/.forgejo/workflows/infra-sync.yml b/.forgejo/workflows/infra-sync.yml index 5c6acfa..8f86697 100644 --- a/.forgejo/workflows/infra-sync.yml +++ b/.forgejo/workflows/infra-sync.yml @@ -2,65 +2,114 @@ name: Sync infra to hosts # The infra DOMAIN's own pipeline (new with the monorepo -- the split era had # no infra deploy workflow at all; infra changes rode along lazily with the -# next app deploy's `git reset`). On a push to `main` touching infra/**, SSH -# to beta AND prod, fast-forward the host's /opt/thermograph monorepo checkout -# and re-render /etc/thermograph.env from the SOPS vault -- so a compose edit -# or a secret rotation lands push-button instead of waiting for the next app -# deploy. +# next app deploy's `git reset`). On a push touching infra/**, SSH to every +# host, fast-forward each ENVIRONMENT's monorepo checkout and re-render that +# environment's env file from the SOPS vault -- so a compose edit or a secret +# rotation lands push-button instead of waiting for the next app deploy. # # Deliberately does NOT roll any service: image tags are the app domains' -# axis, not infra's. A compose change that must recreate containers takes -# effect on the next app deploy, or a by-hand +# axis, not infra's. A compose or stack change that must recreate containers +# takes effect on the next app deploy, or a by-hand # `SERVICE=all ... infra/deploy/deploy.sh` run (see that script's header). # -# Both hosts track infra via `main` (the split-era model, unchanged): prod's -# *app images* are staged by the `release` branch, but its checkout follows -# main -- deploy.sh's own BRANCH default encodes the same thing. +# THREE CHECKOUTS, TWO HOSTS. This is the shape the vps1/vps2 split forces: +# +# vps1 /opt/thermograph-dev branch dev -> /etc/thermograph.env +# vps2 /opt/thermograph-beta branch main -> /etc/thermograph-beta.env +# vps2 /opt/thermograph branch main -> /etc/thermograph.env +# +# The two checkouts on vps2 are separate directories on purpose: a `git reset +# --hard` for beta must not be able to move prod's tree, and each environment +# renders its own env file from its own vault file. Because the paths differ, +# the beta and prod jobs below can safely run at the same time on one box. +# +# Beta and prod both track `main` (infra is not environment-staged -- only app +# IMAGES are, via tags). Dev tracks `dev`, which is why this workflow now +# triggers on both branches: a dev-branch infra change has to reach the dev +# checkout, and only that one. +# +# Each `render_thermograph_secrets` call passes the environment name and the +# output path EXPLICITLY. It used to rely on the host's +# /etc/thermograph/secrets-env marker, which cannot answer the question on vps2 +# -- one marker, two environments. Passing them here means a beta sync can +# never render prod's vault, and vice versa. on: push: - branches: [main] + branches: [dev, main] paths: ['infra/**'] workflow_dispatch: {} jobs: + sync-dev: + # Only the dev branch feeds the dev checkout. + if: github.ref_name == 'dev' + runs-on: docker + concurrency: + group: infra-sync-dev + cancel-in-progress: false + steps: + - name: Sync dev checkout on vps1 + re-render secrets + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} + script: | + set -euo pipefail + cd /opt/thermograph-dev + git fetch --prune origin dev + git reset --hard origin/dev + if [ -f infra/deploy/render-secrets.sh ]; then + . infra/deploy/render-secrets.sh + # SKIP_COMMON is dev's standing rule, not a preference: common.yaml + # is the fleet's shared production credential set, and vps1 also + # runs Forgejo and its CI. See render-secrets.sh. + THERMOGRAPH_SECRETS_SKIP_COMMON=1 \ + render_thermograph_secrets /opt/thermograph-dev/infra dev /etc/thermograph.env + fi + echo "synced to $(git log --oneline -1)" + sync-beta: + if: github.ref_name == 'main' runs-on: docker concurrency: group: infra-sync-beta cancel-in-progress: false steps: - - name: Sync beta checkout + re-render secrets + - name: Sync beta checkout on vps2 + re-render secrets uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail - cd /opt/thermograph + cd /opt/thermograph-beta git fetch --prune origin main git reset --hard origin/main if [ -f infra/deploy/render-secrets.sh ]; then . infra/deploy/render-secrets.sh - render_thermograph_secrets /opt/thermograph/infra + render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env fi echo "synced to $(git log --oneline -1)" sync-prod: + if: github.ref_name == 'main' runs-on: docker concurrency: group: infra-sync-prod cancel-in-progress: false steps: - - name: Sync prod checkout + re-render secrets + - name: Sync prod checkout on vps2 + re-render secrets uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph @@ -68,6 +117,6 @@ jobs: git reset --hard origin/main if [ -f infra/deploy/render-secrets.sh ]; then . infra/deploy/render-secrets.sh - render_thermograph_secrets /opt/thermograph/infra + render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env fi echo "synced to $(git log --oneline -1)" diff --git a/.forgejo/workflows/observability-validate.yml b/.forgejo/workflows/observability-validate.yml index 175eafb..a3d9030 100644 --- a/.forgejo/workflows/observability-validate.yml +++ b/.forgejo/workflows/observability-validate.yml @@ -1,6 +1,7 @@ name: Validate observability stack -# The observability DOMAIN deploys by hand (`docker compose up -d` on beta + +# The observability DOMAIN deploys by hand (`docker compose up -d` on vps1 — +# the monitoring host, not the beta environment, which runs on vps2 now — plus # the Alloy agent on each node) with no build step, so nothing caught a # malformed compose file, a broken dashboard JSON, or an unparseable config # until it failed live on the host. This is that missing guard: it parses diff --git a/.forgejo/workflows/ops-cron.yml b/.forgejo/workflows/ops-cron.yml index 7ee5f20..112d0be 100644 --- a/.forgejo/workflows/ops-cron.yml +++ b/.forgejo/workflows/ops-cron.yml @@ -6,19 +6,32 @@ name: Ops cron (backup + IndexNow) # classification table in # thermograph-docs/architecture/repo-topology-and-infrastructure.md §7. # -# Both jobs SSH into the prod host and run inside the already-running compose -# stack (docker compose exec), the same way deploy.sh already runs its own -# post-deploy IndexNow ping -- no new network exposure, no separate dependency -# install. Runs on the `docker` label (the always-on Swarm-hosted runner -# deploy/forgejo/ stood up). +# The jobs SSH into a host and run inside the already-running stack (docker +# exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new +# network exposure, no separate dependency install. Runs on the `docker` label +# (the always-on Swarm-hosted runner deploy/forgejo/ stood up). # -# Targets PROD via the PROD_SSH_* secrets (prod = 169.58.46.181, `agent` user, -# in the docker group so no sudo needed; /etc/thermograph.env is agent-readable). -# These are the same secrets deploy-prod.yml uses for the release->prod deploy -- -# NOT the SSH_* secrets, which point at BETA (deploy.yml's `main`->beta path). An -# earlier revision reused SSH_* here, so the "prod" backup was silently dumping -# beta; prod itself had no backup at all. The prod database is the one that must -# be backed up, so both jobs use PROD_SSH_*. +# WHICH BOX EACH JOB TALKS TO, and why the secret names say so: +# +# backup, indexnow -> VPS2_SSH_* (vps2 = 169.58.46.181: prod AND beta, the +# shared TimescaleDB, Postfix, the backups) +# forgejo-backup -> VPS1_SSH_* (vps1 = 75.119.132.91: Forgejo, Grafana, +# Loki, and the dev environment) +# +# The secrets are keyed by HOST rather than by environment because an +# environment no longer implies a machine: vps2 runs two of them. The previous +# names encoded the opposite assumption and had already caused one incident -- +# an early revision used SSH_* here, which meant "beta", so the job labelled +# "prod backup" was silently dumping beta while prod had no backup at all. The +# same conflation is why Forgejo's backup used to be described as running "on +# beta": beta and Forgejo happened to share a box, so one secret served both +# meanings. It does not any more. +# +# BOTH APPLICATION DATABASES ARE BACKED UP. prod and beta are separate databases +# (`thermograph` and `thermograph_beta`) on one shared instance. Dumping only +# the prod database would recreate the original failure in a new shape: a job +# that looks like "the backup" while one environment's data is silently +# uncovered. The loop below dumps each, to its own off-box prefix. # Monorepo port: the compose file now lives under infra/ of the host's # /opt/thermograph monorepo checkout, so every docker-compose invocation cd's @@ -44,22 +57,22 @@ jobs: group: ops-backup cancel-in-progress: false steps: - - name: Dump the prod database over SSH + - name: Dump both application databases over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 # S3 creds for the encrypted off-box copy to Contabo Object Storage, passed - # into the remote script via `envs:` (the host env file isn't a reliable - # source across boxes -- beta's /etc/thermograph.env isn't readable by the - # deploy user -- so the CI secret is the uniform home, like PROD_SSH_*). + # into the remote script via `envs:` (a host env file isn't a reliable + # source across boxes -- on vps1 it isn't readable by the deploy user -- + # so the CI secret is the uniform home, like the VPS*_SSH_* pairs). env: S3_ENDPOINT: ${{ secrets.S3_ENDPOINT }} S3_BUCKET: ${{ secrets.S3_BUCKET }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail @@ -73,23 +86,45 @@ jobs: backup_dir="$HOME/thermograph-backups" mkdir -p "$backup_dir" stamp="$(date -u +%Y%m%dT%H%M%SZ)" - out="$backup_dir/thermograph-$stamp.dump" - # The db may run under plain compose OR as a Swarm stack task - # (prod post-cutover); resolve the container either way so the - # backup survives the deploy-mode switch. + # The db may run under plain compose OR as a Swarm stack task; + # resolve the container either way so the backup survives a + # deploy-mode switch. One instance now serves both environments. dbc=$(docker ps -q --filter "label=com.docker.swarm.service.name=thermograph_db" | head -1) [ -z "$dbc" ] && dbc=$(cd /opt/thermograph/infra && docker compose ps -q db 2>/dev/null | head -1) [ -n "$dbc" ] || { echo "!! no db container found (compose or stack)"; exit 1; } - # Write to a .partial and rename on success so a mid-dump failure can - # never leave a truncated file that looks like a good backup. - docker exec "$dbc" pg_dump -U thermograph -d thermograph \ - --format=custom > "$out.partial" - mv "$out.partial" "$out" - echo "wrote $out ($(du -h "$out" | cut -f1))" + # env:database pairs. Both are dumped every night: they are separate + # databases on one instance, and backing up only one would leave the + # other silently uncovered. + for pair in prod:thermograph beta:thermograph_beta; do + envname="${pair%%:*}"; dbname="${pair##*:}" + # A missing database is a HARD failure, never a quiet skip -- a + # backup job that shrugs off an absent database is exactly how an + # environment ends up with no backups and nobody noticing. + if ! docker exec "$dbc" psql -U thermograph -d postgres -tAc \ + "select 1 from pg_database where datname='$dbname'" | grep -q 1; then + echo "!! database '$dbname' ($envname) does not exist on this instance." + if [ "$envname" = prod ]; then + echo "!! That is prod's own database on prod's own instance — this is not a" + echo "!! provisioning gap, something is badly wrong. Do not 'fix' it by creating" + echo "!! an empty database; find out where the real one went." + else + echo "!! provision it first: sudo bash /opt/thermograph-beta/infra/deploy/db/provision-env-db.sh $envname" + fi + exit 1 + fi + out="$backup_dir/$dbname-$stamp.dump" + # Write to a .partial and rename on success so a mid-dump failure can + # never leave a truncated file that looks like a good backup. + docker exec "$dbc" pg_dump -U thermograph -d "$dbname" \ + --format=custom > "$out.partial" + mv "$out.partial" "$out" + echo "wrote $out ($(du -h "$out" | cut -f1))" + dumps="${dumps:-} $envname:$out" + done # The dumps are the disaster-recovery copy, not a versioned # archive -- keep the last 14 days and let the rest age out. - find "$backup_dir" -name 'thermograph-*.dump' -mtime +14 -delete - find "$backup_dir" -name 'thermograph-*.dump.partial' -mtime +1 -delete + find "$backup_dir" -name '*.dump' -mtime +14 -delete + find "$backup_dir" -name '*.dump.partial' -mtime +1 -delete # --- off-box copy to Contabo Object Storage (age-encrypted) --- # Streams the just-written dump through age (to the vault's age # recipient -- the same key each host renders secrets with, so @@ -105,23 +140,33 @@ jobs: RCLONE_CONFIG_ARCHIVE_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ RCLONE_CONFIG_ARCHIVE_REGION=default RCLONE_CONFIG_ARCHIVE_FORCE_PATH_STYLE=true recip=age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - s3base="$S3_BUCKET/backups/db/prod" - age -r "$recip" < "$out" | rclone rcat "archive:$s3base/$(basename "$out").age" - echo "off-box: uploaded $(basename "$out").age to $s3base" - rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true + # One prefix per environment, so a restore never has to guess which + # database a dump came from: backups/db/prod/ and backups/db/beta/. + for entry in $dumps; do + envname="${entry%%:*}"; f="${entry##*:}" + s3base="$S3_BUCKET/backups/db/$envname" + age -r "$recip" < "$f" | rclone rcat "archive:$s3base/$(basename "$f").age" + echo "off-box: uploaded $(basename "$f").age to $s3base" + rclone delete --min-age 30d "archive:$s3base/" 2>/dev/null || true + done else echo "!! S3_* secrets unset -- skipped off-box push (add them as repo secrets)" fi forgejo-backup: - name: Forgejo backup (beta) -> S3 + name: Forgejo backup (vps1) -> S3 runs-on: docker - # Forgejo (the git host + all CI history) lives on beta and had NO backup at + # Forgejo (the git host + all CI history) lives on vps1 and had NO backup at # all. Dumps its Postgres db (consistent) + tars its data volume (repos, LFS, # config, avatars), age-encrypts both, pushes off-box, keeps 30 days. Runs as - # the beta SSH user (SSH_*), which is in the docker group -- so docker exec/run - # need no host sudo, and the data tar runs inside a throwaway container (the - # volume dir is root-owned on the host). rclone + age must be present on beta. + # vps1's SSH user (VPS1_SSH_*), which is in the docker group -- so docker + # exec/run need no host sudo, and the data tar runs inside a throwaway + # container (the volume dir is root-owned on the host). rclone + age must be + # present on vps1. + # + # This job follows FORGEJO, not beta. It used to use the same secret as the + # beta deploy purely because Forgejo and beta shared a box; beta has since + # moved to vps2 and Forgejo has not moved at all. concurrency: group: ops-forgejo-backup cancel-in-progress: false @@ -134,16 +179,16 @@ jobs: S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_KEY }} - port: ${{ secrets.SSH_PORT }} + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} envs: S3_ENDPOINT,S3_BUCKET,S3_ACCESS_KEY,S3_SECRET_KEY script: | set -euo pipefail [ -n "${S3_ACCESS_KEY:-}" ] || { echo "!! S3_* secrets unset"; exit 1; } - command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on beta"; exit 1; } - command -v age >/dev/null 2>&1 || { echo "!! age missing on beta"; exit 1; } + command -v rclone >/dev/null 2>&1 || { echo "!! rclone missing on vps1"; exit 1; } + command -v age >/dev/null 2>&1 || { echo "!! age missing on vps1"; exit 1; } export RCLONE_CONFIG_ARCHIVE_TYPE=s3 RCLONE_CONFIG_ARCHIVE_PROVIDER=Other \ RCLONE_CONFIG_ARCHIVE_ENDPOINT="$S3_ENDPOINT" \ RCLONE_CONFIG_ARCHIVE_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ @@ -177,10 +222,10 @@ jobs: - name: Ping IndexNow if the URL set changed uses: https://github.com/appleboy/ssh-action@v1.2.0 with: - host: ${{ secrets.PROD_SSH_HOST }} - username: ${{ secrets.PROD_SSH_USER }} - key: ${{ secrets.PROD_SSH_KEY }} - port: ${{ secrets.PROD_SSH_PORT }} + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} script: | set -euo pipefail cd /opt/thermograph/infra diff --git a/CLAUDE.md b/CLAUDE.md index 2e5710a..3b640aa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,24 +15,30 @@ every change, so a stale one is a correctness bug, not a documentation bug. | Branch | Deploys to | Workflow | |---|---|---| | feature branch | nothing | PR into `dev` | -| `dev` | nothing | integration branch only — see below | -| `main` | beta (beta.thermograph.org) | `deploy.yml` | -| `release` | prod (thermograph.org) | `deploy.yml` | +| `dev` | dev (vps1, mesh-only, own Postgres) | `deploy.yml` | +| `main` | beta (beta.thermograph.org, vps2) | `deploy.yml` | +| `release` | prod (thermograph.org, vps2) | `deploy.yml` | `dev`, `main` and `release` are protected: **everything is a PR**, for humans and agents alike. Promotion is one PR per hop, `dev` → `main` → `release`. -**`dev` deploys nowhere.** It is purely the integration branch that feature PRs -land on before promotion to `main`. The two LAN-dev deploy workflows were deleted -rather than kept: they called a monorepo path that does not exist on the LAN box -(`~/thermograph-dev` is still a split-era `thermograph-infra` checkout), so they -had been inert since cutover. Run LAN dev locally with `infra/`'s `make dev-up`. +**`dev` is a first-class hosted environment, not just an integration branch.** +It runs on vps1 — the same box as Forgejo and the monitoring stack — with its +own Postgres container, reachable only on the WireGuard mesh +(`10.10.0.2:8137`): no public DNS record, no Caddy site, no TLS. It is deployed +by CI like beta and prod. The desktop hosts no Thermograph environment at all; +`make dev-up` there is a laptop convenience for running the stack locally, not +a deployment target. -**One workflow deploys everything.** `deploy.yml` handles both services and both -environments: the branch selects the environment (`main` → beta, `release` → -prod), a matrix covers backend and frontend, and each leg checks whether this -push actually touched its domain before rolling. `build-push.yml` is the same -shape for images. They replaced six and two near-identical files respectively. +**One workflow deploys everything.** `deploy.yml` handles both services and all +three environments: the branch selects the environment (`dev` → vps1, `main` → +beta on vps2, `release` → prod on vps2), a matrix covers backend and frontend, +and each leg checks whether this push actually touched its domain before +rolling. `build-push.yml` is the same shape for images. `THERMOGRAPH_ENV` is the +input that tells a leg which environment it's deploying — load-bearing on vps2, +which runs beta and prod side by side and has no other way to tell them apart. +See `infra/deploy/env-topology.sh` for the single source of truth on where each +environment's checkout, branch, stack and ports live. ## The deploy contract @@ -44,15 +50,25 @@ SERVICE=backend|frontend|all BACKEND_IMAGE_TAG=sha-<12hex> FRONTEND_IMAGE_TAG= ``` `deploy.sh` resets the host checkout, renders secrets from the SOPS vault, then -either rolls compose services or — if `/etc/thermograph/deploy-mode` contains -`stack` — execs `infra/deploy/stack/deploy-stack.sh`. +either rolls compose services or — if `infra/deploy/env-topology.sh` says the +environment's deploy mode is `stack` — execs `infra/deploy/stack/deploy-stack.sh`. +The old host-wide `/etc/thermograph/deploy-mode` marker survives only as a +fallback for a by-hand run with no environment resolvable any other way; it +cannot describe vps2 alone, since vps2 runs beta and prod side by side. -- **prod runs Swarm.** Its stack is `infra/deploy/stack/thermograph-stack.yml` - (db, web, worker, lake, daemon, frontend, autoscaler, autoscaler-lake). - `deploy-stack.sh` also offers `STACK_TEST=1`: a full parallel rehearsal on - throwaway volumes and ports that cannot touch live data. -- **beta and LAN dev run compose**, from `infra/docker-compose.yml` - (db, backend, lake, daemon, frontend). +- **prod and beta both run Swarm, as two separate stacks co-resident on vps2.** + Prod's is `infra/deploy/stack/thermograph-stack.yml` (db, web, worker, lake, + daemon, frontend, autoscaler, autoscaler-lake). Beta's is the separate + `infra/deploy/stack/thermograph-beta-stack.yml` — its services are prefixed + (`beta-web`, `beta-worker`, …) because Swarm registers a service's short name + as a DNS alias on every network it joins, and beta shares prod's `data` + network to reach the database. Beta has no `db` service of its own: one + TimescaleDB instance serves both, on separate databases and separate + NOSUPERUSER roles. `deploy-stack.sh` also offers `STACK_TEST=1`: a full + parallel rehearsal on throwaway volumes and ports that cannot touch live data. +- **dev runs compose**, from `infra/docker-compose.yml` (db, backend, lake, + daemon, frontend), on its own host (vps1) with its own Postgres container — + the one environment not sharing a database with anything else. - Each service's live tag is persisted host-side, so a single-service roll never disturbs the sibling's running tag. @@ -76,7 +92,7 @@ reunification. - Anything named "prefetch" must never spend the Open-Meteo quota. Nominatim ≤ 1 req/s. - The compose project name is pinned (`name: thermograph` in - `infra/docker-compose.yml`); LAN dev overrides with + `infra/docker-compose.yml`); dev (and a local `make dev-up`) overrides with `COMPOSE_PROJECT_NAME=thermograph-dev`. Don't remove either half — the pinned name is what makes the Swarm stack's external volume names line up. - `CUTOVER-NOTES.md` is the source of truth for what is and isn't live yet. diff --git a/CUTOVER-NOTES.md b/CUTOVER-NOTES.md index 18743d5..cc6ba93 100644 --- a/CUTOVER-NOTES.md +++ b/CUTOVER-NOTES.md @@ -153,3 +153,38 @@ Checked every split-repo branch by dry-run merge; migrated everything real: historical. - Infra `Makefile` local-image targets still assume sibling checkouts — update when first needed locally. + +## vps1/vps2 host-topology re-architecture (landed 2026-07-25, NOT YET EXECUTED) + +**This section documents a second re-architecture, layered on top of the +monorepo cutover above. It is landed IN THE REPO — code, stack files, deploy +scripts, docs — but has NOT been run against the live estate.** The boxes +described elsewhere in this file (prod at `169.58.46.181`, beta at +`75.119.132.91`, a desktop LAN dev box) are still what is actually running +today. Do not treat anything below as live until the cutover runbook has been +executed and this note updated. + +What changes, in one line: hosts are renamed by **role** instead of by +environment, and beta moves onto the same box as prod. + +- **vps1** (mesh `10.10.0.2`, public `75.119.132.91`) — unchanged box, new job. + Keeps Forgejo, Grafana/Loki/Alloy and the `emigriffith.dev` portfolio; gains + the **dev** environment (its own Postgres, plain compose, mesh-only — + `10.10.0.2:8137`, no DNS record, no Caddy site, no TLS). This is the box that + used to be called "beta". +- **vps2** (mesh `10.10.0.1`, public `169.58.46.181`) — unchanged box, new job. + Keeps prod; gains **beta** as a second, co-resident Docker Swarm stack + (prefixed services, own checkout, own env file, own loopback LB ports). One + TimescaleDB instance now serves both prod and beta, on separate databases and + separate roles. This is the box that used to be called "prod". +- **desktop** (mesh `10.10.0.3`) — loses the LAN dev stack and the Forgejo + Actions runner entirely; becomes AI-model hosting plus flex Swarm capacity. + `make dev-up` still works there as a laptop convenience, but it is no longer + an environment. + +The mesh IPs did not move — only which environment lives on which box did. See +`infra/deploy/env-topology.sh` for the single source of truth on per-environment +checkout paths, branches, stack names, ports and DB roles, and +**[`infra/deploy/RUNBOOK-vps1-vps2-cutover.md`](infra/deploy/RUNBOOK-vps1-vps2-cutover.md)** +for the execution runbook. Until that runbook has been run, `CLAUDE.md`, +`README.md` and `docs/onboarding/` describe the target state, not today's. diff --git a/README.md b/README.md index a999ec5..b4fa580 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ for: **per-domain images, per-domain deploys, and an async FE/BE contract**. |---|---|---| | `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `build-push` → image `emi/thermograph/backend`; `deploy` | | `frontend/` | Public client: static JS/CSS + SSR pages | same `build-push` / `deploy` workflows, matrixed by domain; image `emi/thermograph/frontend` | -| `infra/` | Compose (beta, LAN dev) + the Swarm stack (prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | +| `infra/` | Compose (dev, on vps1) + two Swarm stacks co-resident on vps2 (beta, prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | | `observability/` | Loki + Grafana + Alloy stack | `observability-validate` | `thermograph-docs` deliberately **stays its own repo** (ADRs + runbooks, no @@ -36,6 +36,9 @@ The one intentionally *coupled* piece is `pr-build.yml`: a single always-running path-filtered required check would deadlock auto-merge). Branch model (unchanged from the split era): PRs → `dev`, `main` → beta, -`release` → prod; infra tracked via `main` on all hosts. +`release` → prod. Infra isn't environment-staged the same way app images are: +beta's and prod's checkouts (both on vps2) track `main`; dev's checkout (on +vps1) tracks `dev` itself, since it's the one environment that isn't a +rehearsal for something downstream. **Before pointing anything live at this repo, read `CUTOVER-NOTES.md`.** diff --git a/docs/onboarding/01-orientation.md b/docs/onboarding/01-orientation.md index 0194611..b53ad9a 100644 --- a/docs/onboarding/01-orientation.md +++ b/docs/onboarding/01-orientation.md @@ -55,40 +55,59 @@ and `PAYLOAD_VER` is the lever that moves it.** See ## The estate -Four machines on a WireGuard mesh (`10.10.0.0/24`): +Hosts are named by **role**, not by environment, plus the operator's desktop — +all on a WireGuard mesh (`10.10.0.0/24`): | Host | Mesh IP | Public | Runs | |---|---|---|---| -| **prod** | `10.10.0.1` | `169.58.46.181`, thermograph.org | Docker **Swarm** app stack, TimescaleDB, Postfix, backups, Centralis | -| **beta** | `10.10.0.2` | `75.119.132.91`, beta.thermograph.org | **compose** app stack, Forgejo (git + CI + registry), Grafana + Loki | -| **desktop** | `10.10.0.3` | — | LAN dev stack, the Forgejo Actions runner, your editor | +| **vps1** | `10.10.0.2` | `75.119.132.91`, git.thermograph.org, dashboard.thermograph.org | Forgejo (git + CI + registry), Grafana + Loki + Alloy, the `emigriffith.dev` portfolio, and **dev** — its own Postgres, plain compose, **mesh-only** (no public DNS, no Caddy site, no TLS) | +| **vps2** | `10.10.0.1` | `169.58.46.181`, thermograph.org, beta.thermograph.org | **prod** and **beta** as two separate Docker **Swarm** stacks, Centralis, Postfix, backups | +| **desktop** | `10.10.0.3` | — | AI-model hosting (voice-to-text, an upcoming-feature LLM) + flex Swarm capacity. Hosts **no** Thermograph environment — `make dev-up` there is a laptop convenience only | | phone | — | — | alerts | +vps2 runs prod and beta **co-resident on one box**: one TimescaleDB instance +serves both, on separate databases and separate roles (`thermograph` / +`thermograph_beta`, the latter `NOSUPERUSER`/`NOCREATEDB`, `CONNECT` revoked +from `PUBLIC`); separate checkouts, env files and loopback LB ports; beta's +Swarm service names are prefixed (`beta-web`, not `web`) because Swarm +registers a service's short name as a DNS alias on every network it joins, and +beta shares prod's network to reach the database. Beta rehearses prod's actual +orchestrator this way — that's the point of putting it next to prod rather +than next to dev. + +The mesh IPs did not move when this topology landed — vps1 is the box that used +to be called "beta", vps2 the one that used to be called "prod". What moved is +which environment lives where. `infra/deploy/env-topology.sh` is the single +source of truth for every per-environment path, port and role; `THERMOGRAPH_ENV` +(`dev`/`beta`/`prod`) is the input a deploy leg passes, since vps2 can no longer +tell beta and prod apart by which host it's running on. + Two consequences you will hit within the first week: -- **Forgejo is mesh-only.** `git.thermograph.org` resolves publicly to beta's - IP, but beta's Caddy rejects `/v2/*` (the registry API) from outside the mesh. - Any host that pulls images needs `10.10.0.2 git.thermograph.org` in +- **Forgejo is mesh-only.** `git.thermograph.org` resolves publicly to vps1's + IP, but vps1's Caddy rejects `/v2/*` (the registry API) from outside the + mesh. Any host that pulls images needs `10.10.0.2 git.thermograph.org` in `/etc/hosts`. -- **Prod and beta run different orchestrators.** Prod is Swarm - (`infra/deploy/stack/`), beta and LAN dev are compose - (`infra/docker-compose.yml`). `deploy.sh` routes between them by reading - `/etc/thermograph/deploy-mode` on the box. Most of the time you don't care — - but when you're reading logs or naming containers, you do - (see [observability](09-observability.md)). +- **Beta and prod run the same orchestrator now — Swarm, as two stacks on one + box.** Dev is the only environment on compose (`infra/docker-compose.yml`), + and it lives alone on vps1. Most of the time you don't care which stack + you're reading — but when you're reading logs or naming containers, you do + (see [observability](09-observability.md)), and on vps2 you additionally have + to say *which* environment: `deploy.sh` there routes on the explicit + `THERMOGRAPH_ENV` a caller passes, not on which host it's running on. ## How a change travels ``` -PR ──(required check: `gate`)──▶ dev ──▶ LAN dev box +PR ──(required check: `gate`)──▶ dev ──▶ dev (vps1, mesh-only) │ promotion PR ▼ - main ──▶ beta.thermograph.org + main ──▶ beta.thermograph.org (vps2) │ promotion PR ← the owner's call ▼ - release ──▶ thermograph.org + release ──▶ thermograph.org (vps2) ``` `dev`, `main` and `release` are **protected** — no direct pushes, for humans or diff --git a/docs/onboarding/02-setup.md b/docs/onboarding/02-setup.md index 52aa099..3ea84ee 100644 --- a/docs/onboarding/02-setup.md +++ b/docs/onboarding/02-setup.md @@ -172,11 +172,18 @@ ends fail closed. With Discord unconfigured it logs once and runs cron-only. ```bash cd infra make dev-up # docker-compose.yml + docker-compose.dev.yml overlay: - # uncapped CPU, backend published on 0.0.0.0:8137 for the LAN + # uncapped CPU, backend published on 0.0.0.0:8137 for your LAN make dev-down ``` -The dev overlay exports `COMPOSE_PROJECT_NAME=thermograph-dev` so the LAN stack +This is a **laptop convenience**, not the hosted dev environment: the actual +`dev` environment now runs on vps1 with its own Postgres, deployed by CI, and +bound only to the WireGuard mesh (`10.10.0.2:8137`) — never `0.0.0.0`, since +vps1 is a public box. See [Infra and secrets](08-infra-secrets.md). A local +`make dev-up` is fine on `0.0.0.0` because it's your own machine's LAN, not the +public internet. + +The dev overlay exports `COMPOSE_PROJECT_NAME=thermograph-dev` so this stack keeps volumes separate from anything else. **Do not remove either half of the project-name pinning** — `infra/docker-compose.yml` pins `name: thermograph`, and without it running compose from `infra/` derives the project name `infra`, @@ -216,7 +223,7 @@ it will be fresher than this page. | Hook | When | What | |---|---|---| -| `prod-guard.sh` | before Bash / live-host MCP calls | Classifies by **allowlist**: only positively-recognised read-only commands pass; everything else asks. Beta is guarded as strictly as prod — it hosts Forgejo, Grafana *and* beta.thermograph.org. | +| `prod-guard.sh` | before Bash / live-host MCP calls | Classifies by **allowlist**: only positively-recognised read-only commands pass; everything else asks. vps1 is guarded as strictly as vps2 — it hosts Forgejo, Grafana *and* the mesh-only `dev` environment, so a destructive command there takes out git, CI and the registry at once, not just a dev sandbox. | | `secrets-guard.sh` | before Write/Edit | Denies any direct write to `infra/deploy/secrets/*.yaml`. Use `sops edit`. | | `lint-after-edit.sh` | after Write/Edit | shellchecks an edited `*.sh` and feeds findings straight back. Exits quietly if shellcheck is missing. | diff --git a/docs/onboarding/03-repo-map.md b/docs/onboarding/03-repo-map.md index 65cebc4..60a3952 100644 --- a/docs/onboarding/03-repo-map.md +++ b/docs/onboarding/03-repo-map.md @@ -114,24 +114,36 @@ Deep dive: [05-frontend.md](05-frontend.md). ## `infra/` — how and where images run ``` -docker-compose.yml the compose stack (beta + LAN dev): - db, backend, lake, daemon, frontend. +docker-compose.yml the compose stack — dev's, on vps1 (and a local + `make dev-up`): db, backend, lake, daemon, frontend. `name: thermograph` is PINNED — see below. -docker-compose.dev.yml LAN overlay: uncapped, backend on 0.0.0.0:8137 +docker-compose.dev.yml dev overlay: uncapped, backend bound to + `${DEV_BIND_ADDR:-127.0.0.1}` — vps1's mesh IP + (10.10.0.2) at deploy time, never 0.0.0.0 docker-compose.openmeteo.yml self-hosted Open-Meteo overlay (prod-only) Makefile compose orchestration only deploy/ - deploy.sh ★ the single entry point for beta and prod - render-secrets.sh renders /etc/thermograph.env from the SOPS vault + env-topology.sh ★ single source of truth: env → host, checkout, + branch, deploy mode, stack name, env file, LB + ports, DB role/name, service prefix + deploy.sh ★ the single entry point for dev, beta and prod + deploy-dev.sh thin wrapper: routes deploy.sh at dev's compose + overlay and secrets policy + render-secrets.sh renders the env file for a given environment from + the SOPS vault secrets/ the vault: common.yaml, prod/beta/dev.yaml, centralis.prod.yaml, example.yaml - stack/ the SWARM path (live on prod): - thermograph-stack.yml, deploy-stack.sh, - autoscale.sh, the LB + stack/ the SWARM path — both live on vps2: + thermograph-stack.yml (prod), + thermograph-beta-stack.yml (beta, prefixed + services, no `db` of its own), deploy-stack.sh, + autoscale.sh, lb/ (per-environment Caddyfiles) + db/provision-env-db.sh creates each environment's database + role on the + shared TimescaleDB instance + provision-dev.sh one-time bootstrap of the dev environment on vps1 swarm/, forgejo/ mesh + Forgejo/CI-runner provisioning Caddyfile the reverse proxy (path-splits FE vs BE) - provision-*.sh host bootstrap: agent access, dev LAN, mail, secrets - db/init/ TimescaleDB init + tuning + provision-*.sh host bootstrap: agent access, mail, secrets migrations/ hand-run SQL migrations ops/ dbq.sh read-only psql into any environment's db container @@ -141,26 +153,26 @@ ACCESS.md, DEPLOY.md, DEPLOY-DEV.md (partly stale — see traps) ``` **Why the project name is pinned.** Compose creates volumes -`thermograph_pgdata` / `_appdata` / `_applogs`, and the Swarm stack declares +`thermograph_pgdata` / `_appdata` / `_applogs`, and the Swarm stacks declare those exact names as `external: true` at the same mount paths. Running compose from `infra/` without `name: thermograph` derives project `infra`, which makes a whole new stack with empty volumes next to the running one. `deploy-dev.sh` exports `COMPOSE_PROJECT_NAME=thermograph-dev` (env wins over the file key) to -keep LAN dev separate on purpose. Keep both halves. +keep dev separate on purpose. Keep both halves. Deep dive: [08-infra-secrets.md](08-infra-secrets.md). ## `observability/` — the logging stack ``` -docker-compose.yml Loki + Grafana (runs on beta) +docker-compose.yml Loki + Grafana (runs on vps1) loki/config.yml mesh-only, filesystem storage, 30-day retention grafana/provisioning/ datasource + dashboard provider (auto-loaded) grafana/provisioning/alerting/ rules, the Discord contact point, the policy grafana/dashboards/*.json the fleet-logs dashboard alloy/config.alloy the per-node shipper alloy/docker-compose.agent.yml runs Alloy on a node (ALLOY_NODE per host) -caddy-grafana.conf beta's Grafana vhost, kept for reference +caddy-grafana.conf vps1's Grafana vhost, kept for reference ``` No build, no deploy automation — it ships by hand. **The repo is the only @@ -177,10 +189,10 @@ Deep dive: [09-observability.md](09-observability.md). | `pr-build.yml` | PR → `dev`/`main` | The `gate` required check. Diffs the PR, builds only touched domains. Deliberately **not** path-filtered. | | `build.yml` | `workflow_call` | Build one domain's image; run the backend suite **inside** the built image. | | `build-push.yml` | push to `dev`/`main`/`release` touching an app domain, or a `v*.*.*` tag | Builds + pushes `sha-<12hex>` images. A version tag builds **both**. | -| `deploy.yml` | push to `main`/`release` touching an app domain | Branch selects environment; matrix covers services; SSHes and runs `deploy.sh`. | -| `infra-sync.yml` | push to `main` touching `infra/**` | Fast-forwards each host's checkout and re-renders secrets. Rolls **no** service. | +| `deploy.yml` | push to `dev`/`main`/`release` touching an app domain | Branch selects environment — three legs: `dev`→vps1, `main`→beta on vps2, `release`→prod on vps2; matrix covers services; SSHes and runs `deploy.sh` with `THERMOGRAPH_ENV` set. | +| `infra-sync.yml` | push to `main` touching `infra/**` | Fast-forwards beta's and prod's checkouts (both on vps2) and re-renders their secrets. Rolls **no** service. | | `observability-validate.yml` | push touching `observability/**`, or `workflow_call` | Parses every artifact; validates Alloy config with the pinned binary; strict alerting checks. | -| `ops-cron.yml` | daily 03:00 UTC | **THE prod backup** (pg_dump) + IndexNow. Uses `PROD_SSH_*`. | +| `ops-cron.yml` | daily 03:00 UTC | **THE prod backup** (pg_dump) + IndexNow, against prod on vps2. Uses `VPS2_SSH_*`. | | `secrets-guard.yml` | every PR and push | Fails if any `infra/deploy/secrets/*.yaml` isn't SOPS-encrypted. | | `shell-lint.yml` | every PR and push | shellcheck (pinned v0.11.0 + sha256) over every `*.sh`. | diff --git a/docs/onboarding/04-backend.md b/docs/onboarding/04-backend.md index b6e59b4..06371c1 100644 --- a/docs/onboarding/04-backend.md +++ b/docs/onboarding/04-backend.md @@ -17,7 +17,7 @@ them is impossible. | **web** | `THERMOGRAPH_ROLE=web` | Serves requests. Never starts the notifier, even if it would win the election — so the web tier scales to N replicas without also scaling background sweeps. | | **worker** | `THERMOGRAPH_ROLE=worker` | Owns the subscription notifier. Still serves requests today. | | **all** | default | Both — the single-process dev default. | -| **lake** | `THERMOGRAPH_ROLE=lake` | `deploy/entrypoint.sh` execs `uvicorn lake_app:app` on port 8141 instead. No database, no migrations. Prod only. | +| **lake** | `THERMOGRAPH_ROLE=lake` | `deploy/entrypoint.sh` execs `uvicorn lake_app:app` on port 8141 instead. No database, no migrations. Runs in every environment — its own Swarm service on prod and beta (`lake` / `beta-lake`), a compose service on dev. | | **daemon** | compose/Swarm sets the command to `/usr/local/bin/thermograph-daemon` | The Go binary: Discord gateway + cron timers. | `deploy/entrypoint.sh` runs `alembic upgrade head` (retried — a fresh Postgres @@ -169,7 +169,7 @@ warming, also warm-only. **The catch-all proxy** — the frontend owns every page and asset. In prod and beta, Caddy path-splits directly, so this proxy is never exercised. It exists as -the fallback for environments with no Caddy in front (LAN dev, bare-metal). It +the fallback for environments with no Caddy in front (dev, bare-metal). It forwards `X-Forwarded-Host`/`-Proto` so the frontend can build correct absolute URLs instead of resolving to the internal hop's own address. Because it's registered dead last, every real backend route wins first — @@ -236,7 +236,8 @@ never overlaps a job with itself, and drops ticks that fire mid-run. ## The lake (`lake_app.py` + `data/era5lake.py`) -Prod-only Swarm service, same image, `THERMOGRAPH_ROLE=lake`. +Runs as a Swarm service on both prod and beta (`lake` / `beta-lake`, same +image, `THERMOGRAPH_ROLE=lake`), and as a compose service on dev. The lake is parquet in an S3-compatible bucket (Contabo, `era5-thermograph`), extracted once from the public Earthmover ERA5 Icechunk archive by diff --git a/docs/onboarding/05-frontend.md b/docs/onboarding/05-frontend.md index c1dd17e..840a3a3 100644 --- a/docs/onboarding/05-frontend.md +++ b/docs/onboarding/05-frontend.md @@ -80,7 +80,7 @@ The static mount is registered last so the explicit routes win. The shell HTML files in `static/` carry an `__ORIGIN__` placeholder for the link-preview/Open Graph tags — preview crawlers need absolute URLs and the host -differs between LAN and prod. The substituted HTML and its ETag are **memoized +differs between dev and prod. The substituted HTML and its ETag are **memoized per origin**, not recomputed per request. Origin resolution prefers `X-Forwarded-Host` over `Host`. That matters when the diff --git a/docs/onboarding/07-ci-and-release.md b/docs/onboarding/07-ci-and-release.md index 5576594..27db954 100644 --- a/docs/onboarding/07-ci-and-release.md +++ b/docs/onboarding/07-ci-and-release.md @@ -3,15 +3,15 @@ ## The branch model ``` -PR ──(required check: `gate`)──▶ dev ──▶ LAN dev box +PR ──(required check: `gate`)──▶ dev ──▶ dev (vps1, mesh-only) │ promotion PR ▼ - main ──▶ beta.thermograph.org + main ──▶ beta.thermograph.org (vps2) │ promotion PR ← the owner's call, always ▼ - release ──▶ thermograph.org + release ──▶ thermograph.org (vps2) ``` `dev`, `main` and `release` are protected — direct pushes are blocked for @@ -40,9 +40,10 @@ for the separate hotfix path. **Commit counts are not deliverable content.** Before promoting, compare the two tips' *trees*. Identical trees mean the same changes are already on the target under different SHAs — promoting there merges nothing while still firing the -target's deploy workflows, including `infra-sync` re-rendering -`/etc/thermograph.env` on prod and beta. The ahead/behind numbers will look like -real work. `promote` refuses this outright. +target's deploy workflows, including `infra-sync` re-rendering prod's and +beta's env files (`/etc/thermograph.env` and `/etc/thermograph-beta.env`, both +on vps2). The ahead/behind numbers will look like real work. `promote` refuses +this outright. **Expect `dev`, `main` and `release` to be mutually divergent, and expect `--ff-only` to fail.** Every promotion merges the source into the target, @@ -111,16 +112,17 @@ Triggers on pushes to `dev`/`main`/`release` touching `backend/**` or - The registry is mesh-only, so the runner host needs `10.10.0.2 git.thermograph.org` in `/etc/hosts`. -### `deploy.yml` — one file, two environments, two services +### `deploy.yml` — one file, three environments, two services Formerly six near-identical files. Branch selects the environment -(`main → beta`, `release → prod`); a matrix covers the services. +(`dev → dev`, `main → beta`, `release → prod`) — **three legs now, not two** — +and a matrix covers the services. The style is **deliberately boring**: no dynamic `fromJSON` matrix, no `cond && secrets.A || secrets.B` ternary. Those are GitHub idioms a Forgejo/act runner may evaluate differently, and the failure mode here is "production does -not deploy" or, worse, "deploys with an empty SSH host". So the two environments -get two explicit, mutually exclusive steps and the tag is computed in shell. +not deploy" or, worse, "deploys with an empty SSH host". So each environment +gets its own explicit, mutually exclusive step and the tag is computed in shell. Preserved from the originals, all load-bearing: @@ -129,11 +131,34 @@ Preserved from the originals, all load-bearing: - 12-hex truncation matching `build-push.yml` exactly. - Per-service, per-environment concurrency with **`cancel-in-progress: false`** — a half-finished deploy must never be cancelled by a newer one. -- **Separate `PROD_SSH_*` credentials**, so a beta credential leak cannot reach - prod. - `appleboy/ssh-action` referenced by full URL; it isn't mirrored in Forgejo's default action registry. +**SSH credentials are keyed by HOST, not by environment**: `VPS1_SSH_*` and +`VPS2_SSH_*`, replacing the old `SSH_*` (which actually meant beta) and +`PROD_SSH_*` (which actually meant prod). That old naming used to read as a +security boundary — "separate `PROD_SSH_*` credentials, so a beta credential +leak cannot reach prod" — and on the old topology it happened to be true, +because beta and prod were also on separate hosts. They no longer are: beta and +prod are now two co-resident Swarm stacks on vps2, so a `VPS2_SSH_*` credential +reaches both by construction, and there is no host-level SSH boundary between +them to preserve. The boundary that replaces it lives at two other levels +instead: the **database** (beta connects as its own `NOSUPERUSER` role to its +own database, `CONNECT` revoked from `PUBLIC` on prod's) and the **filesystem** +(separate checkouts, separate rendered env files, separate deploy locks — +`env-topology.sh` derives all of it so nothing is shared by accident). + +The credential boundary that *is* still real, and still matters, is +**vps1-vs-vps2**: vps1 runs Forgejo, its own CI, and whatever's currently on +`dev` — unreviewed by definition, since `dev` is exactly the branch PRs land on +before review gates them into `main`. That is precisely why the `dev` +environment renders `dev.yaml` alone and never layers `common.yaml` (the +fleet's shared production credentials, including a read-write S3 keypair and +the VAPID push-signing key) — see [Infra and secrets](08-infra-secrets.md). A +`VPS1_SSH_*` leak should never be able to reach a production credential, which +is a property `common.yaml`'s dev exclusion buys independently of which SSH key +was used to get there. + There's also a per-leg refinement: the workflow-level `paths` filter only says *backend or frontend moved*, so each leg re-checks whether **this** push touched **this** domain. Without it a backend-only push would also roll the frontend, @@ -144,17 +169,20 @@ skipping silently strands a change. ### `infra-sync.yml` — infra's own pipeline -Push to `main` touching `infra/**` → SSH to beta **and** prod, fast-forward -`/opt/thermograph`, re-render `/etc/thermograph.env` from the vault. +Push to `main` touching `infra/**` → SSH to vps2, fast-forward **both** +checkouts there (`/opt/thermograph` for prod, `/opt/thermograph-beta` for +beta), re-render each one's own env file from the vault. **Rolls no service.** Image tags are the app domains' axis, not infra's. A -compose change that must recreate containers takes effect on the next app -deploy, or on a by-hand `SERVICE=all … infra/deploy/deploy.sh`. +compose or stack change that must recreate containers takes effect on the next +app deploy, or on a by-hand `SERVICE=all … infra/deploy/deploy.sh` (or +`deploy-dev.sh` for dev). Note the asymmetry: app code *is* environment-staged (`dev`→`main`→`release` -maps to LAN→beta→prod via image tags); **infra is not** — both hosts track infra -via `main`. Prod's *app images* are staged by `release`, but its checkout -follows `main`. +maps to dev(vps1)→beta→prod(vps2) via image tags); **infra is not** — beta's and +prod's checkouts both track infra via `main`, while dev's checkout tracks `dev` +itself. Prod's *app images* are staged by `release`, but its checkout follows +`main`. ### `observability-validate.yml` @@ -171,13 +199,17 @@ silent no-op is exactly what this check exists to prevent. ### `ops-cron.yml` — the prod backup -Daily at 03:00 UTC: `pg_dump` on prod, plus an IndexNow ping. Both SSH into prod -and run inside the already-running stack. +Daily at 03:00 UTC: `pg_dump` on prod, plus an IndexNow ping. Both SSH into +vps2 and run inside prod's already-running stack — specifically prod's, not +beta's, even though both now live on the same box. -**It uses `PROD_SSH_*`, not `SSH_*`.** An earlier revision reused `SSH_*` — so -the "prod" backup was silently dumping *beta*, and prod had no backup at all. If -you touch this file, verify a dump actually lands in -`agent@prod:~/thermograph-backups/`. +**It uses `VPS2_SSH_*`, not a beta-flavoured secret.** An earlier revision (on +the old topology) reused the credential meant for beta's host — so the "prod" +backup was silently dumping *beta*, and prod had no backup at all. On vps2's +co-residency, the host-keyed secret alone doesn't disambiguate prod from beta +the way it used to; the script also needs to target prod's checkout and `db` +specifically. If you touch this file, verify a dump actually lands in +`agent@vps2:~/thermograph-backups/` and that it's prod's data, not beta's. ### `secrets-guard.yml` and `shell-lint.yml` — the backstops @@ -199,7 +231,8 @@ guard. ## Runner facts -- Jobs run on the `docker` label — the LAN box's Forgejo Actions runner. +- Jobs run on the `docker` label — vps1's Forgejo Actions runner (vps1 is where + Forgejo itself lives; the desktop no longer runs a runner). - Job containers get the **host's docker.sock** automounted (Docker-outside-of-Docker, no privileged mode). The job image (`node:20-bookworm`) ships no docker CLI, which is why several workflows diff --git a/docs/onboarding/08-infra-secrets.md b/docs/onboarding/08-infra-secrets.md index ed8eec5..7f0c513 100644 --- a/docs/onboarding/08-infra-secrets.md +++ b/docs/onboarding/08-infra-secrets.md @@ -7,23 +7,36 @@ Read [`infra/CLAUDE.md`](../../infra/CLAUDE.md) alongside this. ## The machines -| Host | Public | Mesh | Orchestrator | Login | +Two VPS boxes named by **role**, not by environment, plus the operator's +desktop: + +| Host | Public | Mesh | Runs | Login | |---|---|---|---|---| -| **prod** | `169.58.46.181` / thermograph.org | `10.10.0.1` | Docker **Swarm** | `agent`, passwordless sudo | -| **beta** | `75.119.132.91` / beta.thermograph.org | `10.10.0.2` | **compose** | `agent`, passwordless sudo | -| **desktop** | — | `10.10.0.3` | compose (LAN dev) | it's your box | +| **vps1** | `75.119.132.91` / git.thermograph.org, dashboard.thermograph.org | `10.10.0.2` | Forgejo, Grafana + Loki + Alloy, `emigriffith.dev`, and **dev** (own Postgres, compose, mesh-only) | `agent`, passwordless sudo | +| **vps2** | `169.58.46.181` / thermograph.org, beta.thermograph.org | `10.10.0.1` | **prod** and **beta**, as two co-resident Docker **Swarm** stacks; Centralis; Postfix; backups | `agent`, passwordless sudo | +| **desktop** | — | `10.10.0.3` | AI-model hosting + flex Swarm capacity. No Thermograph environment; `make dev-up` is a laptop convenience there | it's your box | ```bash -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # prod -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # beta +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # vps2 (prod + beta) +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # vps1 (Forgejo + Grafana + dev) ``` -Beta also hosts **Forgejo** (git + CI + registry) and **Grafana + Loki**, which -is why beta is guarded as strictly as prod: a destructive command there takes -out git, CI and the registry at once. The LAN dev box is deliberately -unguarded. +The mesh IPs did not move when this topology landed — vps1 is the box that used +to be called "beta" (still mesh `10.10.0.2`), vps2 the one that used to be +called "prod" (still mesh `10.10.0.1`). What moved is which environment runs on +which box: beta moved from vps1 onto vps2 to sit next to prod, and dev moved +onto vps1. -Every root-effective command on the VPSes is logged by `auditd` +**vps1 is guarded as strictly as vps2**, not more loosely because it "used to +be beta" or because dev now lives there. It hosts Forgejo, its CI runner and +the registry, plus Grafana/Loki — a destructive command there takes out git, CI +and the registry at once. Dev being mesh-only and a public VPS's tenant, rather +than someone's desktop, is exactly why it gets the same guard as everything +else on that box: it shares a kernel with the estate's source of truth for code +and the monitoring stack, sudo there is passwordless, and it runs whatever +branch is currently in flight. + +Every root-effective command on both VPSes is logged by `auditd` (`ausearch -k agentcmd`). That's a feature — fixes are attributable. Prefer Centralis for routine work (`run_on_host`, `fleet_status`, @@ -31,20 +44,36 @@ Prefer Centralis for routine work (`run_on_host`, `fleet_status`, ## The two orchestrators -Which path a host takes is decided by **`/etc/thermograph/deploy-mode`**: the -string `stack` makes `deploy.sh` exec `deploy/stack/deploy-stack.sh`; anything -else is compose. The workflows never need to know which mode a host runs. +Which path an environment takes is decided by **`infra/deploy/env-topology.sh`** +(`thermograph_topology `, keyed on `THERMOGRAPH_ENV=dev|beta|prod`): it +derives the checkout, branch, deploy mode, stack/compose name, env file, LB +ports and DB role for that environment, and `deploy.sh` execs +`deploy/stack/deploy-stack.sh` when the mode is `stack`, or rolls compose +otherwise. The old host-wide `/etc/thermograph/deploy-mode` marker still exists +as a **fallback** for a by-hand run on a box that predates this file, but it +cannot describe vps2 on its own — vps2 runs two environments, so "which mode +does this host use" is no longer a well-formed question there. -| | compose (beta, LAN dev) | Swarm (prod) | +| | compose (dev) | Swarm (prod, beta) | |---|---|---| -| File | `infra/docker-compose.yml` | `infra/deploy/stack/thermograph-stack.yml` | -| Services | `db`, `backend`, `lake`, `daemon`, `frontend` | `db`, `web`, `worker`, `lake`, `daemon`, `frontend`, `autoscaler`, `autoscaler-lake` | +| Host | vps1 | vps2, both stacks co-resident | +| File | `infra/docker-compose.yml` | `infra/deploy/stack/thermograph-stack.yml` (prod), `thermograph-beta-stack.yml` (beta) | +| Services | `db`, `backend`, `lake`, `daemon`, `frontend` | prod: `db`, `web`, `worker`, `lake`, `daemon`, `frontend`, `autoscaler`, `autoscaler-lake`. beta: `beta-web`, `beta-worker`, `beta-lake`, `beta-daemon`, `beta-frontend` — prefixed, no `db`, no autoscalers | | Rolling | `up -d --no-deps ` | start-first, health-gated, auto-rollback | | Tag file | `deploy/.image-tags.env` | `deploy/.stack-image-tags.env` | Note prod splits `backend` into **`web`** and **`worker`** (the -`THERMOGRAPH_ROLE` split from [04](04-backend.md)); compose runs one `backend` -service in role `all`. +`THERMOGRAPH_ROLE` split from [04](04-backend.md)); dev's compose stack runs one +`backend` service in role `all`. Beta's Swarm services are prefixed +(`beta-web`, not `web`) because Swarm registers a service's short name as a DNS +alias on every network it joins — two stacks both naming a service `web` on +the shared `data` network would make `web` ambiguous, and beta joins that +network (declared `external` in its stack file) purely to reach prod's `db`. +Beta keeps its own `internal` overlay for beta-to-beta traffic and has no `db` +service of its own: **one TimescaleDB instance serves both environments**, on +separate databases (`thermograph` / `thermograph_beta`) and separate roles +(beta's is `NOSUPERUSER`/`NOCREATEDB`/`NOCREATEROLE`, `CONNECT` revoked from +`PUBLIC` on prod's database). `STACK_TEST=1` rehearses the entire Swarm deploy under stack name `thermograph-test` on throwaway volumes and ports `18137`/`18080`. Leftover @@ -58,11 +87,15 @@ coupling is why `name: thermograph` is pinned in the compose file — running compose from `infra/` without it derives project `infra`, silently creating a whole new stack with empty volumes beside the running one. `deploy-dev.sh` exports `COMPOSE_PROJECT_NAME=thermograph-dev` (env wins over the file key) to -keep LAN dev separate on purpose. **Keep both halves.** +keep dev separate on purpose. **Keep both halves.** ## `deploy/deploy.sh` — read this before you touch a deploy -Single entry point for beta and prod. Contract: +Single entry point for dev, beta and prod — the same script for all three; +`infra/deploy/env-topology.sh` is what tells it which checkout, branch, stack +and ports apply. Contract, run from that environment's own checkout +(`/opt/thermograph` for prod, `/opt/thermograph-beta` for beta, both on vps2; +`/opt/thermograph-dev` on vps1 via the `deploy-dev.sh` wrapper): ```bash SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph/infra/deploy/deploy.sh @@ -76,13 +109,17 @@ What it does, in order, and why each step is the way it is: the whole run. Backend and frontend deploys can fire for the same push seconds apart and both SSH into one checkout; concurrent runs race the git reset, the compose project and the tag file. `-w 600` bounds the wait. -2. **Render secrets** from the SOPS vault into `/etc/thermograph.env`, then - source it, so a by-hand run interpolates the same as the systemd unit does. - Guarded on the helper's existence so the very deploy that *introduces* - `render-secrets.sh` is safe. +2. **Render secrets** from the SOPS vault into that environment's own env file + (`/etc/thermograph.env` for prod, `/etc/thermograph-beta.env` for beta, + likewise `/etc/thermograph.env` on dev's separate host), then source it, so + a by-hand run interpolates the same as CI's does. Guarded on the helper's + existence so the very deploy that *introduces* `render-secrets.sh` is safe. 3. **`git reset --hard origin/$BRANCH`** on the checkout root (`BRANCH` defaults - to `main`). ⚠️ **Uncommitted edits in `/opt/thermograph` evaporate here.** - The one exception is `deploy/.image-tags.env`, untracked on purpose. + to `main` for beta/prod, `dev` for dev). ⚠️ **Uncommitted edits in that + checkout evaporate here** — on vps2 that's `/opt/thermograph` for a prod + deploy or `/opt/thermograph-beta` for a beta one; each reset only ever + touches its own checkout, never the sibling's. The one exception is + `deploy/.image-tags.env`, untracked on purpose. 4. **Read `.image-tags.env`** so a single-service roll re-renders compose with *both* services' real tags and never accidentally recreates or downgrades the sibling. @@ -115,12 +152,15 @@ What it does, in order, and why each step is the way it is: Rollback is redeploying the previous image tag. But **do not treat image retention as a rollback guarantee**: both `deploy.sh` and `deploy-stack.sh` -*end* by deleting images outside the running pair. On beta that succeeds, so -beta typically holds **no local rollback target at all**; on prod it usually -fails only because Swarm's stopped task containers still hold references, which -is why prod keeps a handful. Verify the target tag is actually present before -promising a rollback — Centralis's `rollback_to(dry_run: true)` checks both host -and registry. +*end* by deleting images outside the running pair. Beta now runs Swarm exactly +like prod (it used to be compose, on the old topology, where this pruning +reliably left no rollback target at all); on both Swarm stacks it usually +fails only because Swarm's stopped task containers still hold references, +which is why prod and beta both tend to keep a handful. Dev, the one remaining +compose environment, is the one where this prune still reliably succeeds and +leaves nothing to roll back to. Verify the target tag is actually present +before promising a rollback — Centralis's `rollback_to(dry_run: true)` checks +both host and registry. `docker system prune -a` on a live box is forbidden for the same reason. `docker image prune -f` (dangling only) is the safe form. @@ -129,55 +169,81 @@ and registry. **The single source of truth is `infra/deploy/secrets/*.yaml`**, committed encrypted (values only — keys stay readable so diffs mean something) and -rendered into `/etc/thermograph.env` at deploy time by -`deploy/render-secrets.sh`. +rendered at deploy time by `deploy/render-secrets.sh` into each environment's +own env file (`/etc/thermograph.env` for prod, `/etc/thermograph-beta.env` for +beta, `/etc/thermograph.env` again on dev's separate host). **Cycling a key is: edit → commit → deploy.** No SSH, no hand-edited root file, no per-host duplication. | File | Holds | |---|---| -| `common.yaml` | the 16 values identical on prod **and** beta — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config | +| `common.yaml` | the 16 values identical on prod **and** beta — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config. **Never layered under `dev`** — see below | | `prod.yaml` | prod's own — the three held-back credentials, sizing, `THERMOGRAPH_BASE_URL`, Discord + mail credentials that exist nowhere else | | `beta.yaml` | beta's own — the three held-back credentials, sizing, base URL | -| `dev.yaml` | the LAN box's own, **self-contained**, 12 values, no production credential | +| `dev.yaml` | dev's own, **self-contained**, 12 values, no production credential | | `centralis.prod.yaml` | Centralis's nine variables → `/etc/centralis.env`, its own renderer | | `example.yaml` | format reference / CI fixture (fake values) | | `../../.sops.yaml` | which age recipients files encrypt to (plaintext config) | The renderer concatenates `common.yaml` then `.yaml`, so a **host value -wins** (last occurrence of a duplicate key). `` comes from -`/etc/thermograph/secrets-env` on the box. +wins** (last occurrence of a duplicate key). `` is passed explicitly as +`THERMOGRAPH_ENV` by the deploy caller — required now that vps2 alone hosts two +environments and a host-wide marker can't tell them apart; the marker +(`/etc/thermograph/secrets-env`) survives only as the fallback for a by-hand +run on a single-environment box. -### Two design decisions worth understanding +### Two design decisions worth understanding — re-derived for vps1/vps2 **Three credentials are deliberately *not* in `common.yaml`** — `POSTGRES_PASSWORD`, `THERMOGRAPH_AUTH_SECRET`, `THERMOGRAPH_DATABASE_URL`. -They hold identical values on prod and beta today, so by the mechanical rule -they belong there. They're kept per-host anyway because these are the -credentials that let one environment *act as* another: with them, a foothold on -beta (the more exposed box — public Forgejo and Grafana) is a foothold on prod's -database and prod's session signing. They match because beta was seeded from -prod, not because the two are meant to be one system. Keeping them per-host -costs one extra line and buys the ability to diverge. +They hold identical *values* on prod and beta today (beta was seeded from +prod), so by the mechanical rule they'd belong in `common.yaml`. They're kept +per-host anyway, and the reason used to be phrased as a host-isolation +argument — "a foothold on beta is a foothold on prod's database, and beta is +the more exposed box." **That framing no longer holds**: beta and prod are now +two Swarm stacks co-resident on vps2, so a foothold on the host reaches both +regardless of which file a credential lives in. What these three credentials +still buy, correctly stated, is a **database-level** and **file-level** +boundary, not a host-level one: beta's `THERMOGRAPH_DATABASE_URL` names its own +`NOSUPERUSER` role (`thermograph_beta`) against its own database on the shared +TimescaleDB instance, with `CONNECT` revoked from `PUBLIC` on prod's database, +and each environment's `POSTGRES_PASSWORD`/`THERMOGRAPH_AUTH_SECRET` live in a +separate file so rotating prod's stops implying "and beta's too." Keeping them +per-host costs one extra line and buys the ability to diverge — that part is +unchanged. What changed is what "diverge" is defending against: not a beta +compromise reaching a separate prod host, but a beta compromise reaching +prod's role and session-signing key on the host they now already share. -**`dev` renders `dev.yaml` alone** — `deploy-dev.sh` exports -`THERMOGRAPH_SECRETS_SKIP_COMMON=1`. Eleven of `common.yaml`'s sixteen values -are live production credentials, including a **read-write** object-storage -keypair on the bucket that also holds prod's database backups, and the VAPID -private key that signs push to real subscribers. The render is plaintext -concatenation, so layering it would put all of those into every container in the -dev stack — on the operator's desktop, which is also the CI runner whose -`docker`-labelled jobs get the host socket, running unreviewed `dev` code. An +**The credential boundary that actually maps to hosts now is vps1-vs-vps2, not +beta-vs-prod** — which is exactly why `dev` renders `dev.yaml` **alone**, +never layering `common.yaml` (the fleet's shared production credentials): +`deploy-dev.sh` exports `THERMOGRAPH_SECRETS_SKIP_COMMON=1`. Eleven of +`common.yaml`'s sixteen values are live production credentials, including a +**read-write** object-storage keypair on the bucket that also holds prod's +database backups, and the VAPID private key that signs push to real +subscribers. The render is plaintext concatenation, so layering it would put +all of those into every container in the dev stack — on vps1, the box that +also runs Forgejo and its CI runner, whose `docker`-labelled jobs get the host +socket, running whatever branch is currently `dev` (unreviewed by definition: +that's the branch PRs land on before the gate promotes them to `main`). An override in `dev.yaml` wouldn't help: last-wins governs *consumers*, but the -production value is still physically a line in the file. +production value is still physically a line in the file. This is the load- +bearing boundary in the vault today, not a per-environment credential split +that co-residency has already dissolved at the host level. -Dev works fine without them: no S3 keys means the `lake` service answers 503 and -history falls through to the Open-Meteo archive (an accelerator, never a -dependency); VAPID and IndexNow both self-generate and persist to the `appdata` -volume; with no metrics token `/api/v2/metrics` is direct-loopback-only, which -is right on a LAN box. One `common.yaml` value is simply *wrong* for dev — -`THERMOGRAPH_COOKIE_SECURE=1` silently breaks login over plain HTTP. +Dev works fine without `common.yaml`'s values: no S3 keys means the `lake` +service answers 503 and history falls through to the Open-Meteo archive (an +accelerator, never a dependency); VAPID and IndexNow both self-generate and +persist to the `appdata` volume; with no metrics token `/api/v2/metrics` is +direct-loopback-only, which is right for a mesh-only box. One `common.yaml` +value is simply *wrong* for dev — `THERMOGRAPH_COOKIE_SECURE=1` silently breaks +login over dev's plain-HTTP mesh URL. + +None of this makes dev unimportant or unguarded. It's a public VPS's tenant +now, sharing a box with Forgejo and the monitoring stack — not somebody's +desktop — which is precisely why it gets its own exclusion from `common.yaml` +rather than being treated as beneath the vault's concern. ### Working with the vault @@ -223,12 +289,14 @@ infra/ops/iceberg.sh prod -c "select count(*) from era5_daily" ``` Both exec **into the container**. No database is exposed over TCP — each listens -only on its private docker network, and prod's is a Swarm **overlay the prod -host itself cannot route to**, so `ssh -L` works for beta and is *impossible* -for prod. Exec works identically everywhere with no ports, no tunnels, no infra -changes. The prod container name is a Swarm task name that changes on every -redeploy, so it's resolved at call time via `docker ps --filter name=`, never -hardcoded. +only on its private docker network, and prod's and beta's are both Swarm +**overlays the vps2 host itself cannot route to**, so `ssh -L` is *impossible* +for either. (Dev's, on vps1, is plain compose, so `ssh -L` would actually work +there — but exec is used uniformly across environments anyway, so it doesn't +matter which one happens to allow the shortcut.) Exec works identically +everywhere with no ports, no tunnels, no infra changes. The prod and beta +container names are Swarm task names that change on every redeploy, so they're +resolved at call time via `docker ps --filter name=`, never hardcoded. Queries connect as **`thermograph_ro`** — `NOSUPERUSER`, granted only `pg_read_all_data`. Read-only is enforced by Postgres, not by convention: @@ -244,8 +312,9 @@ confirmation — on prod, know what you are doing. ## Backups -The nightly `ops-cron.yml` `pg_dump` into `agent@prod:~/thermograph-backups/` -is it. Known gaps, stated plainly: **a single copy on the same box as the +The nightly `ops-cron.yml` `pg_dump` into `agent@vps2:~/thermograph-backups/` +(prod's data specifically — see [CI and release](07-ci-and-release.md)) is it. +Known gaps, stated plainly: **a single copy on the same box as the database, no offsite yet**, and restores must handle TimescaleDB's `continuous_agg` circular-FK warning (`--disable-triggers`). The `backups/` prefix in the object-storage bucket belongs to those jobs — don't write outside diff --git a/docs/onboarding/09-observability.md b/docs/onboarding/09-observability.md index 8605fba..5ad6fab 100644 --- a/docs/onboarding/09-observability.md +++ b/docs/onboarding/09-observability.md @@ -1,26 +1,34 @@ # 9. Observability -Fleet-wide log aggregation: **Loki + Grafana on beta**, fed by a **Grafana -Alloy** agent on every node, all over the WireGuard mesh. +Fleet-wide log aggregation: **Loki + Grafana on vps1**, fed by a **Grafana +Alloy** agent on every node, all over the WireGuard mesh. vps2 runs two +environments (beta, prod) behind that one Alloy agent; vps1 runs Alloy +alongside the things it's shipping logs *from* (Forgejo, Grafana itself, dev). ``` - prod (10.10.0.1) beta (10.10.0.2) desktop / LAN dev (10.10.0.3) + vps2 (10.10.0.1) vps1 (10.10.0.2) desktop (10.10.0.3) + prod + beta Loki, Grafana, dev ┌────────────┐ ┌──────────────────┐ ┌────────────┐ │ Alloy agent│──wg0──┐ │ Loki ◀── Alloy │ ┌───│ Alloy agent│ └────────────┘ └──▶│ Grafana (Caddy) │◀─┘ └────────────┘ - docker+caddy+app └──────────────────┘ docker+caddy+app + docker+caddy+app └──────────────────┘ docker (AI models) ``` - **Loki** — filesystem storage, **30-day retention**, listening on the mesh IP `10.10.0.2:3100`. Never public. -- **Grafana** — the UI, fronted by beta's Caddy at +- **Grafana** — the UI, fronted by vps1's Caddy at **`dashboard.thermograph.org`** (Google SSO, pre-provisioned accounts only, `allow_sign_up=false`, plus a break-glass local admin). Use that hostname everywhere; **never** `grafana.thermograph.org`. -- **Alloy** — on each node, shipping three sources: every Docker container's - stdout/stderr, Caddy's host access logs, and the app's structured JSON logs - (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase` become - labels). Every line is tagged `host = prod|beta|dev`. +- **Alloy** — one agent per *host*, not per environment: vps2's single agent + ships logs for both beta and prod, vps1's ships Forgejo/Grafana/dev, and the + desktop's ships whatever runs there now. Each ships three sources: every + Docker container's stdout/stderr, Caddy's host access logs, and the app's + structured JSON logs (`errors`/`access`/`audit` `*.jsonl`, parsed so + `level`/`tag`/`phase` become labels). Every app log line is tagged + `host = prod|beta|dev` — that label is the *environment*, not the box, which + matters on vps2 where one Alloy agent ships lines carrying two different + `host` values. There is **no build and no deploy automation** for this domain — it ships by hand. But `observability-validate.yml` parses every artifact that ships to a @@ -42,20 +50,24 @@ Start wide, then narrow: 2. `logs_overview since="1h"` — where the volume is, and what has gone quiet. 3. `logs_query service=… host=…` — the actual lines. -### ⚠ The prod trap +### ⚠ The prod (and now beta) trap -Prod runs a Swarm stack, and its `job="docker"` streams carry **no `service` -label**. A raw `{host="prod", service="backend"}` query matches nothing and -reads as *"no logs"* when it actually means *"wrong selector"*. +Prod and beta both run Swarm stacks — co-resident on vps2 — and their +`job="docker"` streams carry **no `service` label**. A raw +`{host="prod", service="backend"}` query matches nothing and reads as *"no +logs"* when it actually means *"wrong selector"*. This used to be a prod-only +trap when beta ran compose; it now applies to both environments on vps2, and +dev (compose, on vps1) is the one environment this does *not* affect. Use Centralis's `logs_query` `service` argument, which rewrites it to a container name-regex with the churning task suffix wildcarded. Prod service names are `thermograph_web`, `_worker`, `_frontend`, `_db`, `_autoscaler`, -`_lake`. +`_lake`; beta's are the same names prefixed `beta-` (`beta-web`, `beta-worker`, +…) with no `_db` or autoscaler equivalents. -Related: **prod container names change on every redeploy** (the Swarm task -suffix). Never hardcode one; resolve via `docker ps --filter name=` at call -time. +Related: **prod and beta container names both change on every redeploy** (the +Swarm task suffix). Never hardcode one; resolve via `docker ps --filter name=` +at call time. ## The dashboard @@ -81,16 +93,16 @@ Three reasons, all learned the hard way: - Grafana's factory default pointed at the literal string `` — a paging path that had never delivered a message to anyone. -- Beta's Grafana relays SMTP through **prod's** Postfix. Email alerts therefore - travel through the box most likely to be on fire, and vanish exactly when they - matter. +- vps1's Grafana relays SMTP through **vps2's** Postfix (prod's mail service). + Email alerts therefore travel through the box most likely to be on fire, and + vanish exactly when they matter. - Discord is off-estate, works when prod is dead, and reaches a phone. `#ops-alerts` is dedicated. `#dev` / `#uat` / `#prod` are the *product's* alert subscription output — real deliveries to real subscribers — so routing ops noise there would corrupt the evidence. -The webhook URL is a secret and lives only in beta's gitignored `.env` as +The webhook URL is a secret and lives only in vps1's gitignored `.env` as `DISCORD_ALERT_WEBHOOK_URL`. Grafana refuses to start if it's unset. A literal webhook URL committed to the repo is a **hard CI failure**. @@ -135,7 +147,7 @@ pages nobody. **The only proof is a message actually arriving in ## Deploying an observability change > ⚠️ **Merging is not deploying — and this one has already bitten.** -> `/opt/observability` on beta and prod is a checkout of the **archived** +> `/opt/observability` on vps1 and vps2 is a checkout of the **archived** > `emi/thermograph-observability` repo. It can never `git pull` again; the > content now lives here under `observability/`. A previous observability PR > merged and had **zero live effect** for exactly this reason. @@ -146,13 +158,14 @@ hand: back up the live file with a UTC-timestamped suffix, `scp` the new one up, Two gotchas in that sequence: -- Use `sudo docker restart ` on **prod** — not `docker compose`, +- Use `sudo docker restart ` on **vps2** — not `docker compose`, which there demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to - interpolate from. Beta's stack *does* have an `.env`, so compose works there. + interpolate from. vps1's stack *does* have an `.env`, so compose works there. - **`docker restart` will not pick up a new environment variable.** Any change - that introduces one needs `docker compose up -d grafana` on beta. + that introduces one needs `docker compose up -d grafana` on vps1. -Alerting lives on **beta only** — prod and dev run Alloy agents, not Grafana. +Alerting lives on **vps1 only** — vps2 (prod and beta alike) and dev run Alloy +agents, not Grafana. Full step-by-step, including rollback, is in [`observability/README.md`](../../observability/README.md). @@ -177,9 +190,9 @@ Full step-by-step, including rollback, is in - **Grafana cannot alert on its own death.** `AlertingWatchdog` is a *manual* dead-man's switch — it relies on somebody noticing the daily message stopped. An external uptime pinger from off-estate would close this properly. -- **Beta runs a `docker-compose.override.yml` that is not in this repo**, wiring - Grafana's SMTP to prod's Postfix. It should be mirrored into the tracked - compose file or deleted; alerting no longer depends on it. +- **vps1 runs a `docker-compose.override.yml` that is not in this repo**, wiring + Grafana's SMTP to vps2's Postfix (prod's). It should be mirrored into the + tracked compose file or deleted; alerting no longer depends on it. ## When something breaks diff --git a/docs/onboarding/10-recipes.md b/docs/onboarding/10-recipes.md index 58766e7..8538514 100644 --- a/docs/onboarding/10-recipes.md +++ b/docs/onboarding/10-recipes.md @@ -79,14 +79,17 @@ git commit && push && open a PR ``` Then it reaches the hosts by either route: an `infra/**` push to `main` fires -`infra-sync.yml` (re-renders `/etc/thermograph.env` on beta and prod, rolls -nothing), or the next app deploy renders it as step 2 of `deploy.sh`. +`infra-sync.yml` (re-renders prod's and beta's env files — `/etc/thermograph.env` +and `/etc/thermograph-beta.env`, both on vps2 — rolls nothing), or the next app +deploy renders it as step 2 of `deploy.sh`. Dev's vault (`dev.yaml`) is never +part of that sync — it's rendered only by dev's own deploy on vps1, and it +never layers `common.yaml` regardless. Decide **which file** first: - identical on prod *and* beta, and not a "lets one environment act as another" - credential → `common.yaml`; -- per-host, or one of `POSTGRES_PASSWORD` / `THERMOGRAPH_AUTH_SECRET` / + credential → `common.yaml` (never reaches dev, on vps1, at all); +- per-environment, or one of `POSTGRES_PASSWORD` / `THERMOGRAPH_AUTH_SECRET` / `THERMOGRAPH_DATABASE_URL` → `prod.yaml` / `beta.yaml`; - dev → `dev.yaml`, and give it **its own** value. Never a copy of prod's. @@ -209,9 +212,11 @@ SERVICE=backend BACKEND_IMAGE_TAG=sha- \ ``` **Check the tag actually exists first.** Both deploy scripts *end* by deleting -images outside the running pair, so beta typically holds no local rollback -target at all. `rollback_to(dry_run: true)` checks host and registry and will -tell you when it's in neither. +images outside the running pair. Beta now runs Swarm exactly like prod +(co-resident on vps2), so both usually keep a handful of stopped-task +references; dev, the one remaining compose environment (on vps1), is the one +that typically holds no local rollback target at all. `rollback_to(dry_run: +true)` checks host and registry and will tell you when it's in neither. --- diff --git a/docs/onboarding/11-traps.md b/docs/onboarding/11-traps.md index 4604b5d..86bff0a 100644 --- a/docs/onboarding/11-traps.md +++ b/docs/onboarding/11-traps.md @@ -48,19 +48,31 @@ walkthrough as superseded — but the drift goes further than that note admits. The **key-management** sections (Part 1) and the `/etc/hosts` registry note are still accurate and useful. -### `infra/DEPLOY-DEV.md` and `infra/ACCESS.md` reference deleted workflows +### `infra/DEPLOY-DEV.md` and `infra/ACCESS.md` reference deleted workflows — and predate dev becoming a real environment -Both talk about `.forgejo/workflows/deploy-dev.yml`. It does not exist. The two -LAN-dev deploy workflows were deleted rather than ported at the CI -consolidation — they were documented as inert (they targeted a monorepo layout -at `~/thermograph-dev` on a box still holding a split-era checkout). **LAN dev -is a local `make dev-up` concern, not a CI environment.** +Both talk about `.forgejo/workflows/deploy-dev.yml`. It does not exist under +that name. The two original LAN-dev deploy workflows were deleted rather than +ported at the CI consolidation — at the time they were genuinely inert (they +targeted a monorepo layout at `~/thermograph-dev` on a box still holding a +split-era checkout), which is where the old claim **"LAN dev is a local `make +dev-up` concern, not a CI environment"** came from. + +**That claim is now false, and worth flagging precisely because it used to be +true.** `dev` is a first-class hosted environment on vps1 — its own Postgres +container, mesh-only exposure, deployed by CI like beta and prod, as the third +leg of `deploy.yml` (see [07](07-ci-and-release.md) and +[08](08-infra-secrets.md)). The desktop's `make dev-up` is still a real, +useful thing — a laptop-local rehearsal — but it is no longer the *only* way +`dev` runs, and neither doc has been updated to say so. `ACCESS.md` §3c also lists `{build,pr-build,deploy-dev,deploy}.yml` as the workflow set; there are now nine files and the deploy/build ones were collapsed. +It also still describes the host table from before the vps1/vps2 rename — see +`CUTOVER-NOTES.md`'s vps1/vps2 section for the current one. -Everything else in `ACCESS.md` — the host table, the agent-access model, the key -rotation procedure, the Swarm/Forgejo tracks — is current and worth reading. +Everything else in `ACCESS.md` — the agent-access model, the key rotation +procedure, the Swarm/Forgejo tracks — is current and worth reading; just +mentally translate host names. ### `backend/README.md` is written for the split-repo era @@ -93,7 +105,7 @@ it — that's the standing instruction in the root `CLAUDE.md`. ### `observability/README.md`: merging is not deploying -`/opt/observability` on beta and prod is a checkout of the **archived** +`/opt/observability` on vps1 and vps2 is a checkout of the **archived** `emi/thermograph-observability` repo. It can never `git pull` again. A previous observability PR merged and had **zero live effect** for exactly this reason. The README documents the hand-ship procedure; until the checkouts are re-pointed @@ -153,7 +165,7 @@ test run. `infra/docker-compose.yml` pins `name: thermograph`. Without it, running compose from `infra/` derives project `infra` — a silently **new** stack with empty volumes beside the running one. `deploy-dev.sh` exports -`COMPOSE_PROJECT_NAME=thermograph-dev` to keep LAN dev separate on purpose. +`COMPOSE_PROJECT_NAME=thermograph-dev` to keep dev separate on purpose. Keep both halves. ### `deploy.sh` hard-resets the host checkout @@ -167,11 +179,13 @@ roll the sibling onto `local`. ### The deploy scripts eat your rollback image Both `deploy.sh` and `deploy-stack.sh` **end** by deleting images outside the -running pair. On beta that succeeds, so beta typically holds **no local rollback -target at all**; on prod it usually fails only because Swarm's stopped task -containers still hold references. `docker system prune -a` is forbidden on a -live box for the same reason — but the deploy script is the thing that actually -eats the image, not prune. Verify a tag exists before promising a rollback. +running pair. Beta now runs Swarm co-resident with prod on vps2, so both +usually fail to fully prune only because Swarm's stopped task containers still +hold references — dev, the one remaining compose environment (on vps1), is the +one that typically holds **no local rollback target at all**. `docker system +prune -a` is forbidden on a live box for the same reason — but the deploy +script is the thing that actually eats the image, not prune. Verify a tag +exists before promising a rollback. ### `daemon` and `lake` are never deploy targets on their own @@ -189,13 +203,14 @@ line, in file order. A `| head -1` therefore grabbed `db`'s image, and and dropped the daemon from *every* backend deploy. The script now builds the image reference from the same variables compose interpolates. -### Prod's Loki streams carry no `service` label +### Prod's (and beta's) Loki streams carry no `service` label -Prod runs Swarm; its `job="docker"` streams have no `service` label. A raw -`{host="prod", service="backend"}` query matches nothing and reads as "no logs" -when it means "wrong selector". Use `logs_query`'s `service` argument. Prod -container names also change on every redeploy (the task suffix) — resolve via -`docker ps --filter name=` at call time, never hardcode. +Prod and beta both run Swarm, co-resident on vps2; their `job="docker"` streams +have no `service` label. A raw `{host="prod", service="backend"}` query matches +nothing and reads as "no logs" when it means "wrong selector". Use +`logs_query`'s `service` argument. Prod and beta container names also change on +every redeploy (the task suffix) — resolve via `docker ps --filter name=` at +call time, never hardcode. Dev, on compose, doesn't have this problem. ### A Grafana alert rule can provision cleanly and never fire @@ -212,9 +227,9 @@ It's executable documentation until state is bootstrapped. ### The registry is mesh-only -`git.thermograph.org` resolves publicly to beta's IP, but beta's Caddy rejects +`git.thermograph.org` resolves publicly to vps1's IP, but vps1's Caddy rejects `/v2/*` from outside the WireGuard mesh. Any host that pulls needs -`10.10.0.2 git.thermograph.org` in `/etc/hosts`. Beta itself doesn't — it *is* +`10.10.0.2 git.thermograph.org` in `/etc/hosts`. vps1 itself doesn't — it *is* the box. ### Forgejo runners only have the labels they registered with diff --git a/docs/onboarding/README.md b/docs/onboarding/README.md index 9b177df..9150399 100644 --- a/docs/onboarding/README.md +++ b/docs/onboarding/README.md @@ -42,12 +42,13 @@ readings. the superseded original — see [traps](11-traps.md).) - **`infra/`** — compose and Swarm files, deploy scripts, the SOPS secrets vault, Terraform, ops query tooling. -- **`observability/`** — Loki + Grafana on beta, an Alloy agent per node. +- **`observability/`** — Loki + Grafana on vps1, an Alloy agent per node. -Branches stage environments: PR → `dev` (LAN dev) → `main` (beta) → `release` -(prod). The two app domains build and deploy **independently** — that -independence is the whole reason the split-then-reunify history exists, and -[contracts](06-contracts.md) is the list of things that keep it safe. +Branches stage environments: PR → `dev` (vps1, mesh-only) → `main` (beta, vps2) +→ `release` (prod, vps2). The two app domains build and deploy +**independently** — that independence is the whole reason the split-then-reunify +history exists, and [contracts](06-contracts.md) is the list of things that +keep it safe. ## Your first day diff --git a/infra/.claude/skills/key-gaps/SKILL.md b/infra/.claude/skills/key-gaps/SKILL.md index f243540..d353ebf 100644 --- a/infra/.claude/skills/key-gaps/SKILL.md +++ b/infra/.claude/skills/key-gaps/SKILL.md @@ -28,12 +28,24 @@ python3 .claude/skills/key-gaps/key_gaps.py \ ## Audit the LIVE boxes (read-only, key names only) -Gather the key names over SSH (never the values), then audit. Hosts/keys per INFRA.md: +Gather the key names over SSH (never the values), then audit. Hosts per the +root `CLAUDE.md` topology: + +- **vps2** (`169.58.46.181`) hosts BOTH prod and beta as separate Swarm + stacks — TWO live env files on the same box: `/etc/thermograph.env` (prod) + and `/etc/thermograph-beta.env` (beta). "The live env file per environment" + is no longer one path per host; it's one path per environment, and both + environments' files live on this one host. Get both in the same SSH round + trip, or two separate commands against the same host — never assume one + host means one file here. +- **vps1** (`75.119.132.91`) hosts dev, at the same `/etc/thermograph.env` + path (it's a different host, so no collision with prod's file of the same + name). ```sh K=~/.ssh/thermograph_agent_ed25519 -# sudo: /etc/thermograph.env is root-owned (0640). On prod `agent` can read it -# directly too, but sudo works uniformly on both boxes. +# sudo: the env files are root-owned (0640). On these boxes `agent` can often +# read them directly too, but sudo works uniformly regardless. # # The character class must allow DIGITS. This was `^[A-Z_]+=` until 2026-07-24, # which silently skipped every key whose name contains a digit — in this estate @@ -44,9 +56,10 @@ K=~/.ssh/thermograph_agent_ed25519 # no audit: the invented finding sends someone to provision a credential that # already exists, and it was briefly recorded as a root cause of a real bug. # The match still stops at the `=`, so no value is ever read. -ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/prod.keys # prod -ssh -i $K agent@75.119.132.91 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/beta.keys # beta -python3 .claude/skills/key-gaps/key_gaps.py prod=/tmp/prod.keys beta=/tmp/beta.keys +ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/prod.keys # vps2: prod +ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph-beta.env' > /tmp/beta.keys # vps2: beta +ssh -i $K agent@75.119.132.91 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/dev.keys # vps1: dev +python3 .claude/skills/key-gaps/key_gaps.py prod=/tmp/prod.keys beta=/tmp/beta.keys dev=/tmp/dev.keys ``` ## Verify a SOPS cutover matches live diff --git a/infra/.env.example b/infra/.env.example index 3253125..3da1d93 100644 --- a/infra/.env.example +++ b/infra/.env.example @@ -1,9 +1,11 @@ # Local docker-compose overrides. Copy to .env (gitignored) for `docker compose -# up` / `make up` on a dev box: cp .env.example .env +# up` / `make up` on a laptop: cp .env.example .env # # Compose auto-reads a repo-root .env for ${VAR} interpolation in -# docker-compose.yml. In production these live in /etc/thermograph.env instead -# (loaded by the systemd unit), so this file is only for local runs. +# docker-compose.yml. On the fleet (dev/beta/prod), these values live in each +# environment's own rendered env file instead (/etc/thermograph.env, +# /etc/thermograph-beta.env — see deploy/secrets/README.md), so this file is +# only for a local, unmanaged run. # Database password. Compose uses it to initialize the postgres container AND to # build the app's THERMOGRAPH_DATABASE_URL. Change it before first `up`. @@ -33,8 +35,13 @@ THERMOGRAPH_INTERNAL_TOKEN= # replicate with another -- see docker-compose.yml's db service comment. # TIMESCALEDB_TAG=latest-pg18 -# Postgres sizing. Terraform sets these per host in prod/beta (prod DB_MEMORY -# 16g); local/beta default to 8g / 2 CPUs. +# Postgres sizing. Only meaningful for an environment that runs its OWN db +# service — dev's compose stack (this file) and prod's Swarm stack, which +# sizes the ONE shared TimescaleDB instance on vps2 (see +# deploy/secrets/prod.yaml, currently DB_MEMORY=16g). Beta shares that same +# instance rather than running a second one, so a DB_MEMORY/DB_CPUS value in +# beta's own vault file no longer sizes anything — don't be misled by its +# presence there. Local/dev default to 8g / 2 CPUs. # DB_MEMORY=8g # DB_CPUS=2 diff --git a/infra/ACCESS.md b/infra/ACCESS.md index 8ff1f4f..6713241 100644 --- a/infra/ACCESS.md +++ b/infra/ACCESS.md @@ -5,28 +5,55 @@ provisioning / Postgres / Terraform work described in `terraform/README.md` and (for the historical Track A/Track B plan this whole effort grew from, including the decision to split this repo out of the app monorepo) `thermograph-docs/runbooks/implementation-handoff.md` and -`thermograph-docs/architecture/repo-topology-and-infrastructure.md`. Terraform -provisions **prod** (the new 48 GB / 12-core box, `thermograph.org`) and -**beta** (the old VPS, `75.119.132.91`) — see `terraform/README.md` / -`terraform.tfvars.example`. **None of this touches the app repo** (its -backend/frontend source, `Dockerfile`, or CI) — this repo owns only how and -where the already-built app image runs; the app repo owns building it. +`thermograph-docs/architecture/repo-topology-and-infrastructure.md`. +**Note:** `terraform/README.md`'s own host table still describes the +pre-cutover shape (`prod` and `beta` as two single-purpose boxes) and has not +yet been updated to the vps1/vps2 split below — that file is out of this +pass's scope; treat this document as the current source of truth for the +physical topology in the meantime. ``` Track 1: deploy/provision-agent-access.sh — a dedicated full-root login for me -Track 2: deploy/swarm/ — Swarm cluster spanning THREE nodes +Track 2: deploy/swarm/ — Swarm mesh spanning THREE nodes Track 3: deploy/forgejo/ + .forgejo/ — Forgejo + Forgejo Actions, replacing GitHub ``` -**Three nodes, not two**: prod, beta, and **the desktop** (the LAN dev -machine — same box that already runs the pre-Forgejo GitHub self-hosted -runner). This matches the canonical Track B design in the handoff doc; an -earlier revision of this whole effort covered just prod+beta and has been -realigned. +## The estate (renamed by role, not by environment) + +The same two VPS boxes as always — **neither public IP nor WireGuard mesh +address moved** — plus the operator's desktop, now named for what they *do* +rather than which environment happens to live there: + +| Host | Public IP | Mesh IP | Role | +|------|-----------|---------|------| +| **vps1** | `75.119.132.91` | `10.10.0.2` | "Operational programs": Forgejo (git + CI + registry, `git.thermograph.org`), Grafana/Loki/Alloy (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio, and the **dev** environment (mesh-only, `10.10.0.2:8137`, no public DNS/Caddy/TLS) | +| **vps2** | `169.58.46.181` | `10.10.0.1` | "The deployed environment": **prod** and **beta**, as two separate Docker Swarm stacks, plus Centralis, Postfix, the backups, and the one shared TimescaleDB instance both stacks use | +| **desktop** | — | `10.10.0.3` | AI-model hosting (voice-to-text, an upcoming-feature LLM) plus flex Swarm-worker capacity. Hosts **no** Thermograph environment — `make dev-up` still works there, but only as a laptop-local rehearsal | + +This is a **rename**, not a re-provisioning: vps1 is the box that used to be +called "beta" (it already ran Forgejo and Grafana before this split), and vps2 +is the box that used to be called "prod". What moved is the **beta +environment** — off vps1 and onto vps2, so a beta green light is evidence +about prod (same orchestrator, same Postgres build, same Caddy, same mesh +position) — and the **dev environment**, off the desktop and onto vps1, so the +desktop can retire from the Thermograph estate entirely. See +`deploy/env-topology.sh`'s header comment for the full rationale. + +**Three nodes on the Swarm mesh, not two**: vps2 (manager), vps1 (worker), and +the desktop (worker, flex capacity + AI model hosting). One manager, not more: +Raft needs 3 nodes for real quorum-based HA, and this cluster only has 3 nodes +total, so a second manager would still fall short of real HA while adding +split-brain risk. If the manager (vps2) goes down, the workers keep running +whatever was already scheduled on them (Forgejo, pinned to vps1) but the +cluster can't reschedule anything until vps2 is back — acceptable for a small +cluster whose only Swarm-scheduled workload today is Forgejo (prod and beta's +app stacks are separate Swarm stacks that also happen to run on vps2, the +manager, because their volumes are local to that node — see +`deploy/stack/thermograph-stack.yml`'s header). ## Track 1 — Agent access -Run `sudo bash deploy/provision-agent-access.sh` on **prod and beta** (not +Run `sudo bash deploy/provision-agent-access.sh` on **vps1 and vps2** (not the desktop — that's wherever you're already working from, no separate access-provisioning step needed there). Creates a dedicated `agent` user (not raw root login — a distinct name gives a clean audit trail) with passwordless @@ -42,20 +69,27 @@ Both VPS boxes are provisioned and reachable as of this writing: | Host | Role | Public IP | Login | |------|------|-----------|-------| -| prod | Swarm manager; Thermograph's live prod home (`release`, thermograph.org) | `169.58.46.181` | `agent` | -| beta | Swarm worker; `main`/beta.thermograph.org + hosts Forgejo | `75.119.132.91` | `agent` | -| desktop | Swarm worker, LAN dev machine + Forgejo Actions runner | (local) | (already have access) | +| vps2 | Swarm manager; prod AND beta live here (`thermograph.org`, `beta.thermograph.org`) | `169.58.46.181` | `agent` | +| vps1 | Swarm worker; Forgejo + Grafana/Loki + dev (`git.thermograph.org`, `dashboard.thermograph.org`) | `75.119.132.91` | `agent` | +| desktop | Swarm worker, AI-model hosting | (local) | (already have access) | ``` -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # prod -ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # beta +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@169.58.46.181 # vps2 (prod + beta) +ssh -i ~/.ssh/thermograph_agent_ed25519 agent@75.119.132.91 # vps1 (dev + Forgejo + Grafana) ``` +**On vps2, always say which environment you mean.** A shell on vps2 can act on +either `/opt/thermograph` (prod) or `/opt/thermograph-beta` (beta) — there is +no "the app" on that box any more. `deploy.sh` refuses to run if +`THERMOGRAPH_ENV` disagrees with the checkout it was invoked from (see +`deploy/env-topology.sh`), which catches the one genuinely dangerous mistake +here: running prod's checkout with `THERMOGRAPH_ENV=beta`, or the reverse. + The private half of the agent's dedicated keypair lives at `~/.ssh/thermograph_agent_ed25519` on the operator's own machine (never in this repo, never in a CI secret) — same convention `DEPLOY.md` already uses -for the separate CI deploy key (`~/.ssh/thermograph_ci`). If you're setting -this up fresh elsewhere, generate a new pair the same way +for the separate CI deploy keys. If you're setting this up fresh elsewhere, +generate a new pair the same way (`ssh-keygen -t ed25519 -a 100 -C "claude-agent@thermograph-infra" -f ~/.ssh/thermograph_agent_ed25519 -N ""`), embed the new public half in `AGENT_PUBKEY` at the top of `provision-agent-access.sh`, and re-run it on @@ -67,31 +101,21 @@ directory that gets cleaned up, not anywhere-in-repo. Losing it just means regenerating and re-running the provisioning script; it does not lock either box, since your own login is untouched. -**Live as of 2026-07-21** (was: "no Docker / Terraform not applied / Swarm -inactive" — all now done): Docker is installed on all three nodes; the -WireGuard mesh and Docker Swarm are up with all three nodes `Ready` (prod -manager, beta + desktop workers); Terraform has been applied to both VPS -boxes (prod stood up fresh on `release`, beta rebuilt on `main`); Forgejo is -serving at `git.thermograph.org`; and the desktop runs the Forgejo Actions -runner. See the verification checklist at the end. - ## Track 2 — Swarm See `deploy/swarm/README.md` for the exact order of operations across all **three** nodes (WireGuard mesh first, then swarm init/join, then lock the -Swarm ports down to the tunnel interface, then label beta for Forgejo -placement). This cluster's only job is hosting Forgejo — it does not -orchestrate the Terraform-managed app deploys. +Swarm ports down to the tunnel interface, then label vps1 for Forgejo +placement). This cluster's Swarm-scheduled workload is Forgejo; prod and +beta's app stacks are separate `docker stack deploy`s that happen to live on +the same manager node (vps2) because their volumes are local to it today. ## Track 3 — Forgejo, replacing GitHub ### 3a. Stand up Forgejo -See `deploy/forgejo/README.md` — deploys the stack (pinned to beta), then -walks through registering the Actions runner **on the desktop** as a plain -systemd service (`register-lan-runner.sh`), not as a Swarm-scheduled -container. Only one Swarm secret to mint now (`forgejo_db_password`) — the -runner token is no longer a Swarm secret, since the runner isn't a Swarm -service anymore. +See `deploy/forgejo/README.md` — deploys the stack (pinned to **vps1**), then +walks through registering the Actions runner as a plain systemd service +(`register-lan-runner.sh`), not as a Swarm-scheduled container. ### 3b. Migrate the repo (mirror first, verify, then cut over) 1. In Forgejo: **+ New Migration → GitHub**. Point it at @@ -103,13 +127,12 @@ service anymore. 3. **Don't retarget any secret or disable a GitHub workflow yet** — see 3d. ### 3c. Workflows -`.forgejo/workflows/{build,pr-build,deploy-dev,deploy}.yml` mirror +`.forgejo/workflows/{build,pr-build,deploy}.yml` (a single `Deploy` workflow +now covers all three environments — see its own header comment) mirror `.github/workflows/*.yml`, copied with the mechanical changes Forgejo needs: -- `runs-on: ubuntu-latest` → `runs-on: docker` (one of the two labels the - desktop's runner registers under — see 3a; general CI/deploy jobs get a - fresh container, the LAN-deploy job in `deploy-dev.yml` runs bare/host-native - under the `thermograph-lan` label instead, since it needs real filesystem - access). +- `runs-on: ubuntu-latest` → `runs-on: docker` (the runner's `docker` label — + containerized jobs get a fresh container via the docker-socket-automounted + runner). - `appleboy/ssh-action` referenced by full GitHub URL (not mirrored in Forgejo's default action registry, unlike `actions/checkout` / `actions/setup-python`, which resolve unchanged). @@ -119,44 +142,51 @@ service anymore. `build` status check required (Settings → Branches). Forgejo does **not** auto-merge on green by itself — a ready, green, mergeable PR is merged **explicitly** (`POST .../pulls/{n}/merge`, `{"Do": "squash"}`), per - `CLAUDE.md`. That merge is an ordinary push, so `deploy-dev.yml`'s push + `CLAUDE.md`. That merge is an ordinary push, so `deploy.yml`'s push trigger fires naturally afterward. - **Branch/deploy mapping (settled):** `.forgejo/workflows/deploy.yml` - ("Deploy to beta VPS") triggers on `main` and SSHes to **beta** - (`75.119.132.91`) running `deploy/deploy.sh`. **Prod is not deployed by a - push-triggered workflow** — it's deployed with `terraform apply` - (`terraform/README.md`) on the `release` branch. So the promotion chain is - `dev` → `main` (this workflow deploys beta) → `release` (Terraform deploys - prod). There is deliberately no `release`-triggered workflow. + triggers on `dev`, `main` and `release`, and SSHes to the environment each + branch maps to (`dev` → dev on **vps1**, `main` → beta on **vps2**, + `release` → prod on **vps2**), running `deploy/deploy-dev.sh` or + `deploy/deploy.sh` per `deploy/env-topology.sh`. Credentials are keyed by + **host** (`VPS1_SSH_*`, `VPS2_SSH_*`), not by environment, since vps2 alone + now answers to two of them — see the workflow's own header comment for why + that rename matters. A second, independent set of Forgejo workflows -(`.forgejo/workflows/{ci,build-push,ops-cron}.yml`) was added by a parallel +(`.forgejo/workflows/{build-push,ops-cron}.yml`) was added by a parallel effort implementing `thermograph-docs/runbooks/implementation-handoff.md` Track A chunk -7 — see that doc and its own reconciliation PR (which already fixed its -runner-label and ssh-action guesses to match the real infrastructure here). -`ci.yml` duplicated this PR's `build.yml` exactly and was dropped in that -reconciliation; `build-push.yml` (image → Forgejo's built-in registry) and -`ops-cron.yml` (backup + IndexNow) are novel and still present. +7 — see that doc and its own reconciliation PR. `build-push.yml` (image → +Forgejo's built-in registry) and `ops-cron.yml` (backup + IndexNow) are novel +and still present; the latter's secrets are also keyed by host now (see its +own header comment). ### 3d. Cut over — DONE (2026-07-21) The GitHub → Forgejo cutover is complete; this records what was done: -1. Deploy secrets (`SSH_HOST`, `SSH_USER`, `SSH_KEY`, `SSH_PORT`) live in - Forgejo's repo Secrets, pointing at beta with a dedicated CI deploy key. -2. The LAN dev runner was re-pointed to Forgejo via - `deploy/forgejo/register-lan-runner.sh` (systemd --user, on the desktop). +1. Deploy secrets live in Forgejo's repo Secrets, keyed by host + (`VPS1_SSH_*`, `VPS2_SSH_*`) since the vps1/vps2 split, pointing at each + box with a dedicated CI deploy key. +2. The Actions runner is registered against Forgejo via + `deploy/forgejo/register-lan-runner.sh`. 3. The repo was migrated into Forgejo; `dev`/`main`/`release` all promoted through Forgejo and deploys verified live. 4. GitHub is retired as git host and CI (PR "Retire GitHub as the git host and CI platform") — `origin` remains only as a read-only mirror of history. -## Verification checklist — all met (2026-07-21) -- [x] `ssh agent@` and `ssh agent@` both work; `sudo whoami` → `root` +## Verification checklist — all met (2026-07-21, topology as of 2026-07-25) +- [x] `ssh agent@` and `ssh agent@` both work; `sudo whoami` → `root` - [x] Password SSH auth confirmed dead on both boxes (tested with an actual failed login attempt, not just config inspection) -- [x] `docker node ls` (from prod) shows all three nodes `Ready` +- [x] `docker node ls` (from vps2, the manager) shows all three nodes `Ready` - [x] Swarm ports (2377/7946/4789) unreachable from outside the WireGuard tunnel - [x] `https://git.thermograph.org` serves Forgejo over TLS; `/v2/` registry API is 403 from off-mesh (firewalled to the WireGuard CIDR) - [x] A test PR flow verified: required `build` check runs → explicit squash - merge → LAN dev deploy lands + merge → deploy lands on the branch's mapped environment - [x] GitHub retired as git host + CI; kept only as a read-only history mirror + +TODO(cutover): re-verify the checklist above against the vps1/vps2 rename +specifically (e.g. confirm the Forgejo Swarm label now reads `vps1`'s node +name, not a leftover `beta` node name) — this document was updated to match +the new topology brief, but a fresh live check wasn't run as part of this +pass. diff --git a/infra/CLAUDE.md b/infra/CLAUDE.md index ffa2576..390e0e4 100644 --- a/infra/CLAUDE.md +++ b/infra/CLAUDE.md @@ -8,33 +8,67 @@ orchestrator. ## The machines -Per `ACCESS.md`: **dev** (operator's box — LAN dev server and CI runner), -**prod** (`169.58.46.181`, thermograph.org, `agent` user, passwordless sudo), -**beta** (`75.119.132.91`, beta.thermograph.org + Forgejo + Grafana/Loki, -`agent` user). All four are on a WireGuard mesh. Hosts' `/opt/thermograph` is a -checkout of **this monorepo**, not an infra-only repo. +Two VPS boxes plus the operator's desktop, on one WireGuard mesh, named by +ROLE rather than by environment: + +- **vps1** — `75.119.132.91`, mesh `10.10.0.2`. "Operational programs": Forgejo + (git + CI + registry, `git.thermograph.org`), Grafana/Loki/Alloy + (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio site, and the + **dev** environment (`/opt/thermograph-dev`, tracks `dev`), with its own + Postgres container. Dev is mesh-only here — published on `10.10.0.2:8137`, + no public DNS, no Caddy site, no TLS. +- **vps2** — `169.58.46.181`, mesh `10.10.0.1`. "The deployed environment" — + everything an external user can reach: **prod** (`/opt/thermograph`, tracks + `main`, Swarm stack `thermograph`) AND **beta** (`/opt/thermograph-beta`, + tracks `main`, Swarm stack `thermograph-beta`), Centralis, Postfix, and the + backups. One shared TimescaleDB instance serves both, on separate + databases/roles (`thermograph` / `thermograph_beta`). +- **desktop** — mesh `10.10.0.3`. Hosts no Thermograph environment at all: AI + model hosting plus flex Swarm-worker capacity. `make dev-up` still works + there as a laptop-local rehearsal, but that is not an environment. + +`agent` user, passwordless sudo, on vps1 and vps2. A host's `/opt/thermograph*` +is a checkout of **this monorepo**, not an infra-only repo — note vps2 carries +**two** such checkouts side by side (`/opt/thermograph` and +`/opt/thermograph-beta`), which is the one thing most deploy logic here has to +get right that a single-environment host never had to. ## Deploy paths -- **`deploy/deploy.sh`** — the single entry point for beta and prod. Resets the - host checkout to `BRANCH` (default `main`), renders secrets, then routes: - if `/etc/thermograph/deploy-mode` says `stack` it execs - `deploy/stack/deploy-stack.sh`, otherwise it rolls compose services. Per-service - tags persist in untracked `deploy/.image-tags.env` (compose) and +- **`deploy/env-topology.sh`** — sourced by every path below. `THERMOGRAPH_ENV` + (`dev`/`beta`/`prod`) is the input; host, checkout, branch, deploy mode, + stack/compose name, env file, LB ports, DB role/database and service-name + prefix are all derived from it. This exists because vps2 alone runs two + environments — a host-wide marker can no longer answer "which environment is + this", so the caller (the deploy workflow) says so explicitly. +- **`deploy/deploy.sh`** — the single entry point for dev, beta and prod. + Resets the host checkout to `BRANCH`, renders secrets, then routes: beta and + prod (`TG_DEPLOY_MODE=stack`) exec `deploy/stack/deploy-stack.sh`; dev + (`compose`) rolls compose services. The old host-wide + `/etc/thermograph/deploy-mode` marker is honoured only when + `env-topology.sh` can't resolve an environment at all (a checkout that + predates the split, or a by-hand run with nothing set). Per-service tags + persist in untracked `deploy/.image-tags.env` (compose) and `deploy/.stack-image-tags.env` (stack) so the two never mix. -- **`deploy/stack/deploy-stack.sh`** — the Swarm path, live on prod. `backend` - rolls web **and** worker; `frontend` rolls frontend; `all` runs a full - `docker stack deploy`. Start-first, health-gated, auto-rollback. - `STACK_TEST=1` rehearses the whole thing under stack name `thermograph-test` +- **`deploy/stack/deploy-stack.sh`** — the Swarm path, live on vps2 for both + prod and beta, picking the stack file/ports/DB role/service prefix out of + `env-topology.sh`. `backend` rolls web **and** worker; `frontend` rolls + frontend; `all` runs a full `docker stack deploy`. Start-first, health-gated, + auto-rollback. `STACK_TEST=1` rehearses under stack name `thermograph-test` on throwaway volumes and ports `18137`/`18080`. -- **`deploy/deploy-dev.sh`** — thin LAN-dev wrapper (dev compose overlay, - `~/thermograph-dev`). Its CI trigger is inert; see root `CLAUDE.md`. +- **`deploy/deploy-dev.sh`** — thin wrapper around `deploy.sh` for dev on vps1 + (dev compose overlay, `/opt/thermograph-dev`, `THERMOGRAPH_SECRETS_SKIP_COMMON=1`). + Deployed like beta/prod now — a push to `dev` triggers the same `Deploy` + workflow over SSH, no LAN-specific runner involved. - **`Makefile`** — compose orchestration only: `up`, `down`, `db-up`, `dev-up`, `om-up`, `om-backfill`. Volumes are the reason the compose project name is pinned: compose creates `thermograph_pgdata`/`_appdata`/`_applogs`, and the Swarm stack declares those -same names as `external: true` at the same mount paths. +same names as `external: true` at the same mount paths. Beta's stack has no +`db` of its own — it reaches prod's `db` service over prod's `thermograph_internal` +overlay (declared `external: true` in beta's stack file) and keeps a separate +`internal` overlay for its own beta-to-beta traffic. ## Rules @@ -42,13 +76,29 @@ same names as `external: true` at the same mount paths. an apply would attempt full re-provisioning of live hosts. Terraform here is executable documentation until state is bootstrapped. - **Secrets: SOPS vault only** (`deploy/secrets/*.yaml`, `sops edit` → commit → - deploy). Never hand-edit `/etc/thermograph.env` — it is a rendered artifact. - `secrets-guard` CI rejects plaintext. `deploy/secrets/seed-from-live.sh` reads - production secrets and is **not** for an agent to run. + deploy). Never hand-edit `/etc/thermograph.env` / `/etc/thermograph-beta.env` + — they are rendered artifacts. `secrets-guard` CI rejects plaintext. + `deploy/secrets/seed-from-live.sh` reads production secrets and is **not** + for an agent to run. +- **vps1 must never hold prod credentials.** It's the box that runs Forgejo, + its CI runner, and dev's unreviewed branch — the opposite of an isolation + boundary. This is why dev renders `dev.yaml` **alone** and never layers + `common.yaml` (the fleet's shared production credential set). See + `deploy/secrets/README.md`. +- **A foothold on vps2 is a foothold on both beta and prod's host** — they are + co-resident by design now. What still separates them is the database + (separate roles/databases, `CONNECT` revoked from `PUBLIC`) and the + filesystem (separate checkouts, separate rendered env files). Don't describe + beta and prod as isolated at the host or SSH-credential level; they aren't + anymore. - **The ops cron (`.forgejo/workflows/ops-cron.yml`, at the repo root) is THE - prod backup.** It uses `PROD_SSH_*`, not `SSH_*` — an earlier revision reused - `SSH_*` and silently dumped beta while prod had no backup at all. If you touch - it, verify a dump actually lands in `agent@prod:~/thermograph-backups/`. + backup for both application databases and for Forgejo.** Secrets are keyed + by host, not environment: `VPS2_SSH_*` for the `backup`/`indexnow` jobs + (prod and beta's databases, both on vps2) and `VPS1_SSH_*` for + `forgejo-backup` (Forgejo now lives on vps1, not co-located with beta). If + you touch it, verify a dump actually lands for **both** the prod and beta + databases, not just one — an earlier revision backed up only prod and + silently left beta uncovered. - Shell here runs as root over SSH against live hosts with no test suite in front of it. `shell-lint` CI (pinned shellcheck) is the only guard — keep the tree at zero findings. diff --git a/infra/DEPLOY-DEV.md b/infra/DEPLOY-DEV.md index e66796b..65e3806 100644 --- a/infra/DEPLOY-DEV.md +++ b/infra/DEPLOY-DEV.md @@ -1,33 +1,45 @@ -# Dev CI/CD → LAN server on this machine +# Dev CI/CD → the dev environment on vps1 -Parallel to the prod pipeline (`main` → VPS, see `DEPLOY.md`), the **`dev`** -branch continuously deploys to a **LAN server running on this computer**. Git -hosting and CI are self-hosted **Forgejo** (`git.thermograph.org`, reachable -at `http://10.10.0.2:3080` over the WireGuard mesh) — GitHub is retired. +Parallel to beta/prod's pipeline (`main`→beta, `release`→prod, see `DEPLOY.md`), +the **`dev`** branch continuously deploys to the **dev environment on vps1** +(`75.119.132.91`, mesh `10.10.0.2`) — `/opt/thermograph-dev`, a normal fleet +checkout reached over SSH exactly like beta and prod, **not** a stack on the +operator's desktop or LAN any more. Git hosting and CI are self-hosted +**Forgejo** (`git.thermograph.org`, also on vps1, reachable at +`http://10.10.0.2:3080` over the WireGuard mesh) — GitHub is retired. ``` -open PR ──▶ CI (build + boot/health, on the LAN Forgejo runner) +open PR ──▶ CI (build + boot/health, on the `docker`-labeled Forgejo runner) │ green ▼ merge into dev (explicit — Forgejo does not auto-merge on its own; see "Landing a PR" below) │ ▼ - LAN runner (thermograph-lan label) runs deploy/deploy-dev.sh + the Deploy workflow SSHes into vps1 (VPS1_SSH_*, THERMOGRAPH_ENV=dev) + and runs deploy/deploy-dev.sh │ ▼ - ~/thermograph-dev updated ─▶ docker compose stack (backend + frontend + - Postgres 18) brought up, backend on 0.0.0.0:8137 + /opt/thermograph-dev updated ─▶ docker compose stack (backend + frontend + + Postgres) brought up, backend published + MESH-ONLY on 10.10.0.2:8137 ``` -Reachable at `http://:8137/` from any device on your Wi-Fi. +Reachable at `http://10.10.0.2:8137/` from anything already on the WireGuard +mesh, and from nowhere else — no public DNS record, no Caddy site, no TLS. +vps1 is a public VPS running whatever branch is currently in flight (including +unreviewed code), so unlike the old LAN box it must **never** publish on +`0.0.0.0`. -The dev server runs the **same containerized stack as prod** (`docker-compose.yml`), -overlaid with `docker-compose.dev.yml`: **uncapped** (no CPU limits, unlike prod's -backend=4 / frontend=2 / db=2) and backend published on `0.0.0.0:8137` for the -LAN (prod binds loopback behind Caddy; frontend has no published port here -either way, no Caddy to reach it directly, so it's only reached through -backend's own reverse-proxy fallback). Bring it up by hand with `make dev-up`. +The dev stack runs the **same containerized stack as prod** (`docker-compose.yml`), +overlaid with `docker-compose.dev.yml`: **uncapped** (no CPU limits, unlike +prod's Swarm-managed replicas) and backend published on the WireGuard address +(`10.10.0.2:8137` — set via `DEV_BIND_ADDR` in `deploy/env-topology.sh`; prod +and beta bind loopback behind their own Caddy LBs instead). Frontend has no +published port either way — reached only through backend's own +reverse-proxy fallback. Bring dev up by hand with `make dev-up`, which is a +**laptop-local convenience** (uncapped, loopback-published) — not the dev +environment itself, which only ever runs on vps1. ## Why it's built this way @@ -39,26 +51,40 @@ via the API (`POST .../pulls/{n}/merge`, `{"Do": "squash"}`). Nothing merges a ready PR for you automatically just because checks pass, unlike GitHub's old in-workflow `ci-cd.yml` (retired along with the rest of `.github/`). +**Dev moved off the desktop and onto vps1 so the desktop could retire from the +Thermograph estate entirely.** The desktop now hosts AI models and offers flex +Swarm-worker capacity — it is no longer "the LAN dev server" and no longer the +primary CI runner story. What dev keeps that beta and prod do not: + +- It renders `dev.yaml` **alone**, never layering `common.yaml` (the fleet's + shared production credentials) — vps1 also runs Forgejo and its CI, and dev + runs whatever branch is in flight. See the long note in + `deploy/render-secrets.sh` and `deploy/secrets/README.md`. +- It is **mesh-only** — see above. A public VPS running unreviewed branches + must never be reachable from the open internet the way the old LAN box + (behind a home router, reachable only on the Wi-Fi) safely could be. + ## Moving parts | File | Purpose | |------|---------| | `.forgejo/workflows/pr-build.yml` | required status check for PRs into `dev` (calls `build.yml`) | -| `.forgejo/workflows/deploy-dev.yml` | build + deploy on pushes to `dev` (direct, or a PR merge) | +| `.forgejo/workflows/deploy.yml` | the single Deploy workflow — `dev` branch pushes SSH into vps1 and run `deploy-dev.sh` | | `.forgejo/workflows/build.yml` | shared build gate: deps, backend tests, JS syntax check, boot/health check | -| `deploy/deploy-dev.sh` | pull `dev` into `~/thermograph-dev`, `docker compose up` the uncapped LAN stack, health check | +| `deploy/env-topology.sh` | source of truth: dev = vps1, `/opt/thermograph-dev`, branch `dev`, compose mode, mesh-only bind address | +| `deploy/deploy-dev.sh` | thin wrapper around `deploy.sh`: dev's compose overlay + secrets policy, then delegates to the shared pull/roll/health-check logic | | `deploy/secrets/dev.yaml` | dev's SOPS vault — its own secrets and config, rendered to `/etc/thermograph.env` at deploy time (see "Config and secrets") | -| `docker-compose.dev.yml` | dev overlay: no CPU limits, backend published on `0.0.0.0:8137` (frontend unpublished) | -| `deploy/provision-dev-lan.sh` | one-time sudo-free bootstrap of the checkout + runner | -| `deploy/forgejo/register-lan-runner.sh` | registers the Forgejo Actions runner on this machine | +| `docker-compose.dev.yml` | dev overlay: no CPU limits, backend published on the mesh address (frontend unpublished) | +| `deploy/provision-dev.sh` | one-time bootstrap of the checkout + vault marker on vps1 | +| `deploy/forgejo/register-lan-runner.sh` | registers the Forgejo Actions runner (name predates the vps1/vps2 split — see the script's own header) | -`deploy-dev.yml`'s `deploy` job runs on the `thermograph-lan` label — **not** -`[self-hosted, thermograph-lan]` (the array form is what the original GitHub -version used; GitHub implicitly tags every self-hosted runner `self-hosted`, -but Forgejo's runner has only the labels it was explicitly registered with, -so the array form is permanently unschedulable there — a real bug found and -fixed once, worth knowing about if a "deploy" job ever silently stops -appearing in the Actions history again). +`deploy.yml`'s dev leg SSHes in with `VPS1_SSH_*` and runs +`/opt/thermograph-dev/infra/deploy/deploy-dev.sh` directly over SSH — the same +shape as the beta and prod legs, just a different host and script. There is no +longer a `thermograph-lan`-labeled runner job doing host-native +`systemctl`/`docker-compose` calls from inside a CI job; the deploy happens the +same way for all three environments now (a `docker`-labeled runner opens an SSH +session to the target host and runs that environment's deploy script there). `deploy-dev.sh`'s git auth (`GH_TOKEN`/`GITHUB_TOKEN`, Forgejo's per-job token exposed under that name for compatibility) is scoped to `REPO_URL`'s @@ -68,15 +94,25 @@ hardcoded host would make git silently skip the header (no error) and fall through to whatever ambient credentials happen to be available, not fail loudly. -## The self-hosted runner +## The Forgejo Actions runner -A Forgejo Actions **runner** (`forgejo-runner`) runs on this machine, labels +TODO(cutover): `deploy/forgejo/register-lan-runner.sh`'s own header comment +still describes "the desktop" as the canonical placement for this runner, and +its `docker`-labeled jobs are what actually execute `deploy.yml`'s SSH-based +dev/beta/prod deploy steps today. Whether the runner has actually moved to +vps1 (alongside Forgejo) or still runs on the desktop under the new topology +is not settled by anything read for this pass — that script is out of this +document's edit scope. Confirm where it actually lives before relying on the +description below. + +A Forgejo Actions **runner** (`forgejo-runner`) serves this repo, labels `docker` (containerized jobs, node:20-bookworm, with the host's docker.sock automounted in — `container.docker_host: automount` in `~/forgejo-runner/config.yaml` — so `docker build`/`push` work without -privileged Docker-in-Docker) and `thermograph-lan` (host-native jobs, for -`deploy-dev.sh`'s systemctl/docker-compose calls). Installed under -`~/forgejo-runner`, runs as the `forgejo-runner` systemd `--user` service. +privileged Docker-in-Docker) and `thermograph-lan` (a legacy label from when a +deploy job ran host-native on the LAN box; nothing in the current `deploy.yml` +schedules against it, since every environment now deploys over SSH from the +`docker`-labeled job instead). ```bash # status / logs @@ -87,62 +123,66 @@ journalctl --user -u forgejo-runner -f bash deploy/forgejo/register-lan-runner.sh ``` -`git.thermograph.org`'s public DNS resolves to beta's **public** IP, which -the registry's mesh-only ACL (see `deploy/forgejo/README.md`) rejects for -`/v2/` paths — `docker build-push.yml` pushes run against the *host's* -automounted daemon, so it's this **host's** own resolver that needs to route -git.thermograph.org over the mesh, not anything settable from a job -container. If registry pushes ever start failing with an HTTP/TLS mismatch -or a 403 on `/v2/`, check `getent hosts git.thermograph.org` here — it needs -to resolve to beta's WireGuard IP (`10.10.0.2`), typically via a `/etc/hosts` -line, not the public one. +Registry pushes (`build-push.yml`) run against the runner host's own +automounted Docker daemon, so it's that **host's** own resolver that needs to +route `git.thermograph.org` over the mesh, not anything settable from a job +container. If registry pushes ever start failing with an HTTP/TLS mismatch or +a 403 on `/v2/`, check `getent hosts git.thermograph.org` on the runner host — +it needs to resolve to vps1's WireGuard IP (`10.10.0.2`), typically via a +`/etc/hosts` line, not the public one (a non-issue if the runner is co-located +with Forgejo on vps1 itself, since `git.thermograph.org` then resolves to +itself either way). -## The LAN dev service (Docker Compose) +## The dev environment (Docker Compose, on vps1) Runs as a Docker Compose stack, not a bare systemd unit — `docker ps --filter -name=thermograph-dev` shows `thermograph-dev-backend-1`, `thermograph-dev-frontend-1` -(repo-split Stage 4 split the single "app" container in two — frontend has no -published port, reached only through backend's own reverse-proxy fallback) and -`thermograph-dev-db-1` (TimescaleDB). A legacy pre-container -`thermograph-dev.service` systemd --user unit may still exist from before this -cutover; it's stale and `deploy-dev.sh` stops/disables it on every run — don't -trust `systemctl --user status thermograph-dev` as a liveness check, use `docker -ps` instead. +name=thermograph-dev` on vps1 shows `thermograph-dev-backend-1`, +`thermograph-dev-frontend-1` (frontend has no published port, reached only +through backend's own reverse-proxy fallback) and `thermograph-dev-db-1` +(TimescaleDB, dev's **own** container — not the shared instance on vps2). ```bash docker ps --filter name=thermograph-dev # is it up? -docker compose -f docker-compose.yml -f docker-compose.dev.yml logs -f backend frontend # app logs (from ~/thermograph-dev) +docker compose -f docker-compose.yml -f docker-compose.dev.yml logs -f backend frontend # app logs (from /opt/thermograph-dev/infra) ``` -Bootstrap (or rebuild) it by hand: +Bootstrap (or rebuild) it: ```bash -bash deploy/provision-dev-lan.sh +sudo bash deploy/provision-dev.sh ``` -The dedicated checkout at `~/thermograph-dev` is separate from your working tree, -so a deploy never touches uncommitted edits in `~/Code/Thermograph`. +replacing the retired `deploy/provision-dev-lan.sh`, which bootstrapped dev on +the operator's desktop as a sudo-free `systemd --user` stack. Dev is +provisioned like any other fleet environment now — a checkout at +`/opt/thermograph-dev`, root-owned, deployed over SSH by CI like beta and +prod. The dedicated checkout is separate from your own working tree, so a +deploy never touches uncommitted edits there. ## Config and secrets Dev's configuration lives in the same SOPS vault as prod's and beta's — `deploy/secrets/dev.yaml`, encrypted to the same age recipient, rendered to `/etc/thermograph.env` by the same `render-secrets.sh` at deploy time. Changing a -dev value is `sops deploy/secrets/dev.yaml` → commit → deploy, exactly as for the -VPS boxes. **`deploy/secrets/README.md` is the reference**; what follows is only -what is specific to this machine. +dev value is `sops deploy/secrets/dev.yaml` → commit → deploy, exactly as for +beta and prod. **`deploy/secrets/README.md` is the reference**; what follows +is only what is specific to this environment. Dev renders `dev.yaml` **alone**. It does *not* layer `common.yaml`, which is the fleet's shared production credential set — registry token, both S3 keypairs (one read-write pair, on the bucket that also holds prod's backups), the VAPID private key that signs push to real subscribers, the IndexNow key, the metrics token. This -box is the CI runner and runs unreviewed `dev`-branch code; a plaintext render of -that set into `/etc/thermograph.env` would put all of it in every dev container's -environment. `deploy-dev.sh` exports `THERMOGRAPH_SECRETS_SKIP_COMMON=1` to prevent -it, and **aborts the deploy** if the renderer in the checkout cannot honour that. +box also runs Forgejo and its CI, and executes unreviewed `dev`-branch code; a +plaintext render of that set into `/etc/thermograph.env` would put all of it in +every dev container's environment. `deploy-dev.sh` exports +`THERMOGRAPH_SECRETS_SKIP_COMMON=1` to prevent it, and **aborts the deploy** if +the renderer in the checkout cannot honour that. This is a property of the +`dev` **environment** (encoded in `deploy/env-topology.sh`'s `TG_SKIP_COMMON`), +not just of `deploy-dev.sh` — so it holds regardless of which entry point is +used to deploy it. So dev holds its own `THERMOGRAPH_AUTH_SECRET` (its own, not prod's — the `daemon` -service refuses to start without one), its own `POSTGRES_PASSWORD`, a LAN +service refuses to start without one), its own `POSTGRES_PASSWORD`, a mesh-only `THERMOGRAPH_BASE_URL`, `THERMOGRAPH_COOKIE_SECURE=0` (plain HTTP — the shared value is `1` and would silently break login here), and the non-secret config it would otherwise have inherited. It holds **no** S3 keys, no VAPID keypair, no @@ -150,20 +190,26 @@ IndexNow key, no Discord or mail credentials: the lake falls through to the Open-Meteo archive without bucket creds, and the app generates and persists its own VAPID and IndexNow keys into the `appdata` volume. -Two dev-only mechanics, both because this box has **no passwordless sudo** and the -runner is a `systemd --user` service with no tty: +Two dev-only mechanics carried over from the old LAN setup, both worth keeping +even though dev is no longer sudo-free (vps1's `agent` user has passwordless +sudo like any other fleet host) — the render path just doesn't strictly need +them any more: -- the age key is read from `~/.config/sops/age/keys.txt`, not - `/etc/thermograph/age.key` — a root-owned `0400` key would send the renderer down - its `sudo cat` path, where it would hang on a prompt no CI job can answer; -- `/etc/thermograph.env` is **pre-created** owned by the deploying user, so the - renderer takes its in-place write path and no deploy needs `sudo` at all. +- the age key can be read from `~/.config/sops/age/keys.txt` as a fallback + when `/etc/thermograph/age.key` isn't present or readable — see + `deploy-dev.sh`'s own comment on this; +- `/etc/thermograph.env` being pre-created and owned by the deploying user + lets the renderer take its in-place write path without needing `sudo` at + all, which still matters under the Forgejo runner's `systemd --user` + context (no tty to answer a `sudo` prompt). -Setting this up on a fresh dev machine is four typed commands — see **"Setting up a -new dev machine"** in `deploy/secrets/README.md`. Until the -`/etc/thermograph/secrets-env` marker exists the box renders nothing and falls back -to `deploy-dev.sh`'s built-in `POSTGRES_PASSWORD` default, which is the safe state -and the one to return to (delete the marker) if the vault ever gets in the way. +Setting this up on a fresh dev host is a few typed commands — see **"Setting up a +new dev machine"** in `deploy/secrets/README.md` (written for the old desktop +setup; the mechanics are the same on vps1, just root-owned rather than +sudo-free). Until the `/etc/thermograph/secrets-env` marker exists the box +renders nothing and falls back to `deploy-dev.sh`'s built-in +`POSTGRES_PASSWORD` default, which is the safe state and the one to return to +(delete the marker) if the vault ever gets in the way. `REGISTRY_TOKEN` is deliberately not in the vault: dev pulls with the persistent `docker login` credential already in this host's docker config, like the prod/beta @@ -196,25 +242,28 @@ with `{"Do": "squash", "delete_branch_after_merge": true}`. ## Day-to-day - **Ship:** open a PR into `dev`, confirm CI is green, merge it explicitly — - the merge triggers the deploy here. -- **Manual redeploy:** trigger `deploy-dev.yml` via `workflow_dispatch` - (`POST /api/v1/repos/{owner}/{repo}/actions/workflows/deploy-dev.yml/dispatches`, - `{"ref": "dev"}`), or locally `APP_DIR=~/thermograph-dev bash deploy/deploy-dev.sh`. -- **Promote to prod:** merge/fast-forward `dev` into `main` and push — that - triggers the VPS pipeline in `DEPLOY.md`. + the merge triggers a deploy to vps1. +- **Manual redeploy:** trigger the `Deploy` workflow via `workflow_dispatch` + (`POST /api/v1/repos/{owner}/{repo}/actions/workflows/deploy.yml/dispatches`, + `{"ref": "dev"}`), or directly on vps1: + `APP_DIR=/opt/thermograph-dev bash deploy/deploy-dev.sh`. +- **Promote to beta:** merge/fast-forward `dev` into `main` and push — that + triggers the beta leg of the same `Deploy` workflow, per `DEPLOY.md`. ## Monitoring -Logs and dashboards moved to a **separate project** — a central Grafana + Loki -stack fed by a Grafana Alloy agent on every node (including this LAN dev box), -over the WireGuard mesh (`thermograph-observability` on Forgejo; UI at -`https://dashboard.thermograph.org`). The dev node's Alloy agent ships its -container/app logs under `host="dev"`, so LAN dev shows up alongside prod and -beta in the same dashboards. This replaced the old `scripts/dashboard.py`. +Logs and dashboards live in a **separate project** — a central Grafana + Loki +stack, hosted on **vps1 itself**, fed by a Grafana Alloy agent on every node +(prod and beta on vps2, dev and the desktop's own agent, all over the +WireGuard mesh; `thermograph-observability` on Forgejo; UI at +`https://dashboard.thermograph.org`). Dev's Alloy agent ships its +container/app logs under its own `host`/service labels, so dev shows up +alongside prod and beta in the same dashboards. This replaced the old +`scripts/dashboard.py`. For a quick terminal check without Grafana, the app still serves raw counters at the gated `GET /api/v2/metrics` route. Dev sets no `THERMOGRAPH_METRICS_TOKEN`, so -the route is direct-loopback-only — `curl localhost:8137/api/v2/metrics` from this -machine, which is where you already are. (A token exists on the VPS boxes to allow -reads through an SSH tunnel; dev has no reason to carry a copy, and the estate's -copy must not land here — see `deploy/secrets/README.md`.) +the route is direct-loopback-only — `curl localhost:8137/api/v2/metrics` from vps1 +itself. (A token exists on prod/beta to allow reads through an SSH tunnel; dev has +no reason to carry a copy, and the estate's copy must not land here — see +`deploy/secrets/README.md`.) diff --git a/infra/DEPLOY.md b/infra/DEPLOY.md index 35bee32..958176b 100644 --- a/infra/DEPLOY.md +++ b/infra/DEPLOY.md @@ -1,62 +1,71 @@ -# Deploying Thermograph to a prod VPS +# Deploying Thermograph to prod (and beta) on vps2 -> **Prod and beta are now provisioned by Terraform** (`terraform/README.md`) and -> run as a **`docker compose` stack** (backend + frontend + Postgres/TimescaleDB; -> repo-split Stage 4 split the single "app" service in two), not the -> manual venv + systemd model this document originally described. Current shape: -> prod = new box `169.58.46.181`, branch `release`, deployed with `terraform -> apply`; beta = old box `75.119.132.91`, branch `main`, deployed when `main` is -> pushed (`.forgejo/workflows/deploy.yml` SSHes in and runs `deploy/deploy.sh`, -> which `docker compose pull`s the image `build-push.yml` already pushed for -> that commit and `up`s it -- repo-split Stage 6's registry-pull cutover; -> `deploy.sh` no longer builds in place). The **manual walkthrough below -> (`provision.sh`, `thermograph.service`, the venv) is legacy** — kept as a -> from-scratch, no-Terraform reference; the process model is compose, so the -> systemd/venv specifics no longer match how prod/beta actually run. +> **Prod and beta both run as Docker Swarm stacks on vps2** (`169.58.46.181`, +> mesh `10.10.0.1`) — two separate stacks, two separate checkouts +> (`/opt/thermograph` and `/opt/thermograph-beta`), sharing one host because a +> beta green light is meant to be evidence about prod (same orchestrator, same +> Postgres build, same Caddy, same mesh position). `deploy/env-topology.sh` is +> the single source of truth for which environment lives where; every path +> below sources it rather than hardcoding a path or a port. The **manual +> walkthrough below (`provision.sh`, `thermograph.service`, the venv) is +> legacy** — kept as a from-scratch, no-orchestrator reference; prod and beta's +> actual process model is a Swarm stack (`deploy/stack/`), and dev's is plain +> compose on vps1 (see `DEPLOY-DEV.md`), so the systemd/venv specifics below no +> longer match how any live environment actually runs. -Pipeline (beta, on `main`): **push to `main` on Forgejo → `build-push.yml` builds -+ pushes the image, tagged by SHA → Forgejo Actions SSHes to beta → -`deploy/deploy.sh` (`git reset` + `docker login` + `docker compose pull && up`, -schema migration runs in the app entrypoint) → Caddy fronts it with Let's Encrypt -TLS.** Prod (on `release`) is deployed with `terraform apply`, not a push -trigger — its `remote-exec` provisioner does the same pull-instead-of-build. -(GitHub is retired/archived — Forgejo, self-hosted at `git.thermograph.org` / -`10.10.0.2:3080` over the WireGuard mesh, is now the sole git host and CI.) +Pipeline, both environments: **push to `main` (beta) or `release` (prod) on +Forgejo → `build-push.yml` builds + pushes the image, tagged by SHA → the +single `Deploy` workflow (`.forgejo/workflows/deploy.yml`) SSHes into **vps2** +with `THERMOGRAPH_ENV=beta` or `THERMOGRAPH_ENV=prod` → `deploy/deploy.sh` +resolves the right checkout/branch/stack from `env-topology.sh`, `git reset`s +it, renders that environment's secrets, and execs `deploy/stack/deploy-stack.sh` +(schema migration runs as a one-shot task before the roll) → each environment's +own loopback Caddy LB fronts it with TLS.** Credentials are keyed by **host** +(`VPS2_SSH_*`), not by environment — vps2 answers to both prod and beta, so the +environment is passed as data (`THERMOGRAPH_ENV`), not baked into which secret +is used. (GitHub is retired/archived — Forgejo, self-hosted at +`git.thermograph.org` on **vps1** over the WireGuard mesh, is now the sole git +host and CI.) Files that make this work: -| File | Where it lives in prod | Purpose | +| File | Where it lives on vps2 | Purpose | |------|------------------------|---------| -| `.forgejo/workflows/deploy.yml` | Forgejo | CI job that SSHes in and runs the deploy script (beta, on `main`) | -| `.forgejo/workflows/build-push.yml` | Forgejo | builds + pushes the SHA-tagged image `deploy.sh`/Terraform pull | -| `deploy/deploy.sh` | `/opt/thermograph/deploy/deploy.sh` | `git reset` + `docker login` + `docker compose pull && up` + health check + warm | -| `docker-compose.yml` | `/opt/thermograph/docker-compose.yml` | app + Postgres/TimescaleDB services (the process model) | -| `deploy/thermograph.env.example` | `/etc/thermograph.env` | Postgres password, VAPID/auth secrets, `WORKERS`, sizing, base path | -| `deploy/Caddyfile` | `/etc/caddy/Caddyfile` | reverse proxy + automatic HTTPS | -| `terraform/` | run from your machine | provisions the box + brings the stack up (replaces `provision.sh`) | -| `deploy/provision.sh`, `deploy/thermograph.service` | *(legacy)* | pre-compose venv+systemd bootstrap — superseded by Terraform | +| `.forgejo/workflows/deploy.yml` | Forgejo | CI job that SSHes in and runs the deploy script, for all three environments | +| `.forgejo/workflows/build-push.yml` | Forgejo | builds + pushes the SHA-tagged image `deploy.sh` pulls | +| `deploy/env-topology.sh` | `/opt/thermograph{,-beta}/infra/deploy/env-topology.sh` | resolves `THERMOGRAPH_ENV` to a checkout, branch, stack name, ports, DB role | +| `deploy/deploy.sh` | `/opt/thermograph{,-beta}/infra/deploy/deploy.sh` | `git reset` + render secrets + routes to the Swarm-stack deploy | +| `deploy/stack/thermograph-stack.yml` / `thermograph-beta-stack.yml` | same | prod's / beta's Swarm stack (db, web, worker, lake, daemon, frontend; beta has no `db` of its own) | +| `deploy/thermograph.env.example` | `/etc/thermograph.env` (prod) / `/etc/thermograph-beta.env` (beta) | Postgres password, VAPID/auth secrets, `WORKERS`, sizing, base path | +| `deploy/stack/lb/Caddyfile` / `Caddyfile.beta` | each stack's loopback LB container | reverse proxy from the host's real Caddy into the Swarm overlay | +| `terraform/` | run from your machine | provisions the box (host-level only; does not know about the two-environment split — see `ACCESS.md`) | +| `deploy/provision.sh`, `deploy/thermograph.service` | *(legacy)* | pre-compose venv+systemd bootstrap — superseded by the Swarm stack | -The app serves on **loopback only**; Caddy is the only thing exposed to the -internet (ports 80/443). The parquet cache in `data/cache/` lives inside the -`/opt/thermograph` checkout and is gitignored, so `git pull` never touches it. +Each app serves on **loopback only** (prod `127.0.0.1:8137`/`:8080`, beta +`127.0.0.1:8237`/`:8180`); the host's real Caddy is the only thing exposed to +the internet (ports 80/443), reverse-proxying into whichever loopback LB +matches the domain requested. Each environment's own checkout keeps its own +gitignored cache under `data/cache/`, so a `git pull` in one never touches the +other's. **Any deploy host needs a `/etc/hosts` entry for the registry.** -`git.thermograph.org`'s public DNS resolves to beta's public IP, and beta's -own Caddy rejects `/v2/*` (the registry API) from anything outside the +`git.thermograph.org`'s public DNS resolves to **vps1's** public IP, and +vps1's own Caddy rejects `/v2/*` (the registry API) from anything outside the WireGuard mesh (10.10.0.0/24) — confirmed live: `docker login`/`pull` from -prod failed with a 403 until adding `10.10.0.2 git.thermograph.org` to -`/etc/hosts` (10.10.0.2 is beta's mesh IP; every deploy host is already on -the mesh, this is purely a DNS-routing gap, not a connectivity one). Beta +vps2 failed with a 403 until adding `10.10.0.2 git.thermograph.org` to +`/etc/hosts` (10.10.0.2 is vps1's mesh IP; every deploy host is already on +the mesh, this is purely a DNS-routing gap, not a connectivity one). vps1 itself doesn't need this — it's the box Forgejo runs on, so `git.thermograph.org` just resolves to itself either way. `build-push.yml`'s own header comment documents the same requirement for the CI runner host. -**Current production layout** (`deploy/Caddyfile`): the app owns -**`thermograph.org`** at its root (`THERMOGRAPH_BASE=/`, so uvicorn serves `/`, -`/calendar`, `/api/v2/…` with no prefix), and **`emigriffith.dev`** serves a -static portfolio at its root with `emigriffith.dev/thermograph*` permanently -redirecting to `thermograph.org` (prefix stripped). Both domains' `A` records -point at the same VPS; Caddy provisions a separate cert for each. +**Current production layout**: prod's Caddy config owns **`thermograph.org`** +at its root (`THERMOGRAPH_BASE=/`, so uvicorn serves `/`, `/calendar`, +`/api/v2/…` with no prefix) and beta's owns **`beta.thermograph.org`** the same +way. **`emigriffith.dev`** is a separate static portfolio site, served from +**vps1**, not vps2 — it shares no host, checkout or Caddy config with either +app environment. `emigriffith.dev/thermograph*` permanently redirects to +`thermograph.org` (prefix stripped). > **Note:** `deploy.sh` (the CI deploy) only pulls code, reinstalls deps, and > restarts the app service — it does **not** touch Caddy or `/etc/thermograph.env`. @@ -202,24 +211,32 @@ If the repo is **public**, skip this — `git pull` needs no auth. ## Accounts & notifications Accounts, subscriptions, and notifications are **authoritative and not -regenerable** — back them up. On **prod/beta they live in Postgres/TimescaleDB** -(the compose `db` service, on the `pgdata` volume) alongside the climate record; -on **LAN dev** they fall back to SQLite (`data/accounts.sqlite`) when -`THERMOGRAPH_DATABASE_URL` isn't a Postgres URL. +regenerable** — back them up. **Prod, beta and dev all keep them in +Postgres/TimescaleDB** — prod and beta on the one shared instance on vps2 (as +the `thermograph` and `thermograph_beta` databases, separate roles, `CONNECT` +revoked from `PUBLIC` — see `deploy/db/provision-env-db.sh`), dev in its own +`db` container on vps1. SQLite (`data/accounts.sqlite`) is only a fallback for +running the app **outside** any of these stacks (a bare venv, the test suite) +when `THERMOGRAPH_DATABASE_URL` isn't a Postgres URL — it's not what any live +environment actually uses. -- **Back up the Postgres volume** (e.g. `pg_dump` the `thermograph` database) as - part of the VPS backup routine — losing it wipes all accounts and alerts. (On - a SQLite LAN dev box, back up `data/accounts.sqlite` and its `-wal`/`-shm` - sidecars instead.) -- **Multiple workers, one notifier (leader election).** Prod runs several uvicorn - workers (`WORKERS`, currently 8 on prod / 4 on beta). The subscription - evaluator and the recurring scheduler must run in exactly one, so the workers - elect a single leader — a host-local `flock` (`THERMOGRAPH_SINGLETON_LOCK`, - set by the compose app service to `/app/data/notifier.lock`), or a cluster-wide - Postgres advisory lock (`THERMOGRAPH_SINGLETON_PG=1`) if the notifier must be - single across *hosts*. See `backend/core/singleton.py`. `THERMOGRAPH_ROLE` - (`web`/`worker`/`all`, default `all`) further restricts which processes may own - it, so a stateless web tier can scale without scaling notifiers. +- **Back up the Postgres data** (e.g. `pg_dump` the relevant database) as part + of each environment's backup routine — losing it wipes all accounts and + alerts for that environment. (Only a bare-venv/offline run backs up + `data/accounts.sqlite` and its `-wal`/`-shm` sidecars instead.) +- **Multiple workers, one notifier (leader election).** Each environment runs + several uvicorn workers (`WORKERS`, sized per environment — see + `deploy/secrets/{prod,beta,dev}.yaml` and each stack file's replica/worker + counts). The subscription evaluator and the recurring scheduler must run in + exactly one, so the workers elect a single leader — a host-local `flock` + (`THERMOGRAPH_SINGLETON_LOCK`, set by the compose/stack app service to + `/app/data/notifier.lock`), or a cluster-wide Postgres advisory lock + (`THERMOGRAPH_SINGLETON_PG=1`) if the notifier must be single across + *replicas on multiple hosts* — this is what prod's Swarm stack uses, since + `web` scales to N replicas under the autoscaler. See + `backend/core/singleton.py`. `THERMOGRAPH_ROLE` (`web`/`worker`/`all`, + default `all`) further restricts which processes may own it, so a stateless + web tier can scale without scaling notifiers. - **Env vars** (all optional, sensible defaults): - `THERMOGRAPH_COOKIE_SECURE` — set to `1` when serving over HTTPS (the VPS/TLS deploy) so the session cookie is `Secure`. Leave unset for plain-HTTP LAN, where @@ -296,9 +313,11 @@ Why route through a local MTA rather than a provider's API: `ReadWritePaths=…`) needs no change. **Default is safe:** `THERMOGRAPH_MAIL_BACKEND` defaults to `console` — it logs -the message and sends nothing — so LAN dev and the test suite exercise the whole -signup path with no mail server and no risk of mailing a real person. Production -opts in with `=smtp`. +the message and sends nothing — so dev and the test suite exercise the whole +signup path with no mail server and no risk of mailing a real person. Prod +opts in with `=smtp`; beta stays on `console` too (see +`deploy/stack/thermograph-beta-stack.yml`'s header for why beta must never +hold live mail or Discord credentials). ### Postfix unit topology — and the check that lies @@ -431,8 +450,8 @@ Check placement with , which scores all four at onc confirmation or password-reset link is mailed. It currently defaults to a per-boot random value, so every outstanding link would break on restart. - **Digest signups** land in the `pending_digest` table in the accounts DB - (Postgres on prod/beta; authoritative, back it up). The form ships ahead of - delivery on purpose: + (Postgres on every environment; authoritative, back it up). The form ships + ahead of delivery on purpose: addresses are collected now and confirmed once SMTP is live. - **Logs:** `journalctl -u postfix -f`; queue with `mailq`. @@ -464,26 +483,33 @@ regenerated with `python gen_cities.py [n_global] [n_english] [n_extended]`. proved unreliable (wrong matches), so they're edited by hand; a city not in the list simply shows its flavor blurb. Add entries freely — a test checks the slugs are valid. -- **Archive warming runs automatically on every deploy.** `deploy.sh` (prod) and - `deploy-dev.sh` (dev) launch `warm_cities.py` detached in the background after the - health check, so the ~750 city pages serve from cache and a crawl can't burst the - archive API quota. It's idempotent (skips already-cached cells), so only the first - deploy does the full ~25-min warm; later deploys just top up new cities. A page - also self-heals (fetches its archive once) if hit before warming finishes. To run - it by hand: `cd backend && python warm_cities.py --pace 2`. Logs: prod - `logs/warm-cities.log`; dev `journalctl --user -u thermograph-warm-cities`. +- **Archive warming runs automatically on every deploy of prod and dev — not + beta.** `deploy/stack/deploy-stack.sh` (prod) and `deploy.sh`'s compose path + (dev, via `deploy-dev.sh`) launch `warm_cities.py` detached in the background + after the health check, so the ~750 city pages serve from cache and a crawl + can't burst the archive API quota. Beta deliberately skips this + (`TG_POST_DEPLOY=0` in `env-topology.sh`) — it would spend the shared + upstream archive quota filling a cache only a rehearsal environment reads. + Warming is idempotent (skips already-cached cells), so only the first deploy + does the full ~25-min warm; later deploys just top up new cities. A page + also self-heals (fetches its archive once) if hit before warming finishes. To + run it by hand: `cd backend && python warm_cities.py --pace 2`. Logs land at + `logs/warm-cities.log` inside each environment's own backend container (dev + is a normal container stack on vps1 now, not a bare systemd unit; there's no + `journalctl --user` path any more). - **Submit the sitemap** in Google Search Console: `https://thermograph.org/sitemap.xml`. - `jinja2` is a new dependency (already in `requirements.txt`). ## Monitoring Fleet-wide logs and dashboards live in a **separate project** — a central -Grafana + Loki stack fed by a Grafana Alloy agent on every node, over the -WireGuard mesh (`thermograph-observability` on Forgejo; UI at -**`https://dashboard.thermograph.org`**, Google SSO). It ingests every -container's stdout, Caddy's access logs, and the app's structured JSON logs -(`logs/{errors,access,audit}/*.jsonl`). This replaced the old SSH-tailed -`scripts/dashboard.py`. +Grafana + Loki stack, hosted on **vps1** alongside Forgejo, fed by a Grafana +Alloy agent on every node over the WireGuard mesh (`thermograph-observability` +on Forgejo; UI at **`https://dashboard.thermograph.org`**, Google SSO). It +ingests every container's stdout, Caddy's access logs, and the app's +structured JSON logs (`logs/{errors,access,audit}/*.jsonl`) — from prod and +beta on vps2 as well as dev on vps1, each distinguishable by its `host`/service +labels. This replaced the old SSH-tailed `scripts/dashboard.py`. The app still exposes raw counters at the gated `GET /api/v2/metrics` route (loopback-only; refuses any request carrying a proxy `X-Forwarded-*` header, so diff --git a/infra/Makefile b/infra/Makefile index fd349bb..2feaa42 100644 --- a/infra/Makefile +++ b/infra/Makefile @@ -30,8 +30,14 @@ db-up: db-down: docker compose stop db -# The LAN dev stack: uncapped (no CPU limits) and published on 0.0.0.0:8137 so -# phones on the Wi-Fi can reach it (the base file is prod: capped + loopback). +# The dev overlay: uncapped (no CPU limits). Default publish is 0.0.0.0:8137, +# right for a laptop-local rehearsal (e.g. phones on the same Wi-Fi) -- this is +# also the *fleet* dev environment's own overlay, but that environment now runs +# on vps1 (a public VPS, not a LAN box) and deploys via deploy/deploy-dev.sh, +# which overrides the bind address to the WireGuard mesh IP only +# (DEV_BIND_ADDR, see deploy/env-topology.sh) -- never invoke `make dev-up` +# directly on a fleet host, or it publishes on every interface. (The base file +# is prod/beta's shape: capped + loopback, fronted by their own Caddy LB.) DEV_COMPOSE = docker compose -f docker-compose.yml -f docker-compose.dev.yml dev-up: POSTGRES_PASSWORD=$${POSTGRES_PASSWORD:-thermograph-dev} $(DEV_COMPOSE) up -d --build diff --git a/infra/README.md b/infra/README.md index fdd28e8..ed6b477 100644 --- a/infra/README.md +++ b/infra/README.md @@ -3,8 +3,9 @@ Infrastructure for [Thermograph](https://thermograph.org): Terraform host provisioning, the SOPS+age secrets vault, WireGuard/Swarm networking, Forgejo, Caddy, mail, and the deploy scripts that run the already-built app images on each -host. This is a domain of the `emi/thermograph` monorepo — hosts' `/opt/thermograph` -is a checkout of the whole monorepo, and `infra/` never builds app source; it only +host. This is a domain of the `emi/thermograph` monorepo — a host's checkout +(`/opt/thermograph`, `/opt/thermograph-beta`, or `/opt/thermograph-dev`) is a +checkout of the whole monorepo, and `infra/` never builds app source; it only runs published images. - **`terraform/`** — provisions/configures hosts (SSH-driven by default; an @@ -13,37 +14,61 @@ runs published images. executable documentation, not a routine operation. - **`deploy/secrets/`** — the git-native SOPS+age secrets vault (every app secret, encrypted at rest, rendered at deploy time). See `deploy/secrets/README.md`. -- **`deploy/swarm/`, `deploy/forgejo/`** — the WireGuard/Swarm cluster hosting - Forgejo (git + CI + registry). See `ACCESS.md`. -- **`deploy/deploy.sh`** — the single deploy entry point for beta and prod. - Takes `SERVICE=backend|frontend|all` plus `BACKEND_IMAGE_TAG`/`FRONTEND_IMAGE_TAG`, - resets the host checkout, renders secrets, and routes to the right orchestrator. -- **`deploy/stack/`** — the **Swarm** path, live on **prod**: - `thermograph-stack.yml` (db, web, worker, lake, daemon, frontend, autoscaler, - autoscaler-lake), `deploy-stack.sh`, `autoscale.sh`, and the LB. Rolling updates - are start-first, health-gated, with auto-rollback. `STACK_TEST=1` rehearses the - whole stack on throwaway volumes and ports. -- **`docker-compose*.yml`** — the **compose** path, live on **beta** and LAN dev - (db, backend, lake, daemon, frontend). `docker-compose.dev.yml` is the LAN - overlay; `docker-compose.openmeteo.yml` is the self-hosted Open-Meteo overlay. +- **`deploy/swarm/`, `deploy/forgejo/`** — the WireGuard/Swarm mesh spanning + vps1, vps2 and the desktop, and Forgejo (git + CI + registry), pinned to + vps1. See `ACCESS.md`. +- **`deploy/env-topology.sh`** — the single source of truth for where each + environment (`dev`/`beta`/`prod`) lives: host, checkout path, branch, deploy + mode, stack/compose name, env file, LB ports, DB role/database, service-name + prefix. Every deploy path sources it and derives its behavior from it rather + than guessing from the host it happens to be running on — necessary since + vps2 alone now runs two environments. +- **`deploy/deploy.sh`** — the single deploy entry point for dev, beta and prod. + Takes `SERVICE=backend|frontend|all` plus `BACKEND_IMAGE_TAG`/`FRONTEND_IMAGE_TAG` + (and, on vps2, `THERMOGRAPH_ENV=beta|prod` to say which of the two checkouts + it's acting on), resets the host checkout, renders secrets, and routes to the + right orchestrator per `env-topology.sh`. +- **`deploy/deploy-dev.sh`** — a thin dev-specific wrapper around `deploy.sh` + (dev compose overlay, dev's secrets policy). See `DEPLOY-DEV.md`. +- **`deploy/stack/`** — the **Swarm** path, live on **vps2** for both prod and + beta: `thermograph-stack.yml` (prod: db, web, worker, lake, daemon, frontend, + autoscaler, autoscaler-lake) and `thermograph-beta-stack.yml` (beta: the same + service shape minus `db` and the autoscalers, every service name prefixed + `beta-`). `deploy-stack.sh`, `autoscale.sh` and `lb/` are shared by both. + Rolling updates are start-first, health-gated, with auto-rollback. + `STACK_TEST=1` rehearses the whole stack on throwaway volumes and ports. +- **`docker-compose*.yml`** — the **compose** path, live only on **dev** (vps1) + (db, backend, lake, daemon, frontend). `docker-compose.dev.yml` is dev's + mesh-only overlay; `docker-compose.openmeteo.yml` is the self-hosted + Open-Meteo overlay (prod only). `make dev-up` also runs this path locally as + a laptop convenience — that is not an "environment", just a local rehearsal. -Which path a host takes is decided by `/etc/thermograph/deploy-mode`: the string -`stack` makes `deploy.sh` exec `deploy/stack/deploy-stack.sh`; anything else is -compose. The workflows never need to know which mode a host runs. +Which path an environment takes is decided by `deploy/env-topology.sh` +(`TG_DEPLOY_MODE`, keyed by `dev`/`beta`/`prod`): dev is `compose`, beta and prod +are both `stack`. The old host-wide marker `/etc/thermograph/deploy-mode` still +exists as a fallback for a by-hand run with no explicit environment, but it +cannot describe vps2, which runs two environments in two different checkouts — +so it is no longer the thing that decides where files go. The workflows never +need to know which mode an environment runs; they only pass `THERMOGRAPH_ENV`. ## Branches & how changes reach each environment -- **`main`** — what **prod and beta** run. `infra-sync.yml` fires on a push to - `main` touching `infra/**`, fast-forwards each host's `/opt/thermograph` checkout - and re-renders `/etc/thermograph.env` from the vault. It deliberately does - **not** roll any service: image tags are the app domains' axis, not infra's. A - compose or stack change that must recreate containers takes effect on the next - app deploy, or a by-hand `SERVICE=all … deploy/deploy.sh`. -- **`dev`** — what LAN dev would run via `deploy/deploy-dev.sh`. The CI trigger - for this is currently inert (the LAN box still holds a split-era checkout); use - `make dev-up` locally. -- **`release`** — consumed by app deploys only. Both hosts track infra via `main`; - prod's *app images* are staged by `release`, but its checkout follows `main`. +- **`dev`** — deploys to **dev on vps1** (`/opt/thermograph-dev`). +- **`main`** — deploys to **beta on vps2** (`/opt/thermograph-beta`). +- **`release`** — deploys to **prod on vps2** (`/opt/thermograph`). -Note the asymmetry with the app domains: app code IS environment-staged -(`dev`→`main`→`release` maps to LAN→beta→prod via image tags); infra is not. +App code IS environment-staged this way (`dev`→`main`→`release` maps to +vps1/dev → vps2/beta → vps2/prod via image tags, one `Deploy` workflow keyed by +branch). **Infra itself is not environment-staged** the same way: `infra-sync.yml` +fires on a push touching `infra/**` — on `dev` it fast-forwards vps1's dev +checkout, on `main` it fast-forwards **both** of vps2's checkouts (beta and +prod) — re-rendering each environment's own env file from the vault. It +deliberately does **not** roll any service: image tags are the app domains' +axis, not infra's. A compose or stack change that must recreate containers +takes effect on the next app deploy, or a by-hand `SERVICE=all … +deploy/deploy.sh` (or `deploy-dev.sh`) run. + +Note the asymmetry this leaves: dev's infra checkout tracks `dev`, the same +branch its app images are staged by. Beta's and prod's infra checkouts both +track `main` — prod's *app images* are staged by `release`, but prod's *infra +checkout* follows `main`, same as beta's. diff --git a/infra/deploy/Caddyfile b/infra/deploy/Caddyfile deleted file mode 100644 index 6baf615..0000000 --- a/infra/deploy/Caddyfile +++ /dev/null @@ -1,120 +0,0 @@ -# /etc/caddy/Caddyfile on the VPS. -# Each domain's A/AAAA record must already point at this VPS — Caddy provisions a -# Let's Encrypt cert on first request and auto-renews. Nothing else to do for TLS -# (just make sure ports 80 and 443 are open). -# -# Layout: -# thermograph.org/* -> the Thermograph app (path-split across -# backend/frontend -- see below) -# emigriffith.dev/ -> static portfolio site (served straight from disk) -# emigriffith.dev/thermograph* -> permanent redirect to thermograph.org (the app moved) -# -# Thermograph now owns thermograph.org's root, so both services run with -# THERMOGRAPH_BASE=/ (see /etc/thermograph.env) — pages, assets and API all sit -# at "/" with no sub-path prefix. Repo-split Stage 4: backend and frontend are -# two containers now (docker-compose.yml), each on its own loopback port -- -# Caddy path-splits directly to whichever owns a given path, so the browser -# still sees one apparent origin. Repo-split Stage 7a flipped which side owns -# the enumerated list: frontend now owns everything (content pages, the -# interactive tool's SPA shells, every static asset, the dynamic IndexNow key -# file) except the short, stable set below, which mirrors backend/web/app.py's -# own routing exactly (a single catch-all proxy to frontend for everything -# else) -- unlike frontend's paths, backend's don't grow every time a new -# static asset filename is added. A gap in this list still just degrades to -# "one extra hop" through backend's own proxy fallback, never a 404. - -thermograph.org { - encode zstd gzip - - @backend_paths path /api/* /digest /discord/interactions - - # Active health check on the same cheap /healthz route each container's own - # HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting - # never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10). - # 15s (was 5s): plenty responsive for a process that only restarts on a deploy, - # and a quarter of the polling load. - handle @backend_paths { - reverse_proxy 127.0.0.1:8137 { - health_uri /healthz - health_interval 15s - health_timeout 3s - health_status 2xx - } - } - - handle { - reverse_proxy 127.0.0.1:8080 { - health_uri /healthz - health_interval 15s - health_timeout 3s - health_status 2xx - } - } - - # Access-log hygiene: the default JSON encoder serializes full request headers, - # the TLS block, and response headers on every line (measured ~1,133B/line) -- - # strip those with the `filter` format encoder. Also strip the query string from - # the logged URI: Caddy's default logger records request.uri *including* the - # query string, so every `?q=` a visitor typed sat in Loki next to - # their client IP for the full 30-day retention -- a real privacy leak, not just - # noise. Bot/crawler skipping stays out of here: `log_skip` needs Caddy >= 2.7 - # and an upgrade is out of scope, so that's handled downstream in Alloy's - # loki.process "caddy" stage instead (see observability/alloy/config.alloy). - log { - output file /var/log/caddy/thermograph.log { - roll_size 20MiB - roll_keep 5 - } - format filter { - wrap json - fields { - request>headers delete - request>tls delete - resp_headers delete - request>uri regexp \?.* "" - } - } - } -} - -emigriffith.dev { - encode zstd gzip - - # Thermograph moved to its own domain. Send the old sub-path there with a - # permanent redirect, stripping the /thermograph prefix so deep links map - # straight across (…/thermograph/calendar -> thermograph.org/calendar). The - # bare /thermograph (no trailing slash) goes to the new root. - handle_path /thermograph/* { - redir https://thermograph.org{uri} permanent - } - handle /thermograph { - redir https://thermograph.org/ permanent - } - - # Portfolio at the root. Point `root` at the built static site (for the Astro - # portfolio that's its `dist/` output). file_server serves index.html for - # directories and returns a real 404 for missing paths. - handle { - root * /var/www/emigriffith - file_server - } - - log { - output file /var/log/caddy/emigriffith.log - } -} - -# Old bookmarks to the raw IP (the pre-domain URL) would otherwise get bounced to -# HTTPS-on-the-IP, which has no cert and fails. Redirect them to the portfolio domain. -http://75.119.132.91 { - redir https://emigriffith.dev{uri} permanent -} - -# Optional: redirect www -> apex for either domain. Add the www CNAME/A record -# first, then uncomment the matching block. -# www.emigriffith.dev { -# redir https://emigriffith.dev{uri} permanent -# } -# www.thermograph.org { -# redir https://thermograph.org{uri} permanent -# } diff --git a/infra/deploy/Caddyfile.vps1 b/infra/deploy/Caddyfile.vps1 new file mode 100644 index 0000000..113e5db --- /dev/null +++ b/infra/deploy/Caddyfile.vps1 @@ -0,0 +1,61 @@ +# /etc/caddy/Caddyfile on **vps1** (75.119.132.91) — the operational-programs box. +# +# vps1 serves: +# emigriffith.dev -> static portfolio site (from disk) +# git.thermograph.org -> Forgejo (see deploy/forgejo/caddy-git.conf) +# dashboard.thermograph.org -> Grafana (see observability/caddy-grafana.conf) +# +# and hosts the **dev** environment, which is deliberately ABSENT from this +# file. Dev has no site block, no DNS record and no certificate: it publishes on +# the WireGuard address only (http://10.10.0.2:8137) and is reachable from the +# mesh and nowhere else. Dev runs whatever branch is in flight, on the same box +# as Forgejo and its CI — putting it behind a public hostname would be handing +# the internet an unreviewed build. If you ever need it reachable from a phone, +# join the phone to the mesh; do not add a site block here. +# +# beta.thermograph.org is NOT here either — beta moved to vps2, next to prod. +# That is the one edit in this file most likely to be made by mistake: a "beta" +# reference elsewhere in the repo that still points at 75.119.132.91 means this +# box, which no longer runs beta at all. +# +# Each domain's A/AAAA record must already point here — Caddy provisions a +# Let's Encrypt cert on first request and auto-renews. + +emigriffith.dev { + encode zstd gzip + + # Thermograph moved to its own domain. Send the old sub-path there with a + # permanent redirect, stripping the /thermograph prefix so deep links map + # straight across (…/thermograph/calendar -> thermograph.org/calendar). The + # bare /thermograph (no trailing slash) goes to the new root. + handle_path /thermograph/* { + redir https://thermograph.org{uri} permanent + } + handle /thermograph { + redir https://thermograph.org/ permanent + } + + # Portfolio at the root. Point `root` at the built static site (for the Astro + # portfolio that's its `dist/` output). file_server serves index.html for + # directories and returns a real 404 for missing paths. + handle { + root * /var/www/emigriffith + file_server + } + + log { + output file /var/log/caddy/emigriffith.log + } +} + +# Old bookmarks to the raw IP (the pre-domain URL) would otherwise get bounced to +# HTTPS-on-the-IP, which has no cert and fails. Redirect them to the portfolio domain. +http://75.119.132.91 { + redir https://emigriffith.dev{uri} permanent +} + +# Optional: redirect www -> apex. Add the www CNAME/A record first, then +# uncomment. +# www.emigriffith.dev { +# redir https://emigriffith.dev{uri} permanent +# } diff --git a/infra/deploy/Caddyfile.vps2 b/infra/deploy/Caddyfile.vps2 new file mode 100644 index 0000000..48e2da3 --- /dev/null +++ b/infra/deploy/Caddyfile.vps2 @@ -0,0 +1,145 @@ +# /etc/caddy/Caddyfile on **vps2** (169.58.46.181) — the public-facing box. +# +# vps2 serves BOTH environments an external user can reach: +# thermograph.org -> prod (loopback LB on 127.0.0.1:8137 / :8080) +# beta.thermograph.org -> beta (loopback LB on 127.0.0.1:8237 / :8180) +# +# and Centralis at mcp.thermograph.org, which is provisioned separately. +# +# Each domain's A/AAAA record must already point here — Caddy provisions a +# Let's Encrypt cert on first request and auto-renews. Nothing else to do for +# TLS (just make sure ports 80 and 443 are open). +# +# This file was split out of a single deploy/Caddyfile that described both +# boxes' sites at once. That was workable while each box served an unrelated +# set; it stopped being workable when the two ports 8137/8080 started meaning +# "prod on vps2" here and nothing at all on the other box. See Caddyfile.vps1 +# for git/dashboard/portfolio. +# +# Thermograph owns thermograph.org's root, so both services run with +# THERMOGRAPH_BASE=/ — pages, assets and API all sit at "/" with no sub-path +# prefix. Backend and frontend are separate containers, each on its own loopback +# port; Caddy path-splits directly to whichever owns a given path, so the +# browser still sees one apparent origin. The enumerated backend list below +# mirrors backend/web/app.py's own routing exactly (a single catch-all proxy to +# frontend for everything else) -- unlike frontend's paths, backend's don't grow +# every time a new static asset filename is added. A gap in this list still just +# degrades to "one extra hop" through backend's own proxy fallback, never a 404. + +thermograph.org { + encode zstd gzip + + @backend_paths path /api/* /digest /discord/interactions + + # Active health check on the same cheap /healthz route each container's own + # HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting + # never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10). + # 15s (was 5s): plenty responsive for a process that only restarts on a deploy, + # and a quarter of the polling load. + handle @backend_paths { + reverse_proxy 127.0.0.1:8137 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + handle { + reverse_proxy 127.0.0.1:8080 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + # Access-log hygiene: the default JSON encoder serializes full request headers, + # the TLS block, and response headers on every line (measured ~1,133B/line) -- + # strip those with the `filter` format encoder. Also strip the query string from + # the logged URI: Caddy's default logger records request.uri *including* the + # query string, so every `?q=` a visitor typed sat in Loki next to + # their client IP for the full 30-day retention -- a real privacy leak, not just + # noise. Bot/crawler skipping stays out of here: `log_skip` needs Caddy >= 2.7 + # and an upgrade is out of scope, so that's handled downstream in Alloy's + # loki.process "caddy" stage instead (see observability/alloy/config.alloy). + # + # The FILENAME is load-bearing now: Alloy attributes each access log to an + # environment by filename (thermograph.log -> host="prod", beta.log -> + # host="beta"). One Caddy fronts two environments here, so a single log file + # would file every beta request under prod. Renaming either file means + # updating loki.process "caddy" in observability/alloy/config.alloy. + log { + output file /var/log/caddy/thermograph.log { + roll_size 20MiB + roll_keep 5 + } + format filter { + wrap json + fields { + request>headers delete + request>tls delete + resp_headers delete + request>uri regexp \?.* "" + } + } + } +} + +# Beta — same shape as prod, pointed at beta's loopback LB. It moved here from +# its own box; the DNS A record for beta.thermograph.org must point at vps2. +# +# Deliberately NOT indexed: beta serves the same content as prod at a different +# hostname, which is a duplicate-content problem for search engines and an +# invitation for users to land on a rehearsal environment from a search result. +# The app itself never pings IndexNow from beta (see env-topology.sh's +# TG_POST_DEPLOY), and this header closes the other half. +beta.thermograph.org { + encode zstd gzip + + header { + X-Robots-Tag "noindex, nofollow" + } + + @backend_paths path /api/* /digest /discord/interactions + + handle @backend_paths { + reverse_proxy 127.0.0.1:8237 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + handle { + reverse_proxy 127.0.0.1:8180 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + } + + log { + output file /var/log/caddy/beta.log { + roll_size 20MiB + roll_keep 5 + } + format filter { + wrap json + fields { + request>headers delete + request>tls delete + resp_headers delete + request>uri regexp \?.* "" + } + } + } +} + +# Optional: redirect www -> apex. Add the www CNAME/A record first, then +# uncomment. +# www.thermograph.org { +# redir https://thermograph.org{uri} permanent +# } diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md new file mode 100644 index 0000000..72934a5 --- /dev/null +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -0,0 +1,420 @@ +# Runbook — moving beta to vps2 and dev to vps1 + +**Status: not executed.** Everything this runbook describes is landed in the +repo and live nowhere. The estate still runs the old shape until someone works +through the steps below. + +## What changes + +| | before | after | +|---|---|---| +| `75.119.132.91` (**vps1**) | beta + Forgejo + Grafana/Loki | Forgejo + Grafana/Loki + **dev** | +| `169.58.46.181` (**vps2**) | prod + Centralis + Postfix + backups | **prod + beta** + Centralis + Postfix + backups | +| desktop | LAN dev server + CI runner | AI models + flex Swarm worker, **no environment** | +| databases | one Postgres per environment | one instance on vps2, `thermograph` + `thermograph_beta` | + +Mesh addresses do **not** move: vps1 stays `10.10.0.2`, vps2 stays `10.10.0.1`. +Every "beta = 75.119.132.91" reference anywhere is wrong after this — that +address is vps1, and sending a beta-intended command there is the most likely +way to do damage during this cutover. + +## Order, and why it is this order + +Beta moves first and completely, while dev stays where it is. Then dev moves. +The two halves are independent, so a problem in one never forces a rollback of +the other — and beta is the half that carries a public hostname and a TLS +certificate, so it gets done while you have the most attention. + +Prod is touched twice, and both touches are additive: a new database + role on +its Postgres instance, and a new site block in its Caddyfile. Prod's stack file, +its services, its volumes and its env file are not modified by this work at all. + +**Expected downtime:** beta, roughly the length of a dump/restore plus a stack +bring-up (~10–20 min). Dev, as long as you like. **Prod: none** — unless step 3 +or 8 is done wrong, which is what the verification lines are for. + +--- + +## 0. Before you start + +- [ ] **Lower the DNS TTL** on `beta.thermograph.org` to 300s, at least an hour + before step 7. Do this first; it is the only step with a lead time. +- [ ] **Create the new Forgejo Actions secrets** (Settings → Secrets). The + workflows on this branch reference them and will fail without them: + + VPS1_SSH_HOST=75.119.132.91 VPS1_SSH_USER=agent + VPS1_SSH_KEY= VPS1_SSH_PORT=22 + VPS2_SSH_HOST=169.58.46.181 VPS2_SSH_USER=agent + VPS2_SSH_KEY= VPS2_SSH_PORT=22 + + Keep the old `SSH_*` / `PROD_SSH_*` secrets until step 11 — they are the + rollback path, and deleting them early strands you. + + **Create these BEFORE merging.** `infra-sync.yml` fires on any push to + `dev` or `main` touching `infra/**`, and this change touches a great deal + of it. Without the new secrets those jobs SSH to an empty host and fail; + `sync-beta` additionally targets `/opt/thermograph-beta`, which does not + exist until step 1. Nothing is damaged either way — the jobs only fetch + and render — but you will get a red run and a misleading alert. + + The app `Deploy` workflow does **not** fire on this merge: it is + path-filtered to `backend/**` and `frontend/**`, and this change touches + neither. +- [ ] **Take a fresh prod backup and confirm it landed**, rather than trusting + the schedule: + + ``` + # from a machine with repo access + # Actions -> "Ops cron (backup + IndexNow)" -> Run workflow + ssh agent@169.58.46.181 'ls -lh ~/thermograph-backups | tail -3' + ``` +- [ ] **Confirm you can decrypt the vault** (`sops -d infra/deploy/secrets/beta.yaml` + from `infra/`). Steps 3 and 5 need the rendered beta password. +- [ ] Have the PR merged to `dev` and promoted to `main`. Hosts pull `main`, so + nothing below works from an unmerged branch. + +--- + +## 1. Give beta a checkout on vps2 + +``` +ssh agent@169.58.46.181 +sudo git clone --branch main http://10.10.0.2:3080/emi/thermograph.git /opt/thermograph-beta +sudo chown -R agent:agent /opt/thermograph-beta +``` + +Prod's checkout at `/opt/thermograph` is untouched. Two checkouts side by side +is the whole point: a `git reset --hard` for beta must never be able to move +prod's tree. + +**Verify:** `git -C /opt/thermograph-beta log --oneline -1` matches +`git -C /opt/thermograph log --oneline -1`. + +--- + +## 2. Render beta's env file on vps2 + +``` +ssh agent@169.58.46.181 +cd /opt/thermograph-beta/infra +. deploy/render-secrets.sh +render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env +``` + +The age key is already at `/etc/thermograph/age.key` on this box (prod uses it); +beta needs no second copy. + +**Verify** — beta's file must name beta's role and database, and prod's must be +untouched: + +``` +sudo grep -c . /etc/thermograph-beta.env # non-zero +sudo grep -o 'thermograph_beta' /etc/thermograph-beta.env | head -1 +sudo grep -o '@db:5432/thermograph$' /etc/thermograph.env # prod: still 'thermograph' +``` + +--- + +## 3. Create beta's role and database on the shared instance + +This is the first step that touches prod's Postgres. It only ever adds; it drops +nothing. + +``` +ssh agent@169.58.46.181 +sudo bash /opt/thermograph-beta/infra/deploy/db/provision-env-db.sh beta +``` + +**Verify the isolation is real** — this is the whole justification for one +instance serving two environments, so actually run it: + +``` +DBC=$(docker ps -q --filter label=com.docker.swarm.service.name=thermograph_db | head -1) +# beta's role must NOT be able to reach prod's database: +docker exec "$DBC" psql -U thermograph_beta -d thermograph -c 'select 1' # must FAIL +# beta's read-only role exists and really is read-only: +docker exec "$DBC" psql -U thermograph_beta_ro -d thermograph_beta -c 'select 1' # must SUCCEED +docker exec "$DBC" psql -U thermograph_beta_ro -d thermograph_beta \ + -c 'create table _x(i int)' # must FAIL +# prod's own role is UNTOUCHED — still a superuser: +docker exec "$DBC" psql -U thermograph -d postgres -tAc \ + "select rolsuper from pg_roles where rolname='thermograph'" # must be t +# and prod is still healthy: +curl -fsS -o /dev/null -w '%{http_code}\n' https://thermograph.org/api/health +``` + +The script **refuses to run for prod** — prod's `thermograph` role is the +instance's bootstrap superuser, not a guest, and provisioning it would demote +it. That guard is why the superuser check above should never fail; run it +anyway. + +If the first command SUCCEEDS, stop. `CONNECT` was not revoked and beta would be +able to read production data; re-run the provisioning script and re-check before +going further. + +**Rollback:** `DROP DATABASE thermograph_beta; DROP ROLE thermograph_beta;` — +prod is unaffected either way. + +--- + +## 4. Move beta's data (optional) + +Beta's existing database still lives on vps1. Decide honestly whether you want +it: beta's value is a rehearsal of prod, and a fresh schema with a handful of +test accounts is often *better* than carrying old beta state across. If you do +want it: + +``` +# on vps1 — dump the old beta database +ssh agent@75.119.132.91 \ + 'docker exec thermograph-db-1 pg_dump -U thermograph -d thermograph --format=custom' \ + > /tmp/beta-premove.dump + +# on vps2 — restore into beta's database as beta's role +scp /tmp/beta-premove.dump agent@169.58.46.181:/tmp/ +ssh agent@169.58.46.181 ' + DBC=$(docker ps -q --filter label=com.docker.swarm.service.name=thermograph_db | head -1) + docker cp /tmp/beta-premove.dump "$DBC":/tmp/ + docker exec "$DBC" pg_restore -U thermograph -d thermograph_beta --no-owner --role=thermograph_beta /tmp/beta-premove.dump +' +``` + +`--no-owner --role=thermograph_beta` matters: the dump's objects are owned by the +old `thermograph` role, and restoring them verbatim would leave beta's tables +owned by a role beta does not connect as. + +**Do not** restore a *prod* dump into beta. Beta's daemon runs the notification +timers, and beta's subscriber table would then be full of real people. Beta's +vault sets `THERMOGRAPH_MAIL_BACKEND=console` so nothing would actually send, +but that is one `sops edit` away from not being true. + +--- + +## 5. Bring beta's stack up on vps2 + +``` +ssh agent@169.58.46.181 +cd /opt/thermograph-beta +THERMOGRAPH_ENV=beta SERVICE=all \ + BACKEND_IMAGE_TAG= \ + FRONTEND_IMAGE_TAG= \ + infra/deploy/deploy.sh +``` + +Get the current tags from the old beta host first +(`cat /opt/thermograph/infra/deploy/.image-tags.env` on vps1), so beta comes up +on exactly the code it was already running. + +**Verify** — beta answers on its own loopback ports, and prod's are unmoved: + +``` +curl -fsS -o /dev/null -w 'beta %{http_code}\n' http://127.0.0.1:8237/healthz +curl -fsS -o /dev/null -w 'prod %{http_code}\n' http://127.0.0.1:8137/healthz +docker stack services thermograph-beta # 5 services, all 1/1 +docker stack services thermograph # prod: unchanged, still 1/1 +``` + +**Verify the DNS-collision fix actually held** — this is the failure mode that +would be subtle and awful in production: + +``` +# prod's frontend must resolve prod's web, not beta's +docker exec $(docker ps -q -f label=com.docker.swarm.service.name=thermograph_frontend | head -1) \ + getent hosts web +docker exec $(docker ps -q -f label=com.docker.swarm.service.name=thermograph_frontend | head -1) \ + getent hosts beta-web # should NOT resolve on prod's network +``` + +**Rollback:** `docker stack rm thermograph-beta` and remove the LB container +(`docker rm -f thermograph-beta-lb`). Beta on vps1 is still running and still +serving; nothing has moved yet. + +--- + +## 6. Add beta's site block to vps2's Caddy + +`infra/deploy/Caddyfile.vps2` is the reference copy. Append its +`beta.thermograph.org` block to the live `/etc/caddy/Caddyfile` on vps2. + +``` +ssh agent@169.58.46.181 +sudo caddy validate --config /etc/caddy/Caddyfile # BEFORE reloading +sudo systemctl reload caddy +``` + +Validate before reload, always: a malformed Caddyfile takes `thermograph.org` +down with it, and prod is on this box now. + +Caddy cannot issue the certificate until DNS moves (step 7), so expect the beta +hostname to fail TLS until then. That is fine and expected. + +--- + +## 7. Move DNS + +Point `beta.thermograph.org`'s A record at **169.58.46.181**. + +Then wait for propagation and the certificate: + +``` +dig +short beta.thermograph.org # 169.58.46.181 +curl -fsS -o /dev/null -w '%{http_code}\n' https://beta.thermograph.org/api/health +ssh agent@169.58.46.181 'sudo journalctl -u caddy -n 30 --no-pager | grep -i certificate' +``` + +**Rollback:** point the A record back at 75.119.132.91. Beta on vps1 is still +up (you have not stopped it yet — that is step 9), so this is a clean revert +for as long as you leave it running. + +--- + +## 8. Stand dev up on vps1 + +Independent of everything above; do it whenever. + +``` +ssh agent@75.119.132.91 +sudo bash /opt/thermograph-dev/infra/deploy/provision-dev.sh # clones if absent +``` + +The script writes `/etc/thermograph/secrets-env` = `dev` and removes any +`deploy-mode` marker. Then deploy: + +``` +cd /opt/thermograph-dev +SERVICE=all BACKEND_IMAGE_TAG= FRONTEND_IMAGE_TAG= \ + infra/deploy/deploy-dev.sh +``` + +**Verify it is mesh-only.** This is the security-relevant check of the whole +cutover — vps1 is a public box and dev runs unreviewed branches: + +``` +ss -ltnp | grep 8137 # MUST show 10.10.0.2:8137, never 0.0.0.0:8137 +curl -fsS -o /dev/null -w '%{http_code}\n' http://10.10.0.2:8137/healthz # from the mesh +curl --max-time 5 http://75.119.132.91:8137/healthz # MUST fail/refuse +``` + +Also confirm dev did **not** get the fleet's shared production credentials — +`dev.yaml` is rendered alone, and vps1 is exactly the box that must not hold +them: + +``` +sudo grep -c 'THERMOGRAPH_S3_SECRET_KEY\|REGISTRY_TOKEN\|VAPID_PRIVATE' /etc/thermograph.env # expect 0 +``` + +--- + +## 9. Retire beta from vps1 + +Only after beta has been serving from vps2 through step 7 for long enough that +you believe it. + +``` +ssh agent@75.119.132.91 +cd /opt/thermograph/infra +docker compose down # stops the old beta app stack +# remove beta's site block from /etc/caddy/Caddyfile (see Caddyfile.vps1) +sudo caddy validate --config /etc/caddy/Caddyfile && sudo systemctl reload caddy +``` + +**Keep the old `thermograph_pgdata` volume on vps1** until you are certain beta +on vps2 is healthy and backed up. It is the only copy of pre-move beta data. +Delete it deliberately, later, not as part of this runbook. + +Forgejo, Grafana, Loki and the portfolio site all keep running on this box +untouched — do not stop anything else here. + +--- + +## 10. Repoint the Alloy agents + +Both nodes need the new variables; the label model changed (see +`observability/alloy/config.alloy`). + +``` +# vps2 — two environments, so two log volumes and the beta overlay +ssh agent@169.58.46.181 +cd /opt/thermograph/observability/alloy +ALLOY_NODE=vps2 ALLOY_ENV=prod \ +APPLOGS_VOLUME=thermograph_applogs BETA_APPLOGS_VOLUME=thermograph-beta_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d + +# vps1 +ssh agent@75.119.132.91 +cd /opt/thermograph/observability/alloy +ALLOY_NODE=vps1 ALLOY_ENV=dev APPLOGS_VOLUME=thermograph-dev_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + docker compose -f docker-compose.agent.yml up -d +``` + +**Verify each environment is labelled as itself** — the point of the change is +that beta's logs on vps2 are not filed as prod: + +``` +# in Grafana Explore, or via the Loki API: +# {host="beta"} |= "" -> should show beta traffic, from node="vps2" +# {host="prod"} |= "" -> prod only +# {host="dev"} |= "" -> from node="vps1" +# count by (host) (count_over_time({job="docker"}[5m])) -> three values, not one +``` + +If `{host="beta"}` is empty while beta is clearly serving, the container-name +relabel did not match — check `docker ps --format '{{.Names}}'` on vps2 against +the `/thermograph-beta_.*` regex. + +--- + +## 11. Clean up + +- [ ] Delete the old `SSH_*` and `PROD_SSH_*` Forgejo secrets (only now — they + were the rollback path). +- [ ] Run the ops cron by hand and confirm **both** databases are dumped: + `ssh agent@169.58.46.181 'ls ~/thermograph-backups | tail -4'` should show + both a `thermograph-*.dump` and a `thermograph_beta-*.dump`. +- [ ] Confirm the Forgejo backup job still works — it now uses `VPS1_SSH_*`, + and it follows Forgejo (vps1), not beta. +- [ ] Restore the DNS TTL on `beta.thermograph.org`. +- [ ] Desktop: stop the old LAN dev stack and free the box for the AI models. + Leave it joined to the Swarm as a worker; nothing schedules onto it today + (every app service is pinned `node.role == manager`), which is what makes + it safe to also run models there. +- [ ] Update Centralis — see below. + +--- + +## Centralis (outside this repo) + +Centralis' own text still describes the old estate, and it is not in this +repository. These need a separate change in the Centralis repo: + +- The `onboarding` tool's estate paragraph ("Four machines on a WireGuard + mesh... beta (beta.thermograph.org, Forgejo, Grafana + Loki), the operator's + desktop (LAN dev, the main CI runner)"). +- The MCP server's own protocol-level `instructions` string, which carries a + second, separately-worded copy of the same paragraph. +- The `thermograph-orientation` skill's estate table, and `thermograph-ops`' + release-flow diagram ("LAN dev box"), its "ssh -L works for beta and is + impossible for prod" claim (beta is a Swarm overlay now, so it behaves like + prod), and its prod-only Swarm service-name list (beta has + `beta-`-prefixed ones now). +- Tool descriptions that hardcode the old shape: `fleet_status` ("Swarm services + on prod, compose containers on beta/dev"), `estate_status` and `rollback_to` + ("Centralis cannot SSH to the dev desktop"), `logs_query`'s per-environment + service lists (Forgejo/Grafana/Loki are listed under `beta`; they belong under + `dev`/vps1 now), and `secrets_render`'s assumption that a host's own + `secrets-env` marker identifies one environment — vps2 has two. + +--- + +## If it goes wrong + +| symptom | most likely cause | action | +|---|---|---| +| prod 502s after step 5 | beta's stack collided with prod's service DNS | `docker stack rm thermograph-beta`; prod recovers on its own. Check beta's services are `beta-`-prefixed | +| beta 502s, prod fine | beta's LB or stack not up | `docker ps \| grep beta-lb`; `docker stack ps thermograph-beta --no-trunc` | +| beta can't reach the database | role/database missing, or the wrong network | re-run `provision-env-db.sh beta`; confirm beta's tasks are on `thermograph_internal` | +| beta serves prod's data | beta's env file has prod's URL | check `/etc/thermograph-beta.env` names `thermograph_beta` twice; re-render (step 2). **Stop beta until fixed** | +| prod's nightly backup fails | `thermograph_beta` doesn't exist yet | either finish step 3 or revert the ops-cron change; the job fails loudly by design rather than skipping silently | +| dev reachable from the internet | `DEV_BIND_ADDR` not applied | `ss -ltnp \| grep 8137`; redeploy via `deploy-dev.sh`, which sets it from `env-topology.sh` | diff --git a/infra/deploy/backup/README.md b/infra/deploy/backup/README.md index 5bbc23e..1f2e3b3 100644 --- a/infra/deploy/backup/README.md +++ b/infra/deploy/backup/README.md @@ -6,11 +6,21 @@ single VPS. Driven by `.forgejo/workflows/ops-cron.yml` (03:00 UTC daily + ## What is backed up -| Prefix in bucket `era5-thermograph` | Source | Job | Retention off-box | -|---|---|---|---| -| `backups/db/prod/` | prod Postgres/TimescaleDB (`pg_dump --format=custom`) | `backup` (prod, `PROD_SSH_*`) | 30 days | -| `backups/forgejo/db/` | Forgejo Postgres (`forgejo_db`) | `forgejo-backup` (beta, `SSH_*`) | 30 days | -| `backups/forgejo/data/` | Forgejo data volume (repos/LFS/config) | `forgejo-backup` | 30 days | +| Prefix in bucket `era5-thermograph` | Source | Job | Host / secrets | Retention off-box | +|---|---|---|---|---| +| `backups/db/prod/` | prod's `thermograph` database (`pg_dump --format=custom`) | `backup` | vps2, `VPS2_SSH_*` | 30 days | +| `backups/db/beta/` | beta's `thermograph_beta` database, same shared instance | `backup` | vps2, `VPS2_SSH_*` | 30 days | +| `backups/forgejo/db/` | Forgejo Postgres (`forgejo_db`) | `forgejo-backup` | vps1, `VPS1_SSH_*` | 30 days | +| `backups/forgejo/data/` | Forgejo data volume (repos/LFS/config) | `forgejo-backup` | vps1, `VPS1_SSH_*` | 30 days | + +Both application databases live on the ONE shared TimescaleDB instance on +vps2 now (separate roles/databases, not separate boxes) — the `backup` job +dumps each by name so beta isn't silently left uncovered. Forgejo moved to +**vps1** with the vps1/vps2 rename (it used to be co-located with beta on the +box now called vps2's predecessor); its backup targets vps1 accordingly. See +`ops-cron.yml`'s own header comment for the history: an earlier revision keyed +these secrets by environment name rather than host, which is what let "the +prod backup" silently dump the wrong box for a while. Everything is streamed through **`age`** encryption before upload — encrypted to the vault recipient `age1xx4dz…`, so the private key each host already has at @@ -23,9 +33,10 @@ decrypts it. Nothing plaintext leaves the box. (also mirrored into the SOPS vault `deploy/secrets/{prod,beta}.yaml` for host-side use). Contabo needs **path-style** addressing (`force_path_style=true`, `provider=Other`, `region=default`). -- `rclone` + `age` must be installed on prod and beta (one-time: `apt-get install -y - rclone age`). The prod job apt-installs them if missing; the beta user has no sudo, - so they are pre-installed there. +- `rclone` + `age` must be installed on vps1 and vps2 (one-time: `apt-get install -y + rclone age`). The prod/beta job on vps2 apt-installs them if missing; check + whether the Forgejo-backup job's user on vps1 has the sudo to do the same, or + whether they need pre-installing there. ## Restore (DR runbook) @@ -33,17 +44,27 @@ Prereq on the restoring host: `rclone` + `age` + the age key at `/etc/thermograph/age.key` (or the operator key), and rclone configured for the `archive:` remote (env vars `RCLONE_CONFIG_ARCHIVE_*` or `/etc/rclone/rclone.conf`). -### Prod database +### Prod / beta database + +Both restore into the SAME shared instance on vps2 (`` is that +one instance's container/task, resolved the same way `dbq.sh` does) — only +the bucket prefix and the target database/role differ: + ```sh # list available encrypted dumps rclone ls archive:era5-thermograph/backups/db/prod/ +rclone ls archive:era5-thermograph/backups/db/beta/ # download newest, decrypt, restore into a fresh db OBJ=archive:era5-thermograph/backups/db/prod/.dump.age rclone cat "$OBJ" | sudo age -d -i /etc/thermograph/age.key \ | docker exec -i pg_restore -U thermograph -d thermograph --clean --if-exists +# beta: same instance, its own role/database +OBJ=archive:era5-thermograph/backups/db/beta/.dump.age +rclone cat "$OBJ" | sudo age -d -i /etc/thermograph/age.key \ + | docker exec -i pg_restore -U thermograph -d thermograph_beta --clean --if-exists ``` -### Forgejo +### Forgejo (restore on vps1) ```sh # database rclone cat archive:era5-thermograph/backups/forgejo/db/.dump.age \ diff --git a/infra/deploy/db/init/20-tuning.sh b/infra/deploy/db/init/20-tuning.sh index eabc6fe..4791d10 100755 --- a/infra/deploy/db/init/20-tuning.sh +++ b/infra/deploy/db/init/20-tuning.sh @@ -1,7 +1,10 @@ #!/bin/bash # Postgres memory / performance tuning, scaled to the container's DB_MEMORY budget so -# the same init serves every host (beta 8g; prod 16g on the 48 GB box) with no -# hardcoding. Runs once on a fresh data volume from /docker-entrypoint-initdb.d, after +# the same init works for any host with no hardcoding. There is ONE shared TimescaleDB +# instance now (prod and beta are separate databases/roles on it, not separate +# containers), sized from prod's vault values (16g on the 48 GB box) -- beta no longer +# gets a second database or a second tuning pass. Dev keeps its own, separate container +# (8g). Runs once on a fresh data volume from /docker-entrypoint-initdb.d, after # 10-timescaledb.sql enables timescaledb. Settings are written via ALTER SYSTEM # (persisted to postgresql.auto.conf, which the timescaledb image's own # timescaledb-tune postgresql.conf defers to); the container's post-init restart brings diff --git a/infra/deploy/db/provision-env-db.sh b/infra/deploy/db/provision-env-db.sh new file mode 100755 index 0000000..334612c --- /dev/null +++ b/infra/deploy/db/provision-env-db.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Give an environment its own role + database on the SHARED Postgres instance. +# +# sudo bash infra/deploy/db/provision-env-db.sh beta # run on vps2 +# +# Since beta moved onto vps2 there is ONE TimescaleDB instance serving two +# environments. That is a capacity decision — one Postgres to tune, back up and +# keep on one extension build — and it is emphatically NOT a decision to let the +# two environments see each other's data. This script is what makes the second +# half true: +# +# * a role per environment (thermograph_beta), with its own password taken +# from that environment's own vault render, +# * a database per environment (thermograph_beta) OWNED by that role, +# * CONNECT revoked from PUBLIC on it, so the split is enforced by Postgres +# rather than by everyone remembering to use the right URL, +# * and NO superuser, NO CREATEDB, NO CREATEROLE on that role. +# +# The prod role keeps its own database and cannot be reached with beta's +# credentials; beta's role cannot connect to prod's database at all. +# +# Idempotent by construction — safe to re-run after a password rotation (it +# re-applies the password) or on a fresh instance. It never drops anything. +# +# WHY THIS IS NOT WIRED INTO deploy.sh: creating roles and databases is a +# privileged, once-per-environment act, and a deploy that can mint database +# roles is a deploy that can mint them wrongly at 3am. Run it by hand from the +# runbook when an environment is first stood up, and after a password rotation. +set -euo pipefail + +ENV_NAME="${1:?usage: provision-env-db.sh (beta|prod)}" + +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../env-topology.sh" +thermograph_topology "$ENV_NAME" + +if [ "$TG_DEPLOY_MODE" != stack ]; then + echo "!! $ENV_NAME does not use the shared instance (dev keeps its own db container)" >&2 + exit 2 +fi + +# REFUSE to run for the environment that OWNS the instance. +# +# This script provisions a GUEST environment onto someone else's Postgres. Run +# for prod, it would target prod's `thermograph` role — which is the instance's +# bootstrap superuser, not a guest — and the `ALTER ROLE ... NOSUPERUSER` below +# would strip superuser from the role the whole instance is administered with. +# Prod's roles predate this script and are created by the container's own +# initdb; there is nothing here for prod to need. +if [ "$TG_DB_SERVICE" = "${TG_STACK_NAME}_db" ]; then + echo "!! '$ENV_NAME' OWNS this Postgres instance (${TG_DB_SERVICE} is its own stack's db)." >&2 + echo "!! This script provisions a guest environment onto a shared instance." >&2 + echo "!! Running it here would demote ${TG_DB_USER} from superuser. Refusing." >&2 + exit 2 +fi + +# The database SERVER is prod's, wherever we are provisioning FOR. +DB_CID=$(docker ps -q --filter "label=com.docker.swarm.service.name=${TG_DB_SERVICE}" | head -1) +if [ -z "$DB_CID" ]; then + echo "!! no running task for ${TG_DB_SERVICE} on this host" >&2 + echo "!! run this on vps2, with prod's stack up." >&2 + exit 1 +fi + +# The new role's password is whatever that environment's vault says it is, so +# the database and the app can never disagree about it. Read from the rendered +# env file (a deploy of that environment writes it) rather than from sops here: +# this script should not need the age key. +if [ ! -r "$TG_ENV_FILE" ]; then + PW=$(sudo grep -m1 '^POSTGRES_PASSWORD=' "$TG_ENV_FILE" 2>/dev/null | cut -d= -f2- || true) +else + PW=$(grep -m1 '^POSTGRES_PASSWORD=' "$TG_ENV_FILE" | cut -d= -f2- || true) +fi +if [ -z "${PW:-}" ]; then + echo "!! no POSTGRES_PASSWORD in $TG_ENV_FILE" >&2 + echo "!! deploy $ENV_NAME once first so the vault renders it, then re-run." >&2 + exit 1 +fi + +echo "==> Provisioning role/database '${TG_DB_USER}'/'${TG_DB_NAME}' on ${TG_DB_SERVICE}" + +# The bootstrap superuser of the instance is prod's POSTGRES_USER (`thermograph`), +# and psql runs inside the container over its local socket — the database is +# never exposed on a TCP port anyone outside the overlay can reach. +# +# The password reaches psql as an environment variable rather than a -v +# argument, so it is not in psql's argv inside the container. It IS briefly in +# the `docker exec` argv on the host; that is visible only to root on vps2, who +# can read the vault render anyway. It is never interpolated into SQL text: +# :'pw' is psql's quote-and-escape form, which is also what makes a password +# containing a quote safe. +# +# Note both statements below are generated and run via \gexec rather than a +# DO block. psql does NOT substitute :variables inside dollar-quoted strings, +# so a DO $$ ... :'role' ... $$ body would be sent to the server literally. +docker exec -i -e ENV_DB_PASSWORD="$PW" "$DB_CID" \ + psql -v ON_ERROR_STOP=1 -U thermograph -d postgres \ + -v role="$TG_DB_USER" -v dbname="$TG_DB_NAME" <<'SQL' +\set pw `echo "$ENV_DB_PASSWORD"` + +-- Role: create if absent. +SELECT format('CREATE ROLE %I LOGIN', :'role') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'role') +\gexec + +-- Always (re)apply the password and the negative privileges, so a vault +-- rotation is just "rotate, redeploy, re-run this" — and so a role that was +-- created by hand with more rights than it should have gets corrected. +ALTER ROLE :"role" WITH LOGIN PASSWORD :'pw' NOSUPERUSER NOCREATEDB NOCREATEROLE; + +-- CREATE DATABASE cannot run inside a transaction block, hence \gexec here too. +SELECT format('CREATE DATABASE %I OWNER %I', :'dbname', :'role') +WHERE NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = :'dbname') +\gexec +SQL + +# Extension + privileges have to run INSIDE the new database, hence a second +# connection. CREATE EXTENSION needs superuser, which is why it is done here +# rather than left to the app's own migration to attempt as the env role. +docker exec -i "$DB_CID" \ + psql -v ON_ERROR_STOP=1 -U thermograph -d "$TG_DB_NAME" \ + -v role="$TG_DB_USER" -v dbname="$TG_DB_NAME" -v rorole="${TG_DB_USER}_ro" <<'SQL' +CREATE EXTENSION IF NOT EXISTS timescaledb; + +-- Nobody but this environment's roles connects to this database. Without this, +-- PUBLIC retains CONNECT and any role on the instance could read it. +REVOKE CONNECT ON DATABASE :"dbname" FROM PUBLIC; +GRANT CONNECT ON DATABASE :"dbname" TO :"role"; + +-- The role owns the database but not necessarily the public schema, which is +-- owned by the bootstrap superuser on a fresh database; Alembic needs to create +-- tables in it. +ALTER SCHEMA public OWNER TO :"role"; + +-- The read-only role that ad-hoc queries use (ops/dbq.sh). Every environment +-- gets one, named _ro, so a human or an agent poking at data cannot +-- write. Without it, beta's queries would have had to connect as the OWNER — +-- read-write on its own database — losing a guarantee prod and dev already had +-- purely because beta was newer. +SELECT format('CREATE ROLE %I LOGIN', :'rorole') +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'rorole') +\gexec +ALTER ROLE :"rorole" WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE; + +GRANT CONNECT ON DATABASE :"dbname" TO :"rorole"; +GRANT USAGE ON SCHEMA public TO :"rorole"; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO :"rorole"; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO :"rorole"; +-- Future tables too: without this, every new migration would create a table the +-- read-only role cannot see, and the omission would only surface as a confusing +-- "permission denied" months later. +ALTER DEFAULT PRIVILEGES FOR ROLE :"role" IN SCHEMA public + GRANT SELECT ON TABLES TO :"rorole"; +ALTER DEFAULT PRIVILEGES FOR ROLE :"role" IN SCHEMA public + GRANT SELECT ON SEQUENCES TO :"rorole"; +SQL + +echo "==> OK: ${TG_DB_USER} owns ${TG_DB_NAME} (timescaledb enabled, PUBLIC revoked)" +echo " Verify isolation:" +echo " docker exec $DB_CID psql -U ${TG_DB_USER} -d thermograph -c 'select 1' # must FAIL" diff --git a/infra/deploy/deploy-dev.sh b/infra/deploy/deploy-dev.sh index 8149d8b..bf09ac1 100755 --- a/infra/deploy/deploy-dev.sh +++ b/infra/deploy/deploy-dev.sh @@ -1,41 +1,54 @@ #!/usr/bin/env bash -# LAN-dev deploy: roll the per-service registry-pull stack (see deploy.sh) onto -# the ~/thermograph-dev overlay instead of prod/beta's loopback-only stack. +# Dev deploy: roll the per-service registry-pull stack (see deploy.sh) onto the +# dev compose overlay instead of beta/prod's Swarm stacks. # -# This script assumes $APP_DIR is a checkout of the monorepo (deploy assets under infra/), -# the same way /opt/thermograph is on prod/beta. That is the live state: the LAN -# box's ~/thermograph-dev was reprovisioned as an infra checkout during the -# 2026-07-22 cutover (the app monorepo is archived), provision-dev-lan.sh's -# REPO_URL defaults to this repo, and the app repos' deploy-dev.yml workflows -# invoke this script on the thermograph-lan runner. This IS the live LAN-dev -# deploy path. +# Dev lives on vps1 now, at /opt/thermograph-dev — not on the operator's desktop. +# The desktop hosts no Thermograph environment any more; it runs the AI models +# and offers flex Swarm capacity. What moved is only WHERE and HOW dev is +# reached: it is a normal fleet host now (SSH deploy from CI, SOPS render at +# /etc/thermograph.env, mesh-only exposure on 10.10.0.2:8137), rather than a +# sudo-free systemd --user stack on someone's Wi-Fi. +# +# $APP_DIR is a checkout of the monorepo (deploy assets under infra/), the same +# way /opt/thermograph is for prod. A laptop can still point APP_DIR at a +# checkout under $HOME and get the old local behaviour. # # Design: a thin wrapper around deploy.sh, not a duplicate. deploy.sh already owns # the entire per-service registry-pull mechanism -- secrets sourcing, docker login, # the retry-pull loop, --no-deps single-service rolls vs. --remove-orphans `all`, # .image-tags.env persistence, and the 8137/8080 health checks. None of that is -# dev-specific; the only things LAN dev actually changes are WHERE it deploys -# (a separate checkout + branch) and WHICH compose files are in play (the base -# file plus docker-compose.dev.yml's uncapped/LAN-published overrides). Both are -# expressible as environment (APP_DIR/BRANCH that deploy.sh already reads, and -# docker compose's own COMPOSE_FILE variable), so re-exec'ing deploy.sh with that -# environment set covers it with no forked copy of the pull/roll/health logic to -# drift out of sync. If LAN dev ever needs deploy logic that genuinely diverges -# from prod/beta (not just "different files/directory"), promote this to a -# standalone script at that point rather than growing special cases into deploy.sh. +# dev-specific; the only things dev actually changes are WHERE it deploys (a +# separate checkout + branch), WHICH compose files are in play (the base file +# plus docker-compose.dev.yml's uncapped/mesh-published overrides), and its +# secrets policy (below). All are expressible as environment (APP_DIR/BRANCH +# that deploy.sh already reads, and docker compose's own COMPOSE_FILE +# variable), so re-exec'ing deploy.sh with that environment set covers it with +# no forked copy of the pull/roll/health logic to drift out of sync. If dev ever +# needs deploy logic that genuinely diverges from beta/prod (not just "different +# files/directory"), promote this to a standalone script at that point rather +# than growing special cases into deploy.sh. # # Usage, mirrors deploy.sh directly: # # roll just the backend onto a dev-tagged image: -# ssh dev-box 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> deploy/deploy-dev.sh' +# ssh vps1 'SERVICE=backend BACKEND_IMAGE_TAG=sha-<12hex> /opt/thermograph-dev/infra/deploy/deploy-dev.sh' # # bring the whole dev stack up (both tags required, same as deploy.sh): -# ssh dev-box 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- deploy/deploy-dev.sh' +# ssh vps1 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- /opt/thermograph-dev/infra/deploy/deploy-dev.sh' set -euo pipefail -# Dev context: a separate checkout + branch from prod/beta's /opt/thermograph -# (main), so a dev deploy never touches or is touched by the prod/beta one. -APP_DIR="${APP_DIR:-$HOME/thermograph-dev}" -BRANCH="${BRANCH:-dev}" +# Dev context: its own checkout and its own branch, on its own host (vps1). +# Both come from deploy/env-topology.sh so there is one place that says where +# dev lives. $APP_DIR is still honoured when set explicitly — that is how a +# laptop points this at a checkout under $HOME. +_dev_self="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=infra/deploy/env-topology.sh +. "$_dev_self/env-topology.sh" +thermograph_topology dev +APP_DIR="${APP_DIR:-$TG_APP_DIR}" +BRANCH="${BRANCH:-$TG_BRANCH}" export APP_DIR BRANCH +# deploy.sh resolves the environment itself; say so explicitly rather than +# letting it fall back to a host marker that, on vps1, names dev anyway. +export THERMOGRAPH_ENV=dev # Point every `docker compose` invocation inside deploy.sh at the LAN-dev overlay # (uncapped CPU, backend published on 0.0.0.0:8137, frontend unpublished -- see @@ -72,23 +85,33 @@ export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-thermograph-dev}" # genuinely needs, and nothing else. export THERMOGRAPH_SECRETS_SKIP_COMMON=1 -# (1b) Docker on this box is the snap package, confined to $HOME (plus a short -# allowlist) -- it silently can't see /etc/thermograph.env at all, so -# env_file: /etc/thermograph.env resolves to nothing for every container no -# matter how correctly it's rendered. Mirror the render to a path under -# $APP_DIR too; docker-compose.dev.yml points env_file at this copy instead of -# /etc/thermograph.env. Untracked (see infra/.gitignore); re-rendered on every -# deploy, never committed. -export THERMOGRAPH_SECRETS_ENV_FILE_MIRROR="$APP_DIR/infra/deploy/dev-secrets.env" +# (1b) Snap-packaged Docker is confined to $HOME (plus a short allowlist) and +# silently can't see /etc/thermograph.env at all, so env_file: /etc/thermograph.env +# resolves to nothing for every container no matter how correctly it's rendered. +# Where that is the case, mirror the render to a path under $APP_DIR; +# docker-compose.dev.yml carries a matching optional env_file entry. +# +# CONDITIONAL, not unconditional: this writes a plaintext copy of the render +# into the checkout, and on vps1 (ordinary apt Docker, which reads /etc fine) +# that copy would be pure liability. Detect the confinement rather than add a +# knob someone has to remember — the docker binary resolving under /snap is +# exactly the condition that breaks the /etc path. +_docker_bin="$(command -v docker || true)" +if [ -n "$_docker_bin" ] && case "$(readlink -f "$_docker_bin")" in /snap/*) true ;; *) false ;; esac; then + export THERMOGRAPH_SECRETS_ENV_FILE_MIRROR="$APP_DIR/infra/deploy/dev-secrets.env" +fi # (2) The age key is read from wherever it is READABLE, not necessarily # /etc/thermograph/age.key. render-secrets.sh falls back to `sudo cat` for a -# root-owned 0400 key, and this box has no passwordless sudo -- under the Forgejo -# runner (a systemd --user service, no tty) that sudo cannot prompt, so the -# fleet-standard placement would make every dev deploy fail at the render. Prefer -# the standard path when it is readable; otherwise the operator's own keyring, which -# is where dev's key already lives (this box is the machine `sops` is run on), so -# provisioning dev needs no second copy of the estate's recovery root. +# root-owned 0400 key, which needs passwordless sudo. +# +# On vps1 that is the normal case: the `agent` user has passwordless sudo and +# the key sits at the fleet-standard path, so this block does nothing. The +# fallback exists for the laptop case — a checkout where the key lives only in +# the operator's own keyring (that machine is where `sops` is run), and where a +# non-interactive shell could not prompt for sudo even if the key were in /etc. +# Preferring the standard path when readable keeps vps1 on the fleet convention +# and means provisioning dev needs no second copy of the estate's recovery root. if [ -z "${THERMOGRAPH_AGE_KEY:-}" ] && [ ! -r /etc/thermograph/age.key ] \ && [ -r "$HOME/.config/sops/age/keys.txt" ]; then export THERMOGRAPH_AGE_KEY="$HOME/.config/sops/age/keys.txt" diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index 51748eb..98fd9de 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -26,11 +26,59 @@ # two axes are independent and rarely change together. set -euo pipefail -APP_DIR="${APP_DIR:-/opt/thermograph}" +# --- which environment is this? ------------------------------------------------ +# vps2 runs beta AND prod, so "the host" no longer answers this — the caller does, +# via THERMOGRAPH_ENV (the deploy workflow always passes it). A by-hand run on a +# single-environment box still falls back to the host marker, and a box with +# neither still defaults to prod's historical paths, so nothing about an existing +# single-env host changes. +# +# Resolved from the SCRIPT'S OWN LOCATION, not a hardcoded /opt/thermograph: +# invoking /opt/thermograph-beta/infra/deploy/deploy.sh must act on the beta +# checkout even if something in the environment says otherwise. That is also the +# check that catches the one genuinely dangerous mistake on a two-environment +# host — running prod's checkout with THERMOGRAPH_ENV=beta, or the reverse. +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +SELF_APP_DIR=$(cd "$SELF_DIR/../.." && pwd) + +# Guarded exactly like render-secrets.sh below: the deploy that INTRODUCES this +# file runs with a checkout that predates it (it arrives with the git reset +# further down, after which deploy.sh re-execs). Missing => keep the pre-split +# behaviour rather than fail. +if [ -f "$SELF_DIR/env-topology.sh" ]; then + # shellcheck source=infra/deploy/env-topology.sh + . "$SELF_DIR/env-topology.sh" + ENV_NAME=$(thermograph_env_name) + # Nothing to go on anywhere: this is a pre-split host whose paths are prod's. + [ -n "$ENV_NAME" ] || ENV_NAME=prod + thermograph_topology "$ENV_NAME" + if [ "$SELF_APP_DIR" != "$TG_APP_DIR" ] && [ -z "${APP_DIR:-}" ]; then + echo "!! environment/checkout mismatch: THERMOGRAPH_ENV=$ENV_NAME expects" >&2 + echo "!! $TG_APP_DIR but this script lives in $SELF_APP_DIR." >&2 + echo "!! On vps2 that means beta and prod have been crossed. Refusing to deploy." >&2 + echo "!! If this checkout really is $ENV_NAME (a rehearsal copy, a relocated" >&2 + echo "!! checkout), say so explicitly: APP_DIR=$SELF_APP_DIR ..." >&2 + exit 2 + fi + APP_DIR="${APP_DIR:-$TG_APP_DIR}" + BRANCH="${BRANCH:-$TG_BRANCH}" + ENV_FILE="$TG_ENV_FILE" + DEPLOY_MODE="$TG_DEPLOY_MODE" + [ "$TG_SKIP_COMMON" = 1 ] && export THERMOGRAPH_SECRETS_SKIP_COMMON=1 + # The second pass (after the re-exec) must resolve the SAME environment, and + # the stack path needs it too. + export THERMOGRAPH_ENV="$ENV_NAME" +else + ENV_NAME="" + APP_DIR="${APP_DIR:-/opt/thermograph}" + BRANCH="${BRANCH:-main}" + ENV_FILE=/etc/thermograph.env + DEPLOY_MODE="" +fi + # Monorepo layout: git operations act on the checkout root ($APP_DIR); all # compose files and deploy assets live under infra/. INFRA_DIR="$APP_DIR/infra" -BRANCH="${BRANCH:-main}" # Which service this deploy rolls: backend | frontend | all. Defaults to `all` # (a full-stack bring-up) so a by-hand run with both tags still works; the # per-repo deploy.yml workflows always pass an explicit single service. @@ -69,16 +117,20 @@ fi # after which deploy.sh re-execs), so a missing helper simply falls back to the # existing /etc/thermograph.env. Then source it so a by-hand run interpolates the # same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/. +# +# $ENV_FILE, not a hardcoded /etc/thermograph.env: on vps2 prod renders +# prod.yaml there while beta renders beta.yaml to /etc/thermograph-beta.env. +# One file per environment, never shared. if [ -f "$INFRA_DIR/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$INFRA_DIR/deploy/render-secrets.sh" - render_thermograph_secrets "$INFRA_DIR" + render_thermograph_secrets "$INFRA_DIR" "$ENV_NAME" "$ENV_FILE" fi -# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it -# cannot exist at lint time, so don't ask shellcheck to follow it. +# $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at +# lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null -. /etc/thermograph.env 2>/dev/null || true +. "$ENV_FILE" 2>/dev/null || true set +a # Pre-warm the ~750 city-page archives so /climate pages serve from cache and a @@ -122,12 +174,18 @@ if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then exec "$0" "$@" fi -# Stack-mode routing: a host whose /etc/thermograph/deploy-mode says "stack" -# (prod, after the Swarm cutover) deploys via the Swarm stack path instead of -# compose. Checked AFTER the reset+re-exec so the stack script is always the -# freshly-pulled one, and the SERVICE/tag contract passes through unchanged -- -# the app repos' workflows never need to know which mode a host runs. -if [ "$(cat /etc/thermograph/deploy-mode 2>/dev/null || true)" = "stack" ]; then +# Stack-mode routing: prod and beta are both Swarm stacks, dev is compose. The +# mode is a property of the ENVIRONMENT (env-topology.sh), not of the host -- +# vps2 runs two stacks, and a host-wide /etc/thermograph/deploy-mode marker +# cannot describe a host that runs more than one environment. The marker is +# still honoured when the topology file is absent (a checkout that predates it). +# Checked AFTER the reset+re-exec so the stack script is always the freshly +# pulled one, and the SERVICE/tag contract passes through unchanged -- the +# deploy workflow never needs to know which mode an environment runs. +if [ -z "$DEPLOY_MODE" ]; then + DEPLOY_MODE=$(cat /etc/thermograph/deploy-mode 2>/dev/null || true) +fi +if [ "$DEPLOY_MODE" = "stack" ]; then exec bash "$INFRA_DIR/deploy/stack/deploy-stack.sh" fi @@ -137,6 +195,20 @@ fi # rename the project and recreate the stack under a second name. cd "$INFRA_DIR" +# Compose-mode environments (dev) carry their project name, file list and bind +# address in the topology table. Set only when not already in the environment, +# so deploy-dev.sh's own exports and a by-hand override both still win. +if [ -n "${TG_COMPOSE_PROJECT:-}" ]; then + export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-$TG_COMPOSE_PROJECT}" + export COMPOSE_FILE="${COMPOSE_FILE:-$TG_COMPOSE_FILE}" +fi +# Which address the dev overlay publishes on. Defaults to loopback in the +# compose file; dev on vps1 sets the mesh address here, because a public VPS +# must never publish an unreviewed branch's stack on 0.0.0.0. +if [ -n "${TG_BIND_ADDR:-}" ]; then + export DEV_BIND_ADDR="${DEV_BIND_ADDR:-$TG_BIND_ADDR}" +fi + # Registry-pull cutover: pull the image each app repo's build-push.yml already # built and pushed, instead of building in place. This checkout is # thermograph-infra, not an app repo, so there's no "current commit" to derive diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh new file mode 100644 index 0000000..77b6c82 --- /dev/null +++ b/infra/deploy/env-topology.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# WHERE EACH ENVIRONMENT LIVES — the single source of truth, sourced by every +# deploy path. Not executable on its own; `. env-topology.sh` then call +# `thermograph_topology `. +# +# Why this file exists at all. Until the vps1/vps2 split, "which environment is +# this?" was answerable from the machine you were standing on: one host ran one +# environment, so a host-wide marker (/etc/thermograph/secrets-env) and a +# host-wide deploy mode (/etc/thermograph/deploy-mode) were enough, and every +# path could hardcode /opt/thermograph, /etc/thermograph.env, ports 8137/8080. +# vps2 now runs BOTH beta and prod. Every one of those assumptions becomes a +# collision: two checkouts, two rendered env files, two stacks, two loopback +# port pairs, two image-tag files, two deploy locks — on one box. So the +# environment is now an INPUT (THERMOGRAPH_ENV, passed by the deploy caller), +# and everything else is derived here rather than re-guessed per script. +# +# The host markers survive as the fallback for a by-hand run with no explicit +# env (`deploy.sh` on prod still means prod), but they are no longer the thing +# that decides where files go. +# +# THE ESTATE +# +# vps1 75.119.132.91 10.10.0.2 "operational programs" +# Forgejo (git + CI + registry), Grafana/Loki/Alloy, emigriffith.dev, +# and the DEV environment with its own Postgres. Nothing here is +# reachable by an external user except git/dashboard/the portfolio site; +# dev itself is mesh-only, deliberately (see DEV_BIND_ADDR below). +# +# vps2 169.58.46.181 10.10.0.1 "the deployed environment" +# Everything an external user can touch: prod (thermograph.org) AND beta +# (beta.thermograph.org), Centralis, Postfix, the backups, and the ONE +# TimescaleDB instance that serves both environments on separate +# databases with separate roles. +# +# desktop 10.10.0.3 operator's box +# AI model hosting (voice-to-text, the LLM behind upcoming features) and +# flex capacity as a Swarm worker. It hosts NO Thermograph environment — +# that is the whole point of the vps1/vps2 split. A laptop-style +# `make dev-up` still works locally; it is just not the dev server. +# +# WHY BETA SITS NEXT TO PROD RATHER THAN NEXT TO DEV +# +# Beta is a pre-production rehearsal, so what it needs to rehearse is prod's +# environment: the same orchestrator (Swarm, not compose), the same Postgres +# major and extension build, the same Caddy in front, the same mail path, the +# same mesh position. Co-locating beta with dev bought resemblance to the thing +# it is NOT trying to predict. Co-locating it with prod means a beta green light +# is evidence about prod. The cost — a bad beta deploy shares a machine with +# prod — is bounded by the per-environment isolation this file defines: separate +# stacks, separate volumes, separate DB roles, separate CPU limits. + +# thermograph_topology +# +# Exports the TG_* variables describing that environment. Every value is a +# derived fact about the estate, not a preference: change one here and the +# deploy scripts, the stack files and the runbooks all follow. +thermograph_topology() { + local env_name="${1:?thermograph_topology needs an environment: dev|beta|prod}" + + # Reset first: this function is sourced into long-lived shells (deploy.sh + # sources it before and after its self re-exec), and a stale TG_STACK_NAME + # from a previous call would silently target the wrong stack. + TG_ENV=""; TG_HOST=""; TG_APP_DIR=""; TG_BRANCH=""; TG_DEPLOY_MODE="" + TG_STACK_NAME=""; TG_STACK_FILE=""; TG_LB_CONFIG="" + TG_COMPOSE_PROJECT=""; TG_COMPOSE_FILE="" + TG_ENV_FILE=""; TG_STACK_ENV_FILE=""; TG_LB_NAME="" + TG_LB_HTTP_PORT=""; TG_LB_FE_PORT=""; TG_DB_NAME=""; TG_DB_USER="" + TG_TAGS_FILE=""; TG_LOCK_FILE=""; TG_BIND_ADDR=""; TG_SKIP_COMMON=0 + TG_SVC_PREFIX=""; TG_DATA_NETWORK=""; TG_DB_SERVICE=""; TG_POST_DEPLOY=0 + TG_SSH_HOST=""; TG_SSH_TARGET="" + + case "$env_name" in + prod) + TG_ENV=prod + TG_HOST=vps2 + # Unchanged from before the split — prod keeps every path it already has, + # so nothing about the live prod host moves during the cutover. + TG_SSH_HOST=169.58.46.181 + TG_SSH_TARGET=agent@169.58.46.181 + TG_APP_DIR=/opt/thermograph + TG_BRANCH=main + TG_DEPLOY_MODE=stack + TG_STACK_NAME=thermograph + TG_STACK_FILE=deploy/stack/thermograph-stack.yml + TG_ENV_FILE=/etc/thermograph.env + TG_STACK_ENV_FILE=/etc/thermograph/stack.env + TG_LB_NAME=thermograph-lb + TG_LB_CONFIG=deploy/stack/lb/Caddyfile + TG_LB_HTTP_PORT=8137 + TG_LB_FE_PORT=8080 + TG_DB_NAME=thermograph + TG_DB_USER=thermograph + # Prod's services keep their bare names (web, worker, frontend, ...): the + # cutover must not touch prod's stack file, its env or its LB config. + TG_SVC_PREFIX="" + # Where the database lives. Prod's own overlay, which is `attachable` and + # which beta joins as an external network. + TG_DATA_NETWORK=thermograph_internal + TG_DB_SERVICE=thermograph_db + # Post-deploy city warm + IndexNow ping. Prod only — see the beta note. + TG_POST_DEPLOY=1 + ;; + beta) + TG_ENV=beta + TG_HOST=vps2 + # Same box, same public IP as prod — beta and prod are two stacks on one + # Swarm manager now, not two separate hosts. A beta SSH target is + # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh). + TG_SSH_HOST=169.58.46.181 + TG_SSH_TARGET=agent@169.58.46.181 + # A SECOND checkout on the same box. Separate from prod's so the two can + # sit on different commits of this repo, and so `git reset --hard` in one + # deploy can never yank the tree out from under the other's running + # deploy. Everything below is likewise a distinct name/port/path from + # prod's, on purpose: co-location is only safe if nothing is shared by + # accident. The one deliberate exception is the database SERVER (see + # TG_DB_* — separate role and database on a shared instance). + TG_APP_DIR=/opt/thermograph-beta + TG_BRANCH=main + TG_DEPLOY_MODE=stack + TG_STACK_NAME=thermograph-beta + TG_STACK_FILE=deploy/stack/thermograph-beta-stack.yml + TG_ENV_FILE=/etc/thermograph-beta.env + TG_STACK_ENV_FILE=/etc/thermograph/beta-stack.env + TG_LB_NAME=thermograph-beta-lb + TG_LB_CONFIG=deploy/stack/lb/Caddyfile.beta + # NOT 8137/8080: prod's loopback LB already owns those on this host. + TG_LB_HTTP_PORT=8237 + TG_LB_FE_PORT=8180 + TG_DB_NAME=thermograph_beta + # Its own role, not prod's `thermograph` superuser-of-its-own-database. + # One instance is a capacity decision; it is not a decision to let a beta + # deploy running an unmerged branch read or write the production database. + TG_DB_USER=thermograph_beta + # Beta's Swarm services are named beta-web, beta-worker, ... Swarm + # registers a service's SHORT name as a DNS alias on every network it + # joins, so two stacks that both call a service `web` on one shared + # network make `web` ambiguous — prod's frontend could resolve a beta + # task. Prefixing beta's names removes the collision without editing a + # single line of prod's stack. + TG_SVC_PREFIX="beta-" + # Beta has no db service of its own: it joins prod's overlay (declared + # `external` in its stack file) purely to reach `db`, and keeps its own + # `internal` overlay for beta-to-beta traffic. + TG_DATA_NETWORK=thermograph_internal + TG_DB_SERVICE=thermograph_db + # No warm, no IndexNow on beta. IndexNow announces URLs to Bing/DDG/ + # Yandex — from beta that means asking search engines to index + # beta.thermograph.org, which is the opposite of what beta is for. The + # city warm is worse than useless here: it spends the shared upstream + # archive quota to fill a cache that only a rehearsal environment reads. + # (Both ran on beta under compose; that was a side effect of beta and + # prod sharing one script, not a decision.) + TG_POST_DEPLOY=0 + ;; + dev) + TG_ENV=dev + TG_HOST=vps1 + # vps1 — Forgejo/Grafana/dev, mesh-only for dev itself (see TG_BIND_ADDR). + TG_SSH_HOST=75.119.132.91 + TG_SSH_TARGET=agent@75.119.132.91 + TG_APP_DIR=/opt/thermograph-dev + # The only environment that tracks `dev`; beta and prod both run this + # repo's `main` (infra is not environment-staged — app code is, via image + # tags). See the branch model in infra/README.md. + TG_BRANCH=dev + # Compose, not Swarm: dev's value is a fast, legible, single-box stack + # you can `docker compose logs` at. It is the one environment not + # pretending to be prod. + TG_DEPLOY_MODE=compose + TG_COMPOSE_PROJECT=thermograph-dev + TG_COMPOSE_FILE="docker-compose.yml:docker-compose.dev.yml" + TG_ENV_FILE=/etc/thermograph.env + # Dev keeps its OWN Postgres container (a `db` service in its compose + # project). It is not on the shared instance and must not be: the shared + # instance lives on vps2 and holds real user data. + TG_DB_NAME=thermograph + TG_DB_USER=thermograph + # Mesh-only. vps1 is a public VPS, so the dev overlay must not publish on + # 0.0.0.0 the way it did on the operator's LAN box — dev runs whatever + # branch is in flight, including unreviewed ones. Reachable at + # http://10.10.0.2:8137 from anything on the WireGuard mesh, and from + # nowhere else. There is deliberately no dev DNS record and no Caddy site. + TG_BIND_ADDR=10.10.0.2 + # Dev NEVER layers common.yaml — that file is the internet-facing fleet's + # shared production credential set (both S3 keypairs, the VAPID signing + # key, REGISTRY_TOKEN, the metrics token), and dev runs whatever branch is + # in flight on the same box as Forgejo and its CI runner. This lives here + # rather than only in deploy-dev.sh so the protection is a property of the + # ENVIRONMENT, not of which entry point happened to be used. See the long + # note in render-secrets.sh. + TG_SKIP_COMMON=1 + TG_DB_SERVICE=db + # Dev warms nothing and pings nothing: it must never spend the upstream + # archive quota, and it must never tell a search engine it exists. + TG_POST_DEPLOY=0 + ;; + *) + echo "!! unknown environment '$env_name' (expected dev|beta|prod)" >&2 + return 2 + ;; + esac + + # Derived, never hand-set: keeping these next to the definitions above is what + # stops a second environment on one host from sharing a lock or a tag file. + TG_LOCK_FILE="$TG_APP_DIR/infra/deploy/.deploy.lock" + if [ "$TG_DEPLOY_MODE" = stack ]; then + TG_TAGS_FILE="$TG_APP_DIR/infra/deploy/.stack-image-tags.env" + else + TG_TAGS_FILE="$TG_APP_DIR/infra/deploy/.image-tags.env" + fi + + export TG_ENV TG_HOST TG_APP_DIR TG_BRANCH TG_DEPLOY_MODE + export TG_STACK_NAME TG_STACK_FILE TG_LB_CONFIG TG_COMPOSE_PROJECT TG_COMPOSE_FILE + export TG_ENV_FILE TG_STACK_ENV_FILE TG_LB_NAME + export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER + export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON + export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY + export TG_SSH_HOST TG_SSH_TARGET +} + +# thermograph_env_name +# +# Which environment is this run for? Explicit input first (the deploy workflows +# pass THERMOGRAPH_ENV, and on vps2 it is the ONLY thing distinguishing a beta +# deploy from a prod one), then the host marker for a by-hand run on a +# single-environment box. Empty output means "cannot tell" and the caller must +# fail rather than guess — guessing wrong on vps2 means deploying beta's image +# tag onto prod. +thermograph_env_name() { + local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" + if [ -n "${THERMOGRAPH_ENV:-}" ]; then + printf '%s\n' "$THERMOGRAPH_ENV" + return 0 + fi + cat "$marker" 2>/dev/null || true +} diff --git a/infra/deploy/forgejo/README.md b/infra/deploy/forgejo/README.md index ab7315d..f15711f 100644 --- a/infra/deploy/forgejo/README.md +++ b/infra/deploy/forgejo/README.md @@ -1,22 +1,24 @@ # Forgejo on the Swarm cluster Runs as `deploy/forgejo/docker-stack.yml` — the only Swarm-scheduled workload -this cluster carries (the Thermograph app itself stays on the -Terraform-managed `docker compose` deploys; see `terraform/README.md`). -Pinned to the **beta** node (old VPS) via the `role=forge` label from +this cluster carries (prod and beta's app stacks are separate `docker stack +deploy`s that happen to run on the manager node, vps2 — see +`deploy/stack/README` context in `DEPLOY.md`). Pinned to the **vps1** node +(`75.119.132.91`) via the `role=forge` label from `deploy/swarm/label-forge-node.sh`. -The Actions **runner** is deliberately *not* part of this stack — it runs on -the **desktop** as a plain systemd service (`register-lan-runner.sh` below), -per `thermograph-docs/runbooks/implementation-handoff.md` Track B step 5. That's the -canonical placement; an earlier revision of this stack ran the runner as a -Swarm-scheduled Docker-in-Docker sidecar pinned to beta, which is gone now. +The Actions **runner** is deliberately *not* part of this stack — see +`DEPLOY-DEV.md` and the note in `register-lan-runner.sh`'s own header for +where it runs today. That's the canonical placement per +`thermograph-docs/runbooks/implementation-handoff.md` Track B step 5; an +earlier revision of this stack ran the runner as a Swarm-scheduled +Docker-in-Docker sidecar pinned to the Forgejo node, which is gone now. ## Prerequisites -1. All three nodes have joined the swarm (`deploy/swarm/`) and beta is +1. All three nodes have joined the swarm (`deploy/swarm/`) and vps1 is labeled `role=forge`. -2. `docker node ls` (from the manager) shows all three `Ready`. +2. `docker node ls` (from the manager, vps2) shows all three `Ready`. ## One-time setup: Swarm secret @@ -43,27 +45,27 @@ its first health check just fails harmlessly until it is. This stack has **no auto-deploy trigger** — nothing in `.forgejo/workflows/` redeploys it on push. A change to `docker-stack.yml` only takes effect once -someone re-runs `docker stack deploy` by hand on the manager (prod). +someone re-runs `docker stack deploy` by hand on the manager (vps2). `db`/`forgejo` both carry `resources.limits` (defaults: db 1 CPU/1g, forgejo 2 CPU/2g — several times observed steady-state usage), overridable with `FORGEJO_DB_CPUS`/`FORGEJO_DB_MEMORY`/`FORGEJO_CPUS`/`FORGEJO_MEMORY` env vars before `docker stack deploy`, same convention as the app stack. -## DNS + TLS: reusing beta's existing Caddy, not a second reverse proxy +## DNS + TLS: reusing vps1's existing Caddy, not a second reverse proxy -Forgejo is pinned to beta (`role=forge`) — but beta is also **today's live -thermograph.org host**, and its Caddy already owns ports 80/443 -(`/etc/caddy/Caddyfile` on that box). A second ingress (Traefik) trying to -bind the same ports would collide with it. So there's no Traefik in this -stack: `forgejo`'s web port publishes to `127.0.0.1:3080` only (host-local), -and beta's *existing* Caddy gets one more site block reverse-proxying to it — -same pattern as its `thermograph.org` block, same automatic-HTTPS. +Forgejo is pinned to vps1 (`role=forge`) — the same box that also runs +Grafana/Loki/Alloy and the `emigriffith.dev` portfolio site, each fronted by +that host's own Caddy. A second ingress (Traefik) trying to bind the same +ports would collide with it. So there's no Traefik in this stack: `forgejo`'s +web port publishes to `127.0.0.1:3080` only (host-local), and vps1's +*existing* Caddy gets one more site block reverse-proxying to it — same +pattern as its other site blocks, same automatic-HTTPS. 1. Point the Forgejo domain (default `git.thermograph.org`; override with - `FORGEJO_DOMAIN=...` before `docker stack deploy`) at **beta's** public IP - — that's where the task actually runs, not prod's or the desktop's. -2. Append `deploy/forgejo/caddy-git.conf` to beta's `/etc/caddy/Caddyfile`, + `FORGEJO_DOMAIN=...` before `docker stack deploy`) at **vps1's** public IP + — that's where the task actually runs, not vps2's or the desktop's. +2. Append `deploy/forgejo/caddy-git.conf` to vps1's `/etc/caddy/Caddyfile`, adjusting the domain if you didn't use the default, then `systemctl reload caddy`. 3. That file also resolves the registry-exposure hazard (#15 in @@ -75,28 +77,28 @@ same pattern as its `thermograph.org` block, same automatic-HTTPS. ## Registry access from mesh clients -Any node that needs `docker login`/push/pull against the registry (the -desktop's CI runner building/pushing images, later any Swarm node pulling -them) must reach `git.thermograph.org` **over the WireGuard tunnel**, not -beta's public IP — otherwise Caddy's `/v2/*` block above refuses the -connection. Public DNS resolves the domain to beta's public IP, so add a -`/etc/hosts` override on each such node pinning it to beta's WireGuard -address instead: +Any node that needs `docker login`/push/pull against the registry (the CI +runner building/pushing images, any Swarm node pulling them, prod or beta on +vps2 pulling app images) must reach `git.thermograph.org` **over the +WireGuard tunnel**, not vps1's public IP — otherwise Caddy's `/v2/*` block +above refuses the connection. Public DNS resolves the domain to vps1's public +IP, so add a `/etc/hosts` override on each such node pinning it to vps1's +WireGuard address instead: ``` echo "10.10.0.2 git.thermograph.org" | sudo tee -a /etc/hosts ``` -(`10.10.0.2` is beta's WG address per `deploy/swarm/README.md`'s peer +(`10.10.0.2` is vps1's WG address per `deploy/swarm/README.md`'s peer numbering — adjust if you assigned it differently.) The git/web UI keeps working normally for everyone else since only `/v2/*` is restricted. -## Register the Actions runner (on the desktop, not through Swarm) +## Register the Actions runner Once Forgejo answers at its domain: ```bash -# On the desktop: +# On the runner host (see DEPLOY-DEV.md for where that is today): # Forgejo web UI -> Site Administration -> Actions -> Runners -> Create new Runner # (or, for a repo-scoped runner: -> Settings -> Actions -> Runners) # copy the registration token, then: @@ -104,8 +106,8 @@ bash deploy/forgejo/register-lan-runner.sh https:// ``` See that script's header for exactly what it replaces (the pre-Forgejo GitHub -self-hosted runner on this same machine) and why it registers with two -labels where there used to be two separate runners. +self-hosted runner) and why it registers with two labels where there used to +be two separate runners. `config.yaml`'s `runner.capacity` is raised from the tool's default of 1 to 8 (override with `CAPACITY=`) — a single PR push fires `pr-build`, @@ -113,18 +115,16 @@ labels where there used to be two separate runners. no `needs:` between them), so capacity 1 serializes work that could run in parallel, and even capacity 3 (an earlier, undocumented hand-tune) leaves `build-backend`/`build-frontend`/`validate-observability` queued behind -those three before they get a slot. The desktop has 16 cores / 34GB free -today; 8 concurrent jobs is comfortable headroom without starving LAN dev's -own compose stack. +those three before they get a slot. **Adding more runner capacity should mean raising this number, or adding a -second runner on the desktop itself — not putting a runner on prod or -beta.** `container.docker_host: automount` gives job containers the *host's* -Docker socket; on prod or beta that would mean any CI job has root-equivalent -access to whatever else is running there (the live app stack, or Forgejo -itself). An earlier revision of this stack actually did run the runner as a -Swarm-hosted container on beta and was deliberately reverted to the desktop -for this reason — see the note at the top of `docker-stack.yml`. +second runner alongside it — not putting a runner on prod or beta (vps2).** +`container.docker_host: automount` gives job containers the *host's* Docker +socket; on vps2 that would mean any CI job has root-equivalent access to both +the prod and beta stacks running there. An earlier revision of this stack +actually ran the runner as a Swarm-hosted container on the box Forgejo was +pinned to and was deliberately reverted for this same class of reason — see +the note at the top of `docker-stack.yml`. ## Custom CI job image (`ci-runner/`) @@ -155,8 +155,8 @@ docker push git.thermograph.org/emi/thermograph/ci-runner:vN `register-lan-runner.sh`'s `LABELS` default points at the current tag, so fresh registrations pick it up automatically. The live runner is cut over by -editing the `labels` array in `~/forgejo-runner/.runner` on the desktop (same -runner id/token, no re-registration needed) and restarting the service — +editing the `labels` array in `~/forgejo-runner/.runner` on the runner host +(same runner id/token, no re-registration needed) and restarting the service — **verify a real job runs green under the new image before relying on it**, same way v1's break was caught. Only after that verification should the now-redundant `apt-get install docker.io` / `python3-yaml` steps be removed @@ -165,7 +165,7 @@ still running on the stock `node:20-bookworm` image. ## Why Postgres here and not the Thermograph app's TimescaleDB -Separate instance, separate network (`forgejo_net`, not the app's compose +Separate instance, separate network (`forgejo_net`, not the app's overlay network), separate volume. Forgejo is a distinct product with its own schema and its own backup/restore lifecycle — sharing a database with the app would couple two things that should be able to fail, migrate, and restore @@ -177,7 +177,7 @@ independently. docker service ls # forgejo_db, forgejo_forgejo both Running, 1/1 curl -I https://git.thermograph.org/ # 200, valid cert (Caddy's, not a new one) curl -I https://git.thermograph.org/v2/ # 403 from anywhere off the WireGuard mesh -# On the desktop, after registering the runner: +# On the runner host, after registering the runner: systemctl --user status forgejo-runner # active, both labels registered ``` diff --git a/infra/deploy/forgejo/caddy-git.conf b/infra/deploy/forgejo/caddy-git.conf index 8118682..55cb9c7 100644 --- a/infra/deploy/forgejo/caddy-git.conf +++ b/infra/deploy/forgejo/caddy-git.conf @@ -3,10 +3,11 @@ # ends up as a confusing orphaned comment in production config with no # surrounding context (docker-stack.yml isn't visible from there). # -# Target: /etc/caddy/Caddyfile on beta (the box `forgejo` is pinned to — see -# docker-stack.yml). Reuses beta's existing Caddy instead of a second reverse -# proxy/ACME flow: Forgejo publishes its web UI to 127.0.0.1:3080 only -# (host-local), and this block is the only thing that ever talks to it. +# Target: /etc/caddy/Caddyfile on vps1 (the box `forgejo` is pinned to — see +# docker-stack.yml; this is the box also called "vps1", 75.119.132.91). +# Reuses vps1's existing Caddy instead of a second reverse proxy/ACME flow: +# Forgejo publishes its web UI to 127.0.0.1:3080 only (host-local), and this +# block is the only thing that ever talks to it. # # Registry exposure (hazard #15, see docker-stack.yml's header comment): # git.thermograph.org/v2/* is the built-in OCI registry API. It's blocked @@ -15,7 +16,7 @@ # else (web UI, git-over-HTTP, PR pages) stays public like the rest of the # site. # -# Update the domain below to match whatever you actually pointed at beta's +# Update the domain below to match whatever you actually pointed at vps1's # IP, if not git.thermograph.org. # --- copy from here down into /etc/caddy/Caddyfile --- diff --git a/infra/deploy/forgejo/docker-stack.yml b/infra/deploy/forgejo/docker-stack.yml index 9891c3b..a86d41b 100644 --- a/infra/deploy/forgejo/docker-stack.yml +++ b/infra/deploy/forgejo/docker-stack.yml @@ -9,18 +9,18 @@ # (deploy/forgejo/register-lan-runner.sh) — the same place the pre-Forgejo # GitHub self-hosted runner already lived, not a Swarm-scheduled container. # -# No Traefik here. Forgejo is pinned to beta (node.labels.role == forge) -# because that's what was chosen, but beta is ALSO today's live thermograph.org +# No Traefik here. Forgejo is pinned to vps1 (node.labels.role == forge) +# because that's what was chosen, and vps1 is ALSO the emigriffith.dev portfolio # host — Caddy already owns its ports 80/443 (see /etc/caddy/Caddyfile on that # box). A second reverse proxy binding those same ports would either fail to # start or fight Caddy. Instead: forgejo's web port publishes to -# 127.0.0.1:3080 only (host-local, mode: host), and beta's existing Caddy gets +# 127.0.0.1:3080 only (host-local, mode: host), and vps1's existing Caddy gets # a new site block reverse-proxying git.thermograph.org -> 127.0.0.1:3080, -# same pattern as its thermograph.org block. TLS is Caddy's existing -# automatic-HTTPS (HTTP-01), not a second ACME flow. +# same pattern as its other blocks. TLS is Caddy's existing automatic-HTTPS +# (HTTP-01), not a second ACME flow. # -# Deploy from the manager node (prod), after all three nodes (prod, beta, -# desktop) have joined the swarm and beta is labeled role=forge: +# Deploy from the manager node (vps2, which runs prod), after all three nodes +# (vps1, vps2, desktop) have joined the swarm and vps1 is labeled role=forge: # # docker stack deploy -c deploy/forgejo/docker-stack.yml forgejo # @@ -32,7 +32,7 @@ # firewall the /v2/ registry API paths so only WireGuard-mesh clients can # reach them. That's implemented in the Caddy site block (see # deploy/forgejo/README.md), not here — nothing in this stack file is -# registry-specific, the restriction lives entirely in Caddy's config on beta. +# registry-specific, the restriction lives entirely in Caddy's config on vps1. services: db: @@ -96,9 +96,10 @@ services: FORGEJO__service__ENABLE_PASSWORD_SIGNIN_FORM: "false" FORGEJO__service__REGISTER_MANUAL_CONFIRM: "true" FORGEJO__service__DEFAULT_USER_IS_RESTRICTED: "true" - # Outbound mail via prod's Postfix null client over the WireGuard mesh - # (10.10.0.1:25 — mynetworks permits beta); self-signed cert on :25, so - # trust it. Send-only; used for admin/approval and notification mail. + # Outbound mail via vps2's Postfix null client (prod's box) over the + # WireGuard mesh (10.10.0.1:25 — mynetworks permits vps1, 10.10.0.2, + # where this Forgejo runs); self-signed cert on :25, so trust it. + # Send-only; used for admin/approval and notification mail. FORGEJO__mailer__ENABLED: "true" FORGEJO__mailer__PROTOCOL: "smtp" FORGEJO__mailer__SMTP_ADDR: "10.10.0.1" @@ -117,17 +118,17 @@ services: networks: [forgejo_net] ports: # SSH for git@ clones — published on whichever node the task lands on - # (pinned to beta by the placement constraint below, so effectively - # always beta's public IP, port 2222). + # (pinned to vps1 by the placement constraint below, so effectively + # always vps1's public IP, port 2222). - target: 2222 published: 2222 protocol: tcp mode: host # Web UI/API. Swarm's port schema has no host_ip scoping, so this binds - # 0.0.0.0:3080 on beta — NOT actually localhost-only by itself. A + # 0.0.0.0:3080 on vps1 — NOT actually localhost-only by itself. A # DOCKER-USER iptables rule (see deploy/forgejo/README.md) is what # actually restricts it, since Docker's own iptables rules bypass ufw - # for published ports. beta's Caddy reverse-proxies to 127.0.0.1:3080. + # for published ports. vps1's Caddy reverse-proxies to 127.0.0.1:3080. - target: 3000 published: 3080 protocol: tcp diff --git a/infra/deploy/forgejo/register-lan-runner.sh b/infra/deploy/forgejo/register-lan-runner.sh index 1296a1c..c204798 100755 --- a/infra/deploy/forgejo/register-lan-runner.sh +++ b/infra/deploy/forgejo/register-lan-runner.sh @@ -4,17 +4,26 @@ # runner (see DEPLOY-DEV.md) — this replaces that runner, it doesn't add a # second one. Sudo-free, systemd --user, same pattern as the app service. # -# This is THE runner (docs/runbooks/implementation-handoff.md Track B step 5 -# — canonical placement is the desktop, not a Swarm-hosted container), so it -# registers with BOTH labels the workflows need, where one runner used to be -# two: general CI/build/deploy.yml jobs (`docker`, containerized via this -# machine's own already-installed Docker — no Docker-in-Docker sidecar needed, -# unlike the Swarm-hosted design this replaces, since a real host with a real -# Docker install needs no such indirection) and the LAN-deploy job -# (`thermograph-lan`, bare/host-native — it writes to ~/thermograph-dev and -# restarts a systemd --user service, which only works running directly on the +# It registers with BOTH labels the workflows historically needed: general +# CI/build/deploy jobs (`docker`, containerized via this machine's own +# already-installed Docker — no Docker-in-Docker sidecar needed, since a real +# host with a real Docker install needs no such indirection) and the LAN-deploy +# job (`thermograph-lan`, bare/host-native — it wrote to ~/thermograph-dev and +# restarted a systemd --user service, which only worked running directly on the # host, not inside a container). # +# `thermograph-lan` IS NOW OBSOLETE. Dev moved off this machine to vps1 and is +# deployed over SSH by the same `Deploy` workflow that ships beta and prod; +# there is no host-native LAN deploy job left for that label to serve. Nothing +# breaks by keeping it registered — no workflow requests it — but do not build +# anything new on it. +# +# TODO(cutover): whether this machine keeps serving the `docker` label at all is +# an open call. It is no longer the only runner (the Swarm-hosted one is +# always-on), and the box's new job is AI model hosting plus flex Swarm-worker +# capacity. Keeping it is fine — it is real CI capacity — but the estate no +# longer depends on it, so decide deliberately rather than by inertia. +# # bash deploy/forgejo/register-lan-runner.sh # # Get from the Forgejo web UI: diff --git a/infra/deploy/openmeteo/README.md b/infra/deploy/openmeteo/README.md index fc455dd..238358f 100644 --- a/infra/deploy/openmeteo/README.md +++ b/infra/deploy/openmeteo/README.md @@ -184,6 +184,10 @@ credit; keep it in place. ## 9. Not on dev/beta -This overlay runs only on the self-hosting host (prod). Dev and beta leave -`THERMOGRAPH_ARCHIVE_URL` unset and use the public Open-Meteo archive API — do -not bring the overlay up there. +This overlay runs only for prod specifically — not for every environment on +prod's host. Beta now shares vps2 with prod, but that doesn't extend the +self-hosted archive to it: beta is its own Swarm stack +(`thermograph-beta-stack.yml`), which never sets `THERMOGRAPH_ARCHIVE_URL`, so +it reaches the public Open-Meteo archive API like dev does. Dev leaves it +unset for the same reason on vps1. Do not bring this overlay up for beta or +dev, and don't assume co-residency with prod on vps2 changes that. diff --git a/infra/deploy/provision-dev-lan.sh b/infra/deploy/provision-dev-lan.sh deleted file mode 100755 index 2daaf21..0000000 --- a/infra/deploy/provision-dev-lan.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# One-time bootstrap for the Thermograph LAN dev server on THIS machine. -# -# Sudo-free: the app runs as a Docker Compose stack (see deploy-dev.sh), and -# `linger` keeps the runner/services running across logout/reboot. Re-runnable -# (idempotent). -# -# bash deploy/provision-dev-lan.sh -set -euo pipefail - -APP_DIR="${APP_DIR:-$HOME/thermograph-dev}" -REPO_URL="${REPO_URL:-http://10.10.0.2:3080/emi/thermograph-infra.git}" -BRANCH="${BRANCH:-dev}" -here="$(cd "$(dirname "$0")" && pwd)" - -echo "==> Enabling linger so the service survives logout/reboot" -loginctl enable-linger "$USER" \ - || echo " (couldn't enable linger; the service still runs while you're logged in)" - -echo "==> Cloning/refreshing $APP_DIR and starting the service" -APP_DIR="$APP_DIR" REPO_URL="$REPO_URL" BRANCH="$BRANCH" bash "$here/deploy-dev.sh" - -cat <&2 + exit 1 +fi + +echo "==> Checkout: $APP_DIR on $BRANCH" +if [ -d "$APP_DIR/.git" ]; then + git -C "$APP_DIR" remote set-url origin "$REPO_URL" + git -C "$APP_DIR" fetch --prune origin "$BRANCH" + git -C "$APP_DIR" checkout -B "$BRANCH" "origin/$BRANCH" +else + git clone --branch "$BRANCH" "$REPO_URL" "$APP_DIR" +fi + +echo "==> Marking this checkout's environment" +mkdir -p /etc/thermograph +# The host marker still exists for by-hand runs. vps1 runs exactly one +# environment, so a marker is sufficient here — unlike vps2, where beta and prod +# share a box and THERMOGRAPH_ENV must be passed explicitly. +printf 'dev\n' > /etc/thermograph/secrets-env +# Dev is compose, not Swarm. env-topology.sh is what actually decides this; the +# marker is only the fallback for a checkout that predates it. +rm -f /etc/thermograph/deploy-mode + +if [ ! -f /etc/thermograph/age.key ]; then + cat >&2 <<'EOF' +!! No age key at /etc/thermograph/age.key. +!! Dev cannot render its vault without it, and deploy-dev.sh will fall back to +!! the un-vaulted defaults. Copy the key over (0400 root:root), then re-run: +!! install -m 0400 -o root -g root age.key /etc/thermograph/age.key +EOF +fi + +cat < FRONTEND_IMAGE_TAG=sha- \\ + $APP_DIR/infra/deploy/deploy-dev.sh + +Reach it (mesh only): + http://${TG_BIND_ADDR}:8137 + +Confirm it is NOT publicly exposed — this must show ${TG_BIND_ADDR}:8137 and +never 0.0.0.0:8137: + ss -ltnp | grep 8137 +EOF diff --git a/infra/deploy/provision-mail.sh b/infra/deploy/provision-mail.sh index 61d4788..8c76406 100755 --- a/infra/deploy/provision-mail.sh +++ b/infra/deploy/provision-mail.sh @@ -35,6 +35,12 @@ # sudo MAIL_DOMAIN=thermograph.org bash deploy/provision-mail.sh # sudo MAIL_DOMAIN=thermograph.org RELAYHOST='[smtp.provider.com]:587' \ # RELAY_USER=apikey RELAY_PASSWORD=secret bash deploy/provision-mail.sh +# +# MAIL_ENV picks which environment's deploy mode (env-topology.sh) sizes the +# Docker mail gateway below -- see the "WHICH gateway" comment. Defaults to +# prod; only matters if this box ever runs an environment in compose mode +# (it doesn't today -- prod and beta are both Swarm on this box; compose-mode +# dev lives on the other box entirely and doesn't run Postfix). set -euo pipefail MAIL_DOMAIN="${MAIL_DOMAIN:-thermograph.org}" @@ -43,6 +49,12 @@ RELAYHOST="${RELAYHOST:-}" RELAY_USER="${RELAY_USER:-}" RELAY_PASSWORD="${RELAY_PASSWORD:-}" +MAIL_ENV="${MAIL_ENV:-prod}" +SELF_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/env-topology.sh" +thermograph_topology "$MAIL_ENV" + if [[ $EUID -ne 0 ]]; then echo "run as root (sudo)" >&2 exit 1 @@ -68,17 +80,21 @@ postconf -e "myorigin = ${MAIL_DOMAIN}" # docker-compose.yml). Set DOCKER_MAIL_GATEWAY="" for a pure loopback-only null # client (app running natively on the host, not in a container). # -# WHICH gateway depends on the host's deploy mode: plain compose (beta, LAN) -# uses the pinned compose bridge (172.19.0.1/172.19.0.0/16, the defaults); -# Swarm-stack mode (prod) uses the docker_gwbridge gateway instead -- -# overlay tasks have no compose-bridge gateway -- so prod is provisioned with -# DOCKER_MAIL_GATEWAY=172.18.0.1 DOCKER_MAIL_SUBNET=172.18.0.0/16 (plus its -# MESH_MAIL_* listener below). Do NOT list an address that doesn't exist on -# the host: Postfix's master fails to bind and takes ALL listeners down -- -# exactly what happened when the compose bridge (172.19.0.1) vanished at the -# stack cutover while still listed in inet_interfaces. Also note: postfix on -# this distro is an umbrella unit; restart `postfix@-`, not `postfix`, for -# inet_interfaces changes to take effect. +# WHICH gateway to listen on comes from MAIL_ENV's deploy mode (env-topology.sh, +# TG_DEPLOY_MODE) -- not a hardcoded beta/prod split. Beta moved onto this SAME +# box as prod and is Swarm too now, so "compose" no longer means "beta"; it +# means dev, which lives on the other box entirely and never runs this script. +# Compose mode uses the pinned compose bridge (172.19.0.1/172.19.0.0/16, the +# defaults); stack mode uses the docker_gwbridge gateway instead -- overlay +# tasks have no compose-bridge gateway -- so a stack-mode MAIL_ENV (prod or +# beta; same box, same gateway) gets DOCKER_MAIL_GATEWAY=172.18.0.1 +# DOCKER_MAIL_SUBNET=172.18.0.0/16 (plus the MESH_MAIL_* listener below). Do +# NOT list an address that doesn't exist on the host: Postfix's master fails +# to bind and takes ALL listeners down -- exactly what happened when the +# compose bridge (172.19.0.1) vanished at the stack cutover while still listed +# in inet_interfaces. Also note: postfix on this distro is an umbrella unit; +# restart `postfix@-`, not `postfix`, for inet_interfaces changes to take +# effect. # # THE SAME TRAP FIRES AT BOOT, NOT JUST ON RENUMBERING (prod outage 2026-07-24). # A Docker bridge address does not exist until dockerd creates it, and the stock @@ -92,15 +108,23 @@ postconf -e "myorigin = ${MAIL_DOMAIN}" # it orders postfix@ after docker.service and wg-quick@wg0.service and retries # on failure. If you add an address here that some other daemon creates, add # that daemon to the drop-in too. -DOCKER_MAIL_GATEWAY="${DOCKER_MAIL_GATEWAY-172.19.0.1}" -DOCKER_MAIL_SUBNET="${DOCKER_MAIL_SUBNET-172.19.0.0/16}" -# Optional WireGuard-mesh listener: other mesh nodes (e.g. beta's Forgejo, whose +if [ "$TG_DEPLOY_MODE" = stack ]; then + DOCKER_MAIL_GATEWAY="${DOCKER_MAIL_GATEWAY-172.18.0.1}" + DOCKER_MAIL_SUBNET="${DOCKER_MAIL_SUBNET-172.18.0.0/16}" +else + DOCKER_MAIL_GATEWAY="${DOCKER_MAIL_GATEWAY-172.19.0.1}" + DOCKER_MAIL_SUBNET="${DOCKER_MAIL_SUBNET-172.19.0.0/16}" +fi +# Optional WireGuard-mesh listener: other mesh nodes (e.g. vps1's Forgejo, whose # mailer posts to 10.10.0.1:25 — see deploy/forgejo/docker-stack.yml) can relay -# through this box. Prod runs with MESH_MAIL_LISTEN=10.10.0.1 and -# MESH_MAIL_PEERS=10.10.0.2/32; both default OFF so a plain run stays a strict -# null client. Without these, re-running this script on prod would silently drop -# the mesh listener and break Forgejo's outbound mail — the live config was -# originally hand-applied and this script is the source of truth for it now. +# through this box. This host (vps2 -- prod AND beta) runs with +# MESH_MAIL_LISTEN=10.10.0.1 and MESH_MAIL_PEERS=10.10.0.2/32; both default OFF +# so a plain run stays a strict null client. 10.10.0.2 is vps1 (Forgejo, +# Grafana, dev) -- mesh IPs did not move in the vps1/vps2 split, only which +# environment runs where, so this is NOT "beta's" address. Without these, +# re-running this script on vps2 would silently drop the mesh listener and +# break Forgejo's outbound mail — the live config was originally hand-applied +# and this script is the source of truth for it now. MESH_MAIL_LISTEN="${MESH_MAIL_LISTEN-}" MESH_MAIL_PEERS="${MESH_MAIL_PEERS-}" postconf -e "inet_protocols = ipv4" diff --git a/infra/deploy/provision-secrets.sh b/infra/deploy/provision-secrets.sh index dd5fa4c..c70a8b5 100755 --- a/infra/deploy/provision-secrets.sh +++ b/infra/deploy/provision-secrets.sh @@ -1,16 +1,37 @@ #!/usr/bin/env bash -# Provision a host to render secrets from the SOPS vault at deploy time. Idempotent; -# run once per box (prod/beta) as a sudo-capable user. +# Provision a NAMED ENVIRONMENT to render secrets from the SOPS vault at deploy +# time. Idempotent; run once per environment (not once per box) as a +# sudo-capable user — vps2 carries two (prod AND beta), so it needs this run +# TWICE, each pointing SECRETS_ENV_FILE at that environment's own marker. # # SECRETS_ENV=prod AGE_KEY_SRC=/path/to/age.key bash deploy/provision-secrets.sh +# SECRETS_ENV=beta AGE_KEY_SRC=/path/to/age.key \ +# SECRETS_ENV_FILE=/etc/thermograph/beta-secrets-env bash deploy/provision-secrets.sh # -# Installs `sops` + `age`, then installs the age PRIVATE key at /etc/thermograph/age.key -# (0400) and writes /etc/thermograph/secrets-env. After this, deploy.sh renders -# /etc/thermograph.env from deploy/secrets/*.yaml. See deploy/secrets/README.md. +# Installs `sops` + `age` (idempotent, and shared across every environment on the +# box — one age key decrypts every deploy/secrets/.yaml, since they're all +# encrypted to the same recipient), then installs the age PRIVATE key at +# /etc/thermograph/age.key (0400) and writes SECRETS_ENV_FILE (default +# /etc/thermograph/secrets-env — right for a single-environment host, dev +# included). After this, deploy.sh renders that environment's own +# /etc/thermograph*.env from deploy/secrets/*.yaml. See deploy/secrets/README.md. +# +# SECRETS_ENV_FILE only matters for a by-hand run with THERMOGRAPH_ENV unset: +# env-topology.sh's thermograph_env_name() falls back to reading it to decide +# which environment a bare `deploy.sh` means, and a host running two +# environments (vps2) cannot answer that from one shared marker — a by-hand +# deploy of beta there must export THERMOGRAPH_SECRETS_ENV_FILE to match +# whatever path this script wrote. CI-driven deploys pass THERMOGRAPH_ENV +# explicitly and never consult this file at all. set -euo pipefail -SECRETS_ENV="${SECRETS_ENV:?set SECRETS_ENV=prod|beta}" +SECRETS_ENV="${SECRETS_ENV:?set SECRETS_ENV=prod|beta|dev}" AGE_KEY_SRC="${AGE_KEY_SRC:?set AGE_KEY_SRC=/path/to/the/age/private/key}" +# Where to write the env marker. Default matches the original, single-environment +# behavior; a SECOND environment on the same box (vps2's beta, alongside prod's +# default marker) must pass a distinct path so provisioning one can never +# overwrite the other's marker. +SECRETS_ENV_FILE="${SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" SOPS_VERSION="${SOPS_VERSION:-v3.13.2}" AGE_VERSION="${AGE_VERSION:-v1.3.1}" BIN="${BIN:-/usr/local/bin}" @@ -35,11 +56,16 @@ install_age() { install_sops install_age -echo "==> Installing age key -> /etc/thermograph/age.key (0400) and marker (${SECRETS_ENV})" -sudo mkdir -p /etc/thermograph +echo "==> Installing age key -> /etc/thermograph/age.key (0400) and marker ${SECRETS_ENV_FILE} (${SECRETS_ENV})" +sudo mkdir -p /etc/thermograph "$(dirname "$SECRETS_ENV_FILE")" sudo install -m 0400 "$AGE_KEY_SRC" /etc/thermograph/age.key -printf '%s\n' "$SECRETS_ENV" | sudo tee /etc/thermograph/secrets-env >/dev/null -sudo chmod 0644 /etc/thermograph/secrets-env +printf '%s\n' "$SECRETS_ENV" | sudo tee "$SECRETS_ENV_FILE" >/dev/null +sudo chmod 0644 "$SECRETS_ENV_FILE" echo "==> Done. Verify: sops --version && ls -l /etc/thermograph/" echo " Next: seed deploy/secrets/*.yaml, dry-run render, then deploy (see README)." +if [ "$SECRETS_ENV_FILE" != /etc/thermograph/secrets-env ]; then + echo " NOTE: non-default marker -- a by-hand deploy of ${SECRETS_ENV} on this box" + echo " must export THERMOGRAPH_SECRETS_ENV_FILE=${SECRETS_ENV_FILE} (CI-driven" + echo " deploys pass THERMOGRAPH_ENV explicitly and never consult this file)." +fi diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index b3b5b3c..fb5731c 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -15,12 +15,21 @@ # (prod|beta|dev). A host without them keeps whatever /etc/thermograph.env it already # has — so merging this is a safe no-op until a host is deliberately migrated. See # deploy/secrets/README.md. +# render_thermograph_secrets [env_name] [out_file] +# +# env_name and out_file are explicit since vps2 began running two environments. +# A host-wide marker file cannot name two environments, and one output path +# cannot hold two renders: on vps2 prod renders prod.yaml to /etc/thermograph.env +# while beta renders beta.yaml to /etc/thermograph-beta.env. Both arguments keep +# their pre-split defaults (marker file, /etc/thermograph.env), so a +# single-environment host and a by-hand run behave exactly as before. render_thermograph_secrets() { local repo="${1:-.}" # repo root holding deploy/secrets local key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" - local env_name - env_name=$(cat "$marker" 2>/dev/null || true) + local env_name="${2:-}" + local out="${3:-/etc/thermograph.env}" + [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true) # The per-host file is required; common.yaml is optional so the initial cutover can # seed each host as an exact copy of its live env (byte-identical render, no value @@ -31,7 +40,7 @@ render_thermograph_secrets() { # (a) Not configured for SOPS at all — no env marker, or no age key. The LAN # dev box is legitimately in this state. Saying so and succeeding is right. if [ -z "$env_name" ] || [ ! -f "$key" ]; then - echo "==> SOPS secrets not configured here; using existing /etc/thermograph.env" + echo "==> SOPS secrets not configured here; using existing $out" return 0 fi @@ -57,7 +66,7 @@ render_thermograph_secrets() { return 1 fi - echo "==> Rendering /etc/thermograph.env from deploy/secrets (common + ${env_name})" + echo "==> Rendering $out from deploy/secrets (common + ${env_name})" # The age private key is root-owned (0400). Read it directly if we can, else via # sudo into SOPS_AGE_KEY — so the key never has to be readable by the deploy user. # (The render needs sudo to write /etc/thermograph.env below anyway.) @@ -119,12 +128,12 @@ render_thermograph_secrets() { # failed in-place `cat` write to status 0. Capture the status instead, so a # half-written /etc/thermograph.env fails loudly rather than deploying stale. local rc=0 - if [ -f /etc/thermograph.env ] && [ -w /etc/thermograph.env ]; then - cat "$tmp" > /etc/thermograph.env || rc=1 - elif install -m 0640 "$tmp" /etc/thermograph.env 2>/dev/null; then : - elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" /etc/thermograph.env 2>/dev/null; then : + if [ -f "$out" ] && [ -w "$out" ]; then + cat "$tmp" > "$out" || rc=1 + elif install -m 0640 "$tmp" "$out" 2>/dev/null; then : + elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then : else - echo "!! cannot write /etc/thermograph.env (need file write access or passwordless sudo)" >&2 + echo "!! cannot write $out (need file write access or passwordless sudo)" >&2 rc=1 fi diff --git a/infra/deploy/secrets/README.md b/infra/deploy/secrets/README.md index 7242fd3..4b85e59 100644 --- a/infra/deploy/secrets/README.md +++ b/infra/deploy/secrets/README.md @@ -16,8 +16,8 @@ no per-host duplication. |------|-------|------------| | `common.yaml` | The 16 values identical on **prod and beta** — VAPID keypair, metrics token, IndexNow key, `REGISTRY_TOKEN`, S3 endpoint/bucket and both S3 keypairs, plus shared non-secret config. **Not layered under `dev`** — see below | ✅ | | `prod.yaml` | Prod's own — the three held-back credentials below, sizing (`APP_CPUS`, `DB_CPUS`, `DB_MEMORY`, `WORKERS`), `THERMOGRAPH_BASE_URL`, and the Discord + mail credentials that exist nowhere else | ✅ | -| `beta.yaml` | Beta's own — the three held-back credentials, sizing, `THERMOGRAPH_BASE_URL` | ✅ | -| `dev.yaml` | The LAN dev box's own — **self-contained**, 12 values, no production credential among them | ✅ | +| `beta.yaml` | Beta's own — the three held-back credentials, `THERMOGRAPH_BASE_URL`, `APP_CPUS`/`WORKERS`, plus `THERMOGRAPH_MAIL_BACKEND=console` and `THERMOGRAPH_DISCORD_BOT=0`. Its `POSTGRES_PASSWORD` and `THERMOGRAPH_DATABASE_URL` name beta's **own role and database** (`thermograph_beta`) on the instance it shares with prod. `DB_CPUS`/`DB_MEMORY` were **removed**: beta no longer runs a database of its own, and leaving them would have implied it sized one — the shared instance is sized by `prod.yaml`'s values (see `deploy/stack/thermograph-stack.yml`'s `db` service) | ✅ | +| `dev.yaml` | Dev's own (vps1) — **self-contained**, 12 values, no production credential among them | ✅ | | `centralis.prod.yaml` | **Centralis's** nine variables, rendered to `/etc/centralis.env` on prod. A different service, a different output file, a different renderer — see below | ✅ | | `example.yaml` | Format reference / CI fixture (fake values) | ✅ | | `../../.sops.yaml` | Which age recipient files are encrypted to (plaintext config) | — | @@ -25,28 +25,55 @@ no per-host duplication. ### Three credentials are deliberately NOT in `common.yaml` `POSTGRES_PASSWORD`, `THERMOGRAPH_AUTH_SECRET` and `THERMOGRAPH_DATABASE_URL` -hold **identical values on prod and beta today**, so by the mechanical rule they -belong in `common.yaml`. They are kept per-host anyway. +hold **different values on prod and beta today** (beta has its own database +role/password on the shared instance, and its own auth secret), so unlike the +rest of `common.yaml` they were never candidates for sharing on values alone. +They stay per-environment for a reason that matters more now than it used to: -These are the credentials that let one environment act as another: with them, a -foothold on beta is a foothold on prod's database and prod's session signing. -Beta is the *more* exposed box — it serves public Forgejo and Grafana. They match -only because beta was seeded from prod, not because the two are meant to be one -system. +**These are the credentials that let one environment act as another.** With +matching values, a foothold in one environment's rendered env file is a +foothold on the other's database and the other's session signing. That used +to matter because beta and prod were separate boxes; it matters for a +different reason now that they're co-resident on vps2 — keeping the +credentials themselves separate is what stops co-location on one host from +also becoming equivalence at the credential level. `deploy/db/provision-env-db.sh` +enforces the same boundary from the database side: beta's role +(`thermograph_beta`) is `NOSUPERUSER`/`NOCREATEDB`/`NOCREATEROLE`, owns only +its own database, and `CONNECT` is revoked from `PUBLIC` on it — so even a +leaked beta credential cannot reach prod's data, and the reverse. -Keeping them per-host costs nothing now (same values, one extra line each) and -buys the ability to diverge: rotating prod's database password stops implying -"and beta's too". Moving them into `common.yaml` would encode the equivalence as -intentional and turn breaking it into a migration rather than an edit. +**What actually separates beta and prod now, stated plainly:** they share a +host by design (see `deploy/env-topology.sh`'s header), so a host-level +compromise of vps2 is shared between them — separate SSH credentials no +longer buy a host boundary between beta and prod, the way they once did when +beta and prod were different machines. What remains is isolation at the +**database** level (separate roles, separate databases, `CONNECT` revoked from +`PUBLIC`) and the **file** level (separate checkouts, separate rendered env +files, separate stack env files, separate loopback ports). Keeping +`POSTGRES_PASSWORD`/`THERMOGRAPH_AUTH_SECRET`/`THERMOGRAPH_DATABASE_URL` out of +a shared file is part of that file-level isolation: nothing on vps2 would ever +hand you prod's auth secret merely because you have beta's. -If you ever *do* want them to differ, change one file. That is the whole point. +**vps1 is the box that must never hold prod credentials.** It runs Forgejo, +Forgejo's CI runner, Grafana, and the `dev` environment — which runs whatever +branch is currently in flight, reviewed or not. That is the actual reason +`dev` renders `dev.yaml` **alone** and never layers `common.yaml` (see below): +not squeamishness about a dev box, but that vps1 is uniquely positioned to +leak whatever it's handed, and `common.yaml` is the fleet's shared production +credential set. -`dev.yaml` is the case that already differs: all three of its copies are its own, -generated on the desktop. The rule earned its keep the first time it was used. +Keeping the three held-back credentials per-environment costs nothing (one +line each) and buys the ability for them to diverge further, or for a future +environment to be added without inheriting anyone else's values by accident. + +`dev.yaml` is the case that already differs completely: all three of its +values are its own, generated on vps1 and never shared with prod or beta. At deploy the renderer concatenates `common.yaml` then `.yaml`, so a **host value wins** (env_file / `source` take the last occurrence of a duplicate key). -`` comes from `/etc/thermograph/secrets-env` on the box (`prod`/`beta`/`dev`). +`` comes from an explicit `THERMOGRAPH_ENV`/argument on vps2 (which runs +two environments) or `/etc/thermograph/secrets-env` as a fallback elsewhere — +see `deploy/env-topology.sh`. ### `dev` is vaulted, but renders `dev.yaml` **alone** @@ -65,17 +92,17 @@ is a **plaintext concatenation**, so layering it would put every one of those in `/etc/thermograph.env` on the dev box and, through `env_file:`, into the environment of every container in the dev stack. -The dev box is the operator's desktop. It is also the Forgejo CI runner, whose -`docker`-labelled jobs get the host docker socket automounted, and the stack it -runs is the `dev` branch — code that has not been through the gate that guards -prod. Handing that the estate's read-write object-storage keys and the push -signing key is a strictly larger blast radius than anything the vault buys back. -An override in `dev.yaml` would not help: last-wins governs *consumers*, but the -production value is still physically a line in the file. +The dev box is **vps1** — the same box that runs Forgejo and its CI. Its +`docker`-labelled runner jobs get the host docker socket automounted, and the +stack it runs is the `dev` branch — code that has not been through the gate +that guards prod. Handing that the estate's read-write object-storage keys and +the push signing key is a strictly larger blast radius than anything the vault +buys back. An override in `dev.yaml` would not help: last-wins governs +*consumers*, but the production value is still physically a line in the file. One value in `common.yaml` is also simply **wrong** for dev: `THERMOGRAPH_COOKIE_SECURE=1` is right for the TLS hosts and silently breaks login -over dev's plain-HTTP LAN URL. `dev.yaml` sets `0`. +over dev's plain-HTTP mesh-only URL. `dev.yaml` sets `0`. So `dev.yaml` is self-contained. Where the value is shared-but-harmless (`PORT`, `THERMOGRAPH_BASE`, `TIMESCALEDB_TAG`) it carries its own copy — a @@ -87,8 +114,8 @@ duplicated constant is the cheap half of this trade. |-----|-----| | `POSTGRES_PASSWORD`, `THERMOGRAPH_DATABASE_URL` | dev's own. Deliberately the weak, well-known `thermograph-dev` — the value dev's `pgdata` volume is already initialized with, and the DB is never published off the compose network. Kept per-host for the reason below, and *not* shared with prod | | `THERMOGRAPH_AUTH_SECRET` | dev's **own**, freshly generated. Fixes a live crash loop: the `daemon` service refuses to start without it (it derives the `/internal/*` token from it), so dev's daemon had been restarting every 60s. Prod's value must never be here — it signs sessions and verification links | -| `THERMOGRAPH_BASE_URL` | `http://10.0.1.216:8137`. Unset, the app defaults to `https://thermograph.org`, so dev's IndexNow pings and Discord links claimed to be prod | -| `THERMOGRAPH_COOKIE_SECURE=0` | plain HTTP on the LAN | +| `THERMOGRAPH_BASE_URL` | `http://10.10.0.2:8137` (vps1's mesh address). Unset, the app defaults to `https://thermograph.org`, so dev's IndexNow pings and Discord links claimed to be prod | +| `THERMOGRAPH_COOKIE_SECURE=0` | plain HTTP on the mesh-only URL | | `THERMOGRAPH_MAIL_BACKEND=console`, `THERMOGRAPH_DISCORD_BOT=0` | explicit fail-safes. Both are already the code defaults; stating them means dev cannot start mailing or open a Discord gateway by inheriting a value | | `PORT`, `THERMOGRAPH_BASE`, `TIMESCALEDB_TAG`, `DB_MEMORY`, `WORKERS` | non-secret config that would otherwise have come from `common.yaml` | @@ -104,8 +131,8 @@ duplicated constant is the cheap half of this trade. (`backend/indexnow.py`). Prod's key is what authenticates URL submissions *for thermograph.org*. - **`THERMOGRAPH_METRICS_TOKEN`.** With no token `/api/v2/metrics` is - direct-loopback-only, which is exactly right on a LAN box; dev's Alloy agent - ships logs, not metrics, so nothing scrapes it. + direct-loopback-only, which is exactly right on a mesh-only box; dev's Alloy + agent ships logs, not metrics, so nothing scrapes it. - **`REGISTRY_TOKEN`.** dev pulls with the persistent `docker login` credential already in the host's docker config, like the prod/beta CI paths. A vault copy would only duplicate a credential the box already has into a more readable file. @@ -224,12 +251,17 @@ files), so layering would only push the VAPID private key, both S3 keypairs and - `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time. - **Back it up** in the password manager. The public recipient is in `.sops.yaml`. -On the **dev desktop** those two are the same file, on purpose. `render-secrets.sh` -falls back to `sudo cat` for a root-owned key, the desktop has no passwordless sudo, -and the Forgejo runner is a `systemd --user` service with no tty — a `/etc` copy -would make every CI-triggered dev deploy hang on a prompt it cannot answer. -`deploy-dev.sh` points `THERMOGRAPH_AGE_KEY` at the operator's keyring instead, so -the box keeps **one** copy of the recovery root rather than two. +On **vps1** (dev) those two can end up being the same file. `render-secrets.sh` +falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes +the account driving a CI-triggered dev deploy has no passwordless sudo and no +tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) — +inherited from the old desktop setup, where that was true of the whole box. +Whether it's still true of vps1's CI-runner account specifically (as opposed +to the `agent` login, which does have passwordless sudo there per `ACCESS.md`) +wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY` +at the operator's keyring as a fallback either way, so the box keeps **one** +copy of the recovery root rather than two regardless of which account ends up +mattering. ## Prerequisites (once per machine) @@ -246,8 +278,10 @@ go install github.com/getsops/sops/v3/cmd/sops@latest # if you have Go ```sh sops deploy/secrets/common.yaml # opens DECRYPTED in $EDITOR; re-encrypts on save git commit -am "rotate " && git push forgejo -# beta auto-deploys on push to main. prod (no CI) — one command: -ssh agent@169.58.46.181 'cd /opt/thermograph && git pull && deploy/deploy.sh' +# beta and prod both auto-deploy on push (main -> beta, release -> prod) via +# the Deploy workflow. To force it by hand on vps2, say which environment: +ssh agent@169.58.46.181 'cd /opt/thermograph && git pull && THERMOGRAPH_ENV=prod deploy/deploy.sh' +ssh agent@169.58.46.181 'cd /opt/thermograph-beta && git pull && THERMOGRAPH_ENV=beta deploy/deploy.sh' ``` A host-specific value (e.g. `POSTGRES_PASSWORD`) is the same, editing `prod.yaml` / @@ -264,10 +298,13 @@ commit, and the next `deploy-dev.sh` run (merge to `dev`, or by hand) renders it ## Setting up a new dev machine -The desktop is hands-on: Centralis cannot SSH to it, so this is typed, not -automated. `provision-secrets.sh` is **not** the right tool here — it installs the -age key root-owned at `/etc/thermograph/age.key`, which the runner cannot read -(above). +Dev is provisioned like any fleet host now (`deploy/provision-dev.sh`, run as +root on vps1), not the sudo-free desktop bootstrap this predates. The +secrets-specific steps below are still worth doing by hand rather than +folding into that script: `provision-secrets.sh` is **not** the right tool +here — it installs the age key root-owned at `/etc/thermograph/age.key`, +which the CI-runner-driven render may not be able to read (see the age-key +note above). ```sh # 1. sops + age on PATH, and the age private key in the operator's keyring: @@ -293,17 +330,17 @@ SOPS_AGE_KEY_FILE=~/.config/sops/age/keys.txt \ # 5. Deploy. deploy-dev.sh refuses to run if render-secrets.sh in the checkout # cannot honour THERMOGRAPH_SECRETS_SKIP_COMMON, rather than leak. -APP_DIR=~/thermograph-dev bash deploy/deploy-dev.sh +APP_DIR=/opt/thermograph-dev bash deploy/deploy-dev.sh ``` To take a dev box **back** off the vault, delete `/etc/thermograph/secrets-env`: presence detection turns rendering off and `deploy-dev.sh` falls back to its built-in `POSTGRES_PASSWORD` default. -A *different* dev machine (a second desktop, a laptop) needs its own values, not a -copy of this one's: give it its own `.yaml` and its own marker name, so the two -can never be confused for one host — the same argument as the three held-back -credentials above. +A *different* dev machine (a laptop running `make dev-up` locally, or a future +second dev host) needs its own values, not a copy of vps1's: give it its own +`.yaml` and its own marker name, so the two can never be confused for one +host — the same argument as the three held-back credentials above. ## Add a new secret @@ -332,17 +369,24 @@ git commit -am "rotate age identity" && push + deploy ## First-time bootstrap (seed from the live boxes) +**Historical** — this ran once, before the vps1/vps2 split, when beta was its +own box at `75.119.132.91`. That address is **vps1** now (dev + Forgejo + +Grafana), not beta; re-running the beta line below as written would seed +`beta.yaml` from the wrong host. Kept for the shape of the process, which is +unchanged if a new environment is ever bootstrapped the same way — just point +`seed-from-live.sh` at wherever that environment actually lives today. + Order matters — values must match the live env exactly so `AUTH_SECRET` / VAPID / `POSTGRES_PASSWORD` don't rotate unintentionally: 1. Install `sops`+`age` and drop `/etc/thermograph/age.key` (0400) + - `/etc/thermograph/secrets-env` on prod & beta (`provision-secrets.sh`). + `/etc/thermograph/secrets-env` on the target hosts (`provision-secrets.sh`). 2. Seed each host's file from its live env with the helper (pulls over SSH, encrypts, and verifies the render round-trips — run it on your own machine, it reads live - secrets): + secrets), e.g. as originally run: ```sh deploy/secrets/seed-from-live.sh prod agent@169.58.46.181 - deploy/secrets/seed-from-live.sh beta agent@75.119.132.91 + deploy/secrets/seed-from-live.sh beta agent@75.119.132.91 # beta's address AT THE TIME; now vps1's ``` That writes exact per-host copies (`prod.yaml`/`beta.yaml`); factor shared values into `common.yaml` later. Commit. diff --git a/infra/deploy/secrets/beta.yaml b/infra/deploy/secrets/beta.yaml index 2f3fa15..7a24542 100644 --- a/infra/deploy/secrets/beta.yaml +++ b/infra/deploy/secrets/beta.yaml @@ -1,23 +1,23 @@ -APP_CPUS: ENC[AES256_GCM,data:Wg==,iv:9kB0WruqLCIkJp7i+t5kGB4qv8NzHwgTQlFmYwhTEXc=,tag:q6cKDI86YrFJjGRnEkdweA==,type:str] -DB_CPUS: ENC[AES256_GCM,data:7w==,iv:xDOeZo3dJ5gnXIEGyOBGZm4RUTNyzinMkvxWoY/EbQs=,tag:ikmAjzeM/U4w5HDZ+C7mgA==,type:str] -DB_MEMORY: ENC[AES256_GCM,data:apg=,iv:4+GHbccVBFUGtrP12a2oEya7Hz0KUhJAFzdwpqQJ2sc=,tag:X0h2nYza80c9PjeNgqX7Uw==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:X5HmOvHVtm2mjeLZO7FaEWP0Qq767D0quycr2iI3/Mw=,iv:BQsjJaDPW9RxnMldE1rCU3yP5WBOXvc9VjnTim3kMZs=,tag:yfdi+T7lBbBK544bSvEZtQ==,type:str] -THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:YjgknnUhAU0gW6YuMCmlZYfVhOgye52MLhVF7nKMuc0ToGn0OTqqoiXN0A==,iv:Q9XCoKa5Y/7V9b72tktGZ12jgPumC4kJKjrtCM39ihw=,tag:3kEpMjLCEFQcpSir3N9f6w==,type:str] -THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:bus0/ibfLohpsszHZKKoGe5P0/Su5i36Hk2nzg==,iv:QI3crdbipse62xdakCGxPIA4wmdq/T4KzZSvi/NE1Xg=,tag:/S4Bzk2aDoA6ahDlW7qG8w==,type:str] -THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:xMd8m6VajAjtwQy8DDqTs4VbyASG8ua8FUZHfPmmEzZEb4cTtYILJ+z6LXhcyvdenDsPg2/GJx5BApvVAKSrWrnSzNWnp+pAs7x+VnNzZ2giZ9meiQ==,iv:MOU+SB/taEc/ExRc07ChF1ksOQiaDj7TPMQq+aV4Ltg=,tag:XP2h9KBGJE7JzripO7yjcg==,type:str] -WORKERS: ENC[AES256_GCM,data:BQ==,iv:XYgQ+Sn9OnbUR8LVW+4r9vhnQQJZmliVomfyRmUqMsw=,tag:JzM3JsNR08z1pePSTl/dhw==,type:str] +APP_CPUS: ENC[AES256_GCM,data:qg==,iv:T3eyF+9ssUCWDiFr8Wg0tRpgeV3BSLIjOSBE2i7WOlc=,tag:wisw0gQ6E8QCNom/Zd3mWQ==,type:str] +POSTGRES_PASSWORD: ENC[AES256_GCM,data:r5ygtwgycLjjGXSehpLt/RpUkhLyZ4lHPt6KiqNd+ng=,iv:qgqfU1mtm7ZuqSbeUEB9cgT56bOAkc0UFb3BMxED+xg=,tag:WiEoVHrA96WNFROqmFyElA==,type:str] +THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:aZySO3pAVB24S7uCsblRtd65U+OkBatK2tmuzeALdlPsqRhMqRioXQLgMQ==,iv:5zxNwj39E/AOJXzV/AEX9w1n3LSnl9PnS3cp5wVRi2E=,tag:jQRGr2436E4qqBE53iltqQ==,type:str] +THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:9IxftZqwSBG/1MY+jcALQeeTbIk+tJ/8GXGUgQ==,iv:EvsDUExWQHpidriMDDj0flM8Rz25iFycVPJLrK240+s=,tag:7rKr5KUDMe4J7U4OHUvPRg==,type:str] +THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:SnD/achZHduuWwwrQYOseqw0mquV1B4k4c2eom81dqtib6SxGj1OabHYj0rJd+o9Y5sH+kTh+zkFHpBi0J7c5GEGzhBqLctt530uMlOstvu1Hh7qL+DareyBF+SHwG4=,iv:qPFUnfRpQ6gIfBx6phyl5RWsNgEFYdNqW8oVzDZgXU8=,tag:djaqFRpDetNoH7Ux1PmQ3Q==,type:str] +WORKERS: ENC[AES256_GCM,data:gQ==,iv:jkwIzTpiEtEtbt7acP8DWRbl51baTFuQiw4oJxb3eyA=,tag:X2Gqy3icYRKrI6F99lnuyQ==,type:str] +THERMOGRAPH_MAIL_BACKEND: ENC[AES256_GCM,data:OrchUc9tpw==,iv:SmAZ+USGesaCyaOdKopqKmPeG7y/WtDry9G3aDVgP/Q=,tag:EGh5foV6uClrIJCdkjP3Hg==,type:str] +THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Yw==,iv:jsHid5lQ75shgfIJkrsLnlhmcJ1aJWuxkUyhmpg52Dc=,tag:UqtZjQpYwQH5uvEYC9mp7w==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFWHVZU2xSS1pmcy91S2w5 - MWNhQW90TDdQdDdJV1BHM2Y0a1d6aFJOS2hnCjRodHZlY0orODdpS3QwTVdTdXM5 - Z3RJOFhyR3AzbjljaExkQjN0YUJ2ckUKLS0tIGpua1Z2TDJrRzE3WFZlMWNpL1Ja - VG45eVM0MldLejRNd0pweWNna3JYWlUKWuNU+6PqKlbr7F0ckrNxsMF2OyXh1fMu - cLFBIQg/7vO7O7PJ0VIy0Ugfq6gj2Gv91qKJUGeOXOw3tv1Y6HHbjA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqdFZ3MWVCSDdvdnVLWlZH + My9JMzF2TmhiSlJjbWRYbUIzSm0zb1dtdTI4CmI4N2o0YzR1NGNoTmI5OXBJM1Bj + OVF3WWEvYmlWczR2dUJpYTROR2pvK2sKLS0tIDM5blFFdnZwRHpDOEZ6aHhyL0dn + d1BDWVV6YXNQNWo3dWRqN2FVUC85UmMK8jjGvQxKDsnlr7i95Ar509nihDgRb/JW + qzLM5fwMU16HOPMiAPd/zH3rUyKcZxMtITR6QLcuSNLByURg4ZMBnQ== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-25T00:38:24Z" - mac: ENC[AES256_GCM,data:lTpEmA8gweS+QlAakIziXxAEiHBUsNDk5lGwVG+uaXNhifIYWbu+QWkklP4wES9o3pu4TNCzFboRWkBbnzhW8HYRglkSChsmmFXDChz67IYlaveCFwczxffp39iypnV/EKaBCa8m1eDJ5kugzr2s7+blagCH0TwpNXBqZShyGNE=,iv:KBF6CreB4oTJGwNetTj27Grox30X3s3qEEYb1cJXmIE=,tag:YfRb5yAoY8K/TXspoZYdcA==,type:str] + lastmodified: "2026-07-25T21:43:52Z" + mac: ENC[AES256_GCM,data:/+kmyMP20n7fQRO2FdiHWaLl8MAfnMjtF8OE5HEUCLpRGet3s+IqYb8DajFIF9SCibW7aEqb6ir9m0KBx0IRW4TTsvK6JYFyRZKKtt4oiCzRaGwUMwhI75DE6xYdHxEbdAC6Q+qFUKGtRcIGhSX1LrlGEloSJggdEFSgfTlJLNg=,iv:UX1zMFw70ODiPolKSjEj8adKxa49Kf5dzfDA7ePtxmE=,tag:dA0OqIPTvXBlMPNH4oCNtg==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/secrets/dev.yaml b/infra/deploy/secrets/dev.yaml index eb84992..76e5b51 100644 --- a/infra/deploy/secrets/dev.yaml +++ b/infra/deploy/secrets/dev.yaml @@ -1,27 +1,27 @@ -PORT: ENC[AES256_GCM,data:0LjaWw==,iv:3e8Q71QFlMAhQStgoiVh/c2ELI3+7UkWseAFv8BnElU=,tag:/wYfKmqErJpsLBjg1gs4Tw==,type:str] -THERMOGRAPH_BASE: ENC[AES256_GCM,data:Ww==,iv:e2yNIAAw4dtlahfbA+PCM/jyODKFwWFNfVpVJ1JoREc=,tag:QzoGwDgIwdA9OFAMIPej7Q==,type:str] -TIMESCALEDB_TAG: ENC[AES256_GCM,data:7Oj7OoxoaG51whg=,iv:PPbfdn4DHMxTPA5FUqv+lEY29wQsVX+g0aVuWvraYrU=,tag:NAyJZlRXU1iKo4jwBctUtw==,type:str] -DB_MEMORY: ENC[AES256_GCM,data:/3U=,iv:jFI/9NQaz8oIqBWAKydWUEEI7bI3SmLnsoBlVgfKw7I=,tag:oBoFANuTgZV0jXC9qR5IHQ==,type:str] -WORKERS: ENC[AES256_GCM,data:VQ==,iv:+d84ByEU22XVQi43NcRn8cHmbW9XudJMj0guidsHxBQ=,tag:XM//7y0VQCwini4CGL7pSg==,type:str] -THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:FK2kOaZG8ESwDi3VMOlo0XvBKvdADQ==,iv:c6GvCrAnAf+ZFh53xg3ZOlf934Ls+PyR/+CXRi933lE=,tag:cYLXvxP5hk7/6gmz/CCP0A==,type:str] -THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:zg==,iv:sVr33CvZoZmn0LcHlP+FVA8esjGy0aKe1A2iPrrrT0I=,tag:3YneNF7m05fCRGfMgYzByA==,type:str] -THERMOGRAPH_MAIL_BACKEND: ENC[AES256_GCM,data:pRBtOu6cTw==,iv:88QgyY6rwwp/6Y32r6Us1M+b7YpSLPfKIsCL2j/YuIY=,tag:FrchZZrMcCSeOFRWbePM4Q==,type:str] -THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:NQ==,iv:Zl888EtEoQQd8FdqGb4/N7gGMT4yEymQfWd2fZZCNVA=,tag:Xspr8/taeQwh1qHhfFbLfw==,type:str] -POSTGRES_PASSWORD: ENC[AES256_GCM,data:l076hUutOAKw7+X6wboX,iv:9WVz3SVUikWREVjB6GsJ+EXWO3PvX+g+OaZNGN9f+jU=,tag:OkVuEqEbTSikpCNEySPX5A==,type:str] -THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:kXCYmZXewa8JSY5iJd0cHDtsKrxyKEl5tSVwWeQEd8u7XU+/dn4BExT4RDB3vUWVFVoHXYIXedMnDO8h0QfDCxSI13s=,iv:s1bDjdH4E4T+kF126saOrKlx3AnKEkRWF68KEPcCUe0=,tag:nCzD9U/7g28bJmUXkiYh3Q==,type:str] -THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:EjwKGKh1od5vgvm8LLv1l6TXO7nshUwh5ihXDpETl+3ldtF8L4H3CWszsw==,iv:4mqwEXWpGpLYbIUotHj9Qf7FSD0ATwkxwhfuL7pG+Zc=,tag:0iLU0ee86H0CgmgPCCtatQ==,type:str] +PORT: ENC[AES256_GCM,data:VCheDw==,iv:u9IC7psxzcsLuuWhU89fUJfVsmCI7lECbjxzKmC/nGM=,tag:ZTG/kG2vSNvUUlItAjsCHg==,type:str] +THERMOGRAPH_BASE: ENC[AES256_GCM,data:Aw==,iv:Pcmbm6D126jncQnmBquZ/t6qMaaUF7UDnQV6q85CsrY=,tag:SGakFybX3TNKAWvFo9gsEw==,type:str] +TIMESCALEDB_TAG: ENC[AES256_GCM,data:ghewNqRFuYJfBgE=,iv:q1GsQM3QZBW4A95U9NY90RR9WRmPV6miIxGLgLmPTps=,tag:eJARZNyqnJbHhtXXUI1AWw==,type:str] +DB_MEMORY: ENC[AES256_GCM,data:k2U=,iv:ZRCjRUHbNvVrCdGWqmOoJBtsJLeLVSVm32szpQ23HN4=,tag:KE5jUadHZnKFPis0friNbg==,type:str] +WORKERS: ENC[AES256_GCM,data:yQ==,iv:6VE9mjS7QLuOWazReqyt1DhAhamkvWSm92B2d/ZCZYo=,tag:hDl1/E11xwUyPVhFNhzOuQ==,type:str] +THERMOGRAPH_BASE_URL: ENC[AES256_GCM,data:/sOgmM0eCf7dWldK0iZaDWE0W4aN,iv:dpiLeHLanO2zwtMmPMbsqR4aTVaXfT+V2Ux9K8Alh3w=,tag:PDd/L2nt4X750xCKjHnSPw==,type:str] +THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:Bw==,iv:dsLcLu126A3d7dTqBgQY1SOvJfKOBT/ye4ucIB4w6B4=,tag:/G7Qvlzb6KKVVjqmn3d/Nw==,type:str] +THERMOGRAPH_MAIL_BACKEND: ENC[AES256_GCM,data:ryWhCkSdxg==,iv:Lpl1CIjTINV8rF8BW5MHZD4Ji802i96zcyqF3xRMLcY=,tag:I0nbb6IP6LM/Y+KaHjBDpg==,type:str] +THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Rw==,iv:o5tr5lQGXFkB6kTJ/dDARBWECDYRdofFlZ7v/HdMe5k=,tag:Lv97fH6u/nIKZlJr5HjbZw==,type:str] +POSTGRES_PASSWORD: ENC[AES256_GCM,data:IBVel17GKtrW5yzqDjBi,iv:yEPSSYR/Ng9rjSvSl0veNRDPLayF0SvWZ1T8BSY1l50=,tag:hbB4Uo6YluQPgl9XyvgfKw==,type:str] +THERMOGRAPH_DATABASE_URL: ENC[AES256_GCM,data:KjsSo+IglfY5MhtEsX3HJpExAtBwzmSu3E+O7Y22B2WuIrTKGDxG81WwvQGABYfKVel6kzzv8uKczq1qEtOwHRdMsRI=,iv:VPJNe1GrJa7BeAA/N/fqyBjBF+ciF8btE4j/TwrV3cA=,tag:tkyzkR1LxY8z28bjb0bJ7w==,type:str] +THERMOGRAPH_AUTH_SECRET: ENC[AES256_GCM,data:ccEQxcNxMEn8/Gf2knQaVMSoBp7BRJ7zWhr9KmawaSMrwHGU7kADDWCoMw==,iv:nDpMHhP0ubjzz2LIY0QMGFLGP1M10zOIJb6pPgri0PI=,tag:+Z2j5XLdV+LuSK7hWf5r0Q==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzV0lrcUpjeU81TmJDRVpa - elppeU1mMUlpc3lKSTVtU3MxUi83bzN2dVVRCmxZbDJFUkJQVjhCSmljNFpPaWVu - dkNqTkJTd1ZEaHZsZitpTWlJdElCK28KLS0tIEdSTXN6S1Y2eE12U3o2RGxBNXNN - bDBlYkNxMDVTVEFLYVIyVkhIRnNwOGMKv/TMt307XqvzBLCOCA5C4kXFV9iJeVBn - 7gyzb5MRbHbDoNAY/5ckU3341uWUXUQ+IrPvVt2snAdXIlWghm+HwA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyb2dCeVgrbGxGSnNpaUE3 + RlovN0RrejVmREY0ckNGaExpNWkvSjcyRW1RCjYvc1IrMWwvdmJGU2g5bU52Mmgw + SVY1dnFHd0cyNkM1ZWhJaWlIZldzZlEKLS0tIGtLZDRSMS82UjRFRVdZU3J1ckMz + T0g1WnRUUGFFRTVyMlF6WnZiSEoweEEKPLwcTNiE4s24J0kComnNJj5jXIBRBzMh + PVktsQm9m1ojaG0hgCZsFaMOOnE5O0Vi41jgG7WWLSNZRaVggvRbJg== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-25T00:59:09Z" - mac: ENC[AES256_GCM,data:Ga1lMooQrBhU+YasAbN3THZvjrct21KRbuxOoXf+72K5mrgPmqZyFDdB71B8dOUzmz4cqTPw3+jQx2s2ZrZCEiR3qmb897I8hWbTiFWnURSC7rjytxIe6kLhDBjCtso8ThKUMqNjAglqjkytHXljP7Cg4zCuGLEgczoakEJVB7s=,iv:NFHljXdG619XJMjy+1OuUknu+QaG5nH68fvo/zAf/TE=,tag:MyFf/edujKzypOwloVUGAA==,type:str] + lastmodified: "2026-07-25T21:44:03Z" + mac: ENC[AES256_GCM,data:ok66XdbCyLyU/2UxQ6Fo63S16r6VETT7Qk/KVJ2b4rrdd4hfOMMAvmUfFmBNkxclxBp+UMOlGJ3XpGRZRzcmWR9o1eXpFlussgDbWHU+KLoIVqAVkUsltVM9aDkg+3dl6/vInftWGw24LbfDh1WhYQ9+OwyKxZntj0sNHAShc8E=,iv:eBTeVtkAv6qcldT0x+zyO2yWVrCm0nhRUJjzaAfxLBM=,tag:BtgIWfkLSoZDcbJML9uO7g==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/secrets/seed-encrypt-on-host.sh b/infra/deploy/secrets/seed-encrypt-on-host.sh index 36769c1..45eaf59 100755 --- a/infra/deploy/secrets/seed-encrypt-on-host.sh +++ b/infra/deploy/secrets/seed-encrypt-on-host.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Seed deploy/secrets/.yaml by encrypting a box's live /etc/thermograph.env +# Seed deploy/secrets/.yaml by encrypting an environment's live env file # ENTIRELY ON THE BOX. The plaintext never leaves the box; only the already-encrypted # ciphertext is written back here. Encryption uses the age PUBLIC key (safe to hardcode) # so the machine you run this from needs nothing but SSH access + the agent key — no @@ -7,7 +7,13 @@ # # Run from any checkout of this branch that can SSH to the target with the agent key: # deploy/secrets/seed-encrypt-on-host.sh prod agent@169.58.46.181 -# deploy/secrets/seed-encrypt-on-host.sh beta agent@75.119.132.91 +# deploy/secrets/seed-encrypt-on-host.sh beta agent@169.58.46.181 +# deploy/secrets/seed-encrypt-on-host.sh dev agent@75.119.132.91 +# +# prod and beta share a box (vps2); they differ only by which env FILE is read +# (/etc/thermograph.env vs /etc/thermograph-beta.env), resolved from +# deploy/env-topology.sh. Hardcoding that path meant "seed beta" would encrypt +# PROD's live secrets into beta.yaml. # Then commit + push the resulting deploy/secrets/.yaml. Verify faithfulness with # the key-gaps skill after. See README.md. set -euo pipefail @@ -17,14 +23,18 @@ ENV_NAME="${1:?usage: seed-encrypt-on-host.sh [ssh-key]}" SSH_TARGET="${2:?ssh target, e.g. agent@169.58.46.181}" SSH_KEY="${3:-$HOME/.ssh/thermograph_agent_ed25519}" OUT="deploy/secrets/${ENV_NAME}.yaml" +# shellcheck source=infra/deploy/env-topology.sh +. deploy/env-topology.sh +thermograph_topology "$ENV_NAME" +REMOTE_ENV_FILE="$TG_ENV_FILE" # Public age recipient — NOT a secret (matches .sops.yaml). The private key never # leaves the operator's machine / the hosts' /etc/thermograph/age.key. AGE_PUB="age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2" SOPS_URL="https://github.com/getsops/sops/releases/download/v3.13.2/sops-v3.13.2.linux.amd64" -echo "==> Encrypting ${SSH_TARGET}:/etc/thermograph.env on-host -> ${OUT}" -ssh -i "$SSH_KEY" "$SSH_TARGET" "AGE_PUB='$AGE_PUB' SOPS_URL='$SOPS_URL' bash -s" > "$OUT" <<'REMOTE' +echo "==> Encrypting ${SSH_TARGET}:${REMOTE_ENV_FILE} on-host -> ${OUT}" +ssh -i "$SSH_KEY" "$SSH_TARGET" "AGE_PUB='$AGE_PUB' SOPS_URL='$SOPS_URL' ENV_FILE='$REMOTE_ENV_FILE' bash -s" > "$OUT" <<'REMOTE' set -euo pipefail exec 3>&1 # real stdout carries ONLY the ciphertext { # setup noise -> stderr, so it can't corrupt the file @@ -34,7 +44,9 @@ exec 3>&1 # real stdout carries ONLY the ciphertext fi } >&2 tmp="$(mktemp)"; tmy="$(mktemp)"; trap 'rm -f "$tmp" "$tmy"' EXIT -sudo cat /etc/thermograph.env > "$tmp" # plaintext stays on this box only +# $ENV_FILE comes from the ssh command line above (env-topology.sh resolved it +# for the named environment) — NOT hardcoded, because vps2 holds two of them. +sudo cat "$ENV_FILE" > "$tmp" # plaintext stays on this box only python3 -c ' import json, sys for line in open(sys.argv[1], encoding="utf-8"): diff --git a/infra/deploy/secrets/seed-from-live.sh b/infra/deploy/secrets/seed-from-live.sh index 365a636..d220c19 100755 --- a/infra/deploy/secrets/seed-from-live.sh +++ b/infra/deploy/secrets/seed-from-live.sh @@ -1,11 +1,19 @@ #!/usr/bin/env bash -# Seed (or re-seed) an encrypted secrets file from a box's live /etc/thermograph.env. +# Seed (or re-seed) an encrypted secrets file from an environment's live env file. # Run on the operator's machine — it reads production secrets, so it is deliberately # NOT something the agent runs for you. Requires: sops + age on PATH, the age private # key at ~/.config/sops/age/keys.txt (or SOPS_AGE_KEY_FILE), and SSH access to the box. # # deploy/secrets/seed-from-live.sh prod agent@169.58.46.181 ~/.ssh/thermograph_agent_ed25519 -# deploy/secrets/seed-from-live.sh beta agent@75.119.132.91 ~/.ssh/thermograph_agent_ed25519 +# deploy/secrets/seed-from-live.sh beta agent@169.58.46.181 ~/.ssh/thermograph_agent_ed25519 +# deploy/secrets/seed-from-live.sh dev agent@75.119.132.91 ~/.ssh/thermograph_agent_ed25519 +# +# Note prod and beta share a box (vps2) and therefore differ only by which env +# FILE is read: /etc/thermograph.env vs /etc/thermograph-beta.env. That path is +# resolved from deploy/env-topology.sh rather than hardcoded — hardcoding it +# meant "seed beta" would read PROD's live secrets and write them into +# beta.yaml, which is how one environment silently ends up holding another's +# credentials. # # Writes deploy/secrets/.yaml ENCRYPTED (exact copy of the live env, so the # deploy render is faithful), then verifies the render round-trips to the same @@ -21,13 +29,17 @@ ENV_NAME="${1:?usage: seed-from-live.sh [ssh-key]}" SSH_TARGET="${2:?ssh target, e.g. agent@169.58.46.181}" SSH_KEY="${3:-$HOME/.ssh/thermograph_agent_ed25519}" OUT="deploy/secrets/${ENV_NAME}.yaml" +# shellcheck source=infra/deploy/env-topology.sh +. deploy/env-topology.sh +thermograph_topology "$ENV_NAME" +REMOTE_ENV_FILE="$TG_ENV_FILE" command -v sops >/dev/null || { echo "!! sops not on PATH" >&2; exit 1; } tmp="$(mktemp)"; trap 'rm -f "$tmp" "$tmp.env"' EXIT umask 077 -echo "==> Pulling ${SSH_TARGET}:/etc/thermograph.env (sudo)" -ssh -i "$SSH_KEY" "$SSH_TARGET" 'sudo cat /etc/thermograph.env' > "$tmp" +echo "==> Pulling ${SSH_TARGET}:${REMOTE_ENV_FILE} (sudo)" +ssh -i "$SSH_KEY" "$SSH_TARGET" "sudo cat $REMOTE_ENV_FILE" > "$tmp" n=$(grep -cE '^[A-Za-z_][A-Za-z0-9_]*=' "$tmp" || true) [ "$n" -gt 0 ] || { echo "!! no KEY=VALUE lines read (permission? path?)" >&2; exit 1; } echo " ${n} vars" @@ -47,7 +59,7 @@ for line in open(src, encoding="utf-8"): continue pairs[k] = v with open(dst, "w", encoding="utf-8") as fh: - fh.write(f"# SOPS-encrypted — {env} host secrets, seeded from the live /etc/thermograph.env.\n") + fh.write(f"# SOPS-encrypted — {env} host secrets, seeded from that environment’s live env file.\n") for k, v in pairs.items(): fh.write(f"{k}: {json.dumps(v)}\n") # JSON-escaped scalar is valid YAML print(f" {len(pairs)} keys") diff --git a/infra/deploy/stack/deploy-stack.sh b/infra/deploy/stack/deploy-stack.sh index c3badc2..f53c149 100755 --- a/infra/deploy/stack/deploy-stack.sh +++ b/infra/deploy/stack/deploy-stack.sh @@ -18,8 +18,20 @@ # rehearsal on the same host that cannot touch live data or ports. set -euo pipefail -APP_DIR="${APP_DIR:-/opt/thermograph}" SERVICE="${SERVICE:-all}" + +# Two stacks now run on vps2 (prod and beta), so nothing here may assume "the" +# stack: the name, the stack file, the loopback ports, the env file, the +# database role and the service-name prefix all come from env-topology.sh, +# keyed by the environment deploy.sh already resolved and exported. +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../env-topology.sh" +ENV_NAME=$(thermograph_env_name) +[ -n "$ENV_NAME" ] || ENV_NAME=prod +thermograph_topology "$ENV_NAME" + +APP_DIR="${APP_DIR:-$TG_APP_DIR}" cd "$APP_DIR" case "$SERVICE" in @@ -27,10 +39,26 @@ case "$SERVICE" in *) echo "!! SERVICE must be backend|frontend|all, got '$SERVICE'" >&2; exit 2 ;; esac +# Which stack-file services this environment's roll targets. Prod's are bare +# (web, worker, ...); beta's carry a `beta-` prefix so its short DNS names never +# collide with prod's on the shared overlay — see env-topology.sh. +SVC_WEB="${TG_SVC_PREFIX}web" +SVC_WORKER="${TG_SVC_PREFIX}worker" +SVC_FRONTEND="${TG_SVC_PREFIX}frontend" +SVC_LAKE="${TG_SVC_PREFIX}lake" +SVC_DAEMON="${TG_SVC_PREFIX}daemon" + +STACK_FILE="$APP_DIR/infra/$TG_STACK_FILE" +ENV_FILE="$TG_ENV_FILE" +STACK_ENV_FILE="$TG_STACK_ENV_FILE" + if [ "${STACK_TEST:-0}" = "1" ]; then + # Test mode rehearses PROD's stack only; it is not a second-environment path. STACK_NAME="thermograph-test" LB_NAME="thermograph-test-lb" LB_HTTP_PORT=18137; LB_FE_PORT=18080 + DATA_NETWORK="thermograph-test_internal" + DB_SERVICE="thermograph-test_db" export PGDATA_VOLUME="thermograph-test_pgdata" export APPDATA_VOLUME="thermograph-test_appdata" export APPLOGS_VOLUME="thermograph-test_applogs" @@ -38,29 +66,42 @@ if [ "${STACK_TEST:-0}" = "1" ]; then docker volume create "$APPDATA_VOLUME" >/dev/null docker volume create "$APPLOGS_VOLUME" >/dev/null else - STACK_NAME="${STACK_NAME:-thermograph}" - LB_NAME="thermograph-lb" - LB_HTTP_PORT=8137; LB_FE_PORT=8080 + STACK_NAME="${STACK_NAME:-$TG_STACK_NAME}" + LB_NAME="$TG_LB_NAME" + LB_HTTP_PORT="$TG_LB_HTTP_PORT"; LB_FE_PORT="$TG_LB_FE_PORT" + # Where the database is. Prod owns it inside its own overlay; beta reaches + # the same service across that overlay, which its stack declares external. + DATA_NETWORK="$TG_DATA_NETWORK" + DB_SERVICE="$TG_DB_SERVICE" fi export STACK_NAME +echo "==> Environment: $ENV_NAME (stack $STACK_NAME on $TG_HOST, checkout $APP_DIR)" # --- secrets ------------------------------------------------------------------ # Render (SOPS) + source /etc/thermograph.env exactly like deploy.sh, then # install the uid-10001-readable copy the tasks' env-entrypoint shim sources. # 10001 = the app images' `thermograph` user; the file is 0400 to that uid. +# +# Per-environment paths throughout: on vps2 prod uses /etc/thermograph.env + +# /etc/thermograph/stack.env while beta uses /etc/thermograph-beta.env + +# /etc/thermograph/beta-stack.env. Sharing either file would give beta prod's +# database credentials and vice versa. if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then # shellcheck source=infra/deploy/render-secrets.sh . "$APP_DIR/infra/deploy/render-secrets.sh" - render_thermograph_secrets "$APP_DIR/infra" + render_thermograph_secrets "$APP_DIR/infra" "$ENV_NAME" "$ENV_FILE" fi -# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it -# cannot exist at lint time, so don't ask shellcheck to follow it. +# $ENV_FILE is rendered at deploy time from the SOPS vault — it cannot exist at +# lint time, so don't ask shellcheck to follow it. set -a # shellcheck source=/dev/null -. /etc/thermograph.env 2>/dev/null || true +. "$ENV_FILE" 2>/dev/null || true set +a -sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \ - || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env +sudo install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" \ + || install -o 10001 -g 0 -m 0400 "$ENV_FILE" "$STACK_ENV_FILE" +# The stack file bind-mounts this path into every task; export it so the +# `volumes:` interpolation resolves to the right environment's copy. +export STACK_ENV_FILE # --- image tags ----------------------------------------------------------------- # Same persisted-tags contract as deploy.sh: incoming env wins, the file @@ -90,7 +131,14 @@ FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}: # Hazard #7: the db image under an existing volume must never drift. Resolve # the digest-pinned ref from whatever is running (stack task or compose # container), falling back to the local latest-pg18's digest on first bring-up. -if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then +# +# Only the environment that OWNS the database needs this — i.e. the one whose +# own stack declares the db service. Beta's stack declares none (it uses prod's +# instance), so resolving an image pin there answers a question beta's stack +# file never asks. +OWNS_DB=0 +[ "$DB_SERVICE" = "${STACK_NAME}_db" ] && OWNS_DB=1 +if [ -z "${TIMESCALEDB_IMAGE:-}" ] && [ "$OWNS_DB" = 1 ]; then cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) [ -z "$cid" ] && cid=$(docker ps -q --filter "name=thermograph-db-1" | head -1) if [ -n "$cid" ]; then @@ -102,7 +150,11 @@ if [ -z "${TIMESCALEDB_IMAGE:-}" ]; then [ -n "$TIMESCALEDB_IMAGE" ] || TIMESCALEDB_IMAGE="$img" fi export TIMESCALEDB_IMAGE -echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" +if [ "$OWNS_DB" = 1 ]; then + echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE db=$TIMESCALEDB_IMAGE" +else + echo "==> Images: web/worker=$BACKEND_IMAGE frontend=$FRONTEND_IMAGE (db: shared $DB_SERVICE)" +fi # --- registry ------------------------------------------------------------------ if [ -n "${REGISTRY_TOKEN:-}" ]; then @@ -124,15 +176,22 @@ done # in the stack). Runs on the stack's overlay so `db` resolves. First-ever # deploy: the network doesn't exist yet — create it exactly as the stack will # (attachable overlay) so the name is adopted, then migrate, then deploy. -NET="${STACK_NAME}_internal" -# Never pre-create $NET: docker stack deploy must own it (a pre-existing -# unlabeled network makes it fail with "already exists"). On first deploy the -# migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE below). +# +# The migrate task joins the network the DATABASE is on, which for beta is +# prod's overlay rather than beta's own — and it connects as this environment's +# own role to this environment's own database (beta: thermograph_beta on both +# counts), so a beta migration can never touch prod's schema. +NET="$DATA_NETWORK" +MIGRATE_URL="postgresql+asyncpg://${TG_DB_USER}:${POSTGRES_PASSWORD}@db:5432/${TG_DB_NAME}" +# Never pre-create $NET when this stack owns it: docker stack deploy must own it +# (a pre-existing unlabeled network makes it fail with "already exists"). On +# first deploy the migrate runs AFTER stack deploy instead (FIRST_DEPLOY_MIGRATE +# below). Beta's data network is prod's and always already exists. if [ "$SERVICE" = "backend" ] || [ "$SERVICE" = "all" ]; then - if docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1; then - echo "==> One-shot migrate ($BACKEND_IMAGE)" + if docker service inspect "$DB_SERVICE" >/dev/null 2>&1; then + echo "==> One-shot migrate ($BACKEND_IMAGE -> ${TG_DB_NAME})" docker run --rm --network "$NET" \ - -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ + -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate else echo "==> First deploy: db not up yet; replicas will be rolled after stack deploy runs migrate below" @@ -141,38 +200,38 @@ fi # --- deploy ---------------------------------------------------------------------- FIRST_DEPLOY_MIGRATE=0 -docker service inspect "${STACK_NAME}_db" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 -if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/null 2>&1; then +docker service inspect "$DB_SERVICE" >/dev/null 2>&1 || FIRST_DEPLOY_MIGRATE=1 +if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_${SVC_WEB}" >/dev/null 2>&1; then echo "==> docker stack deploy ($STACK_NAME)" - docker stack deploy --with-registry-auth -c "$APP_DIR/infra/deploy/stack/thermograph-stack.yml" "$STACK_NAME" + docker stack deploy --with-registry-auth -c "$STACK_FILE" "$STACK_NAME" # First-ever deploy ran no migrate above (db didn't exist): wait for db, # migrate, then force web/worker to restart cleanly against the schema. if [ "${FIRST_DEPLOY_MIGRATE:-0}" = "1" ]; then echo "==> Waiting for db, then first-boot migrate" for i in $(seq 1 60); do - cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" | head -1) - [ -n "$cid" ] && docker exec "$cid" pg_isready -U thermograph -d thermograph >/dev/null 2>&1 && break + cid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${DB_SERVICE}" | head -1) + [ -n "$cid" ] && docker exec "$cid" pg_isready -U "$TG_DB_USER" -d "$TG_DB_NAME" >/dev/null 2>&1 && break sleep 5 done docker run --rm --network "$NET" \ - -e THERMOGRAPH_DATABASE_URL="postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph" \ + -e THERMOGRAPH_DATABASE_URL="$MIGRATE_URL" \ --entrypoint /app/deploy/entrypoint.sh "$BACKEND_IMAGE" migrate - docker service update --force --detach=false "${STACK_NAME}_web" - docker service update --force --detach=false "${STACK_NAME}_worker" + docker service update --force --detach=false "${STACK_NAME}_${SVC_WEB}" + docker service update --force --detach=false "${STACK_NAME}_${SVC_WORKER}" fi else case "$SERVICE" in backend) - echo "==> Rolling web + worker + lake to $BACKEND_IMAGE" - docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web" - docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker" + echo "==> Rolling $SVC_WEB + $SVC_WORKER + $SVC_LAKE to $BACKEND_IMAGE" + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WEB}" + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${SVC_WORKER}" # lake and daemon ship in the same image; a stack file predating either # has no service yet — the next SERVICE=all stack deploy creates it, so # don't fail here. The daemon especially must roll with web: they share # the /internal/* contract, and a version skew between them is exactly # what pinning one BACKEND_IMAGE_TAG exists to prevent (seen live: the # first post-creation backend roll left the daemon a release behind). - for extra in lake daemon; do + for extra in "$SVC_LAKE" "$SVC_DAEMON"; do if docker service inspect "${STACK_NAME}_${extra}" >/dev/null 2>&1; then docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${extra}" else @@ -181,8 +240,8 @@ else done ;; frontend) - echo "==> Rolling frontend to $FRONTEND_IMAGE" - docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_frontend" + echo "==> Rolling $SVC_FRONTEND to $FRONTEND_IMAGE" + docker service update --with-registry-auth --detach=false --image "$FRONTEND_IMAGE" "${STACK_NAME}_${SVC_FRONTEND}" ;; esac fi @@ -192,13 +251,20 @@ fi # 0.0.0.0) on the attachable overlay, proxying to the service VIPs. Recreated # only when missing/dead — its config rarely changes; `docker rm -f $LB_NAME` # to force a refresh after editing lb/Caddyfile. +# +# Note the network: the LB proxies to THIS stack's own service VIPs, so it joins +# this stack's own overlay — which for beta is beta's `internal`, not the shared +# data network its tasks also sit on. And its config is per-environment, because +# beta's services are named beta-web/beta-frontend. +LB_NETWORK="${STACK_NAME}_internal" +LB_CONFIG="$APP_DIR/infra/${TG_LB_CONFIG:-deploy/stack/lb/Caddyfile}" if ! docker ps --format '{{.Names}}' | grep -qx "$LB_NAME"; then docker rm -f "$LB_NAME" >/dev/null 2>&1 || true echo "==> Starting loopback LB bridge $LB_NAME (127.0.0.1:$LB_HTTP_PORT, :$LB_FE_PORT)" docker run -d --name "$LB_NAME" --restart unless-stopped \ - --network "$NET" \ + --network "$LB_NETWORK" \ -p "127.0.0.1:${LB_HTTP_PORT}:8137" -p "127.0.0.1:${LB_FE_PORT}:8080" \ - -v "$APP_DIR/infra/deploy/stack/lb/Caddyfile:/etc/caddy/Caddyfile:ro" \ + -v "$LB_CONFIG:/etc/caddy/Caddyfile:ro" \ caddy:2-alpine >/dev/null fi @@ -233,8 +299,14 @@ docker images --format '{{.Repository}}:{{.Tag}}' \ # Post-deploy warm + IndexNow, via any web task (skip in test mode: no data, # and the warmer would burn upstream quota against an empty cache). -if [ "${STACK_TEST:-0}" != "1" ] && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then - wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_web" | head -1) +# +# TG_POST_DEPLOY gates this per environment: prod does both, beta does neither +# (IndexNow from beta would ask search engines to index beta.thermograph.org, +# and the warm would spend the shared upstream archive quota on a rehearsal +# cache). See env-topology.sh. +if [ "${STACK_TEST:-0}" != "1" ] && [ "${TG_POST_DEPLOY:-1}" = 1 ] \ + && { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; }; then + wcid=$(docker ps -q --filter "label=com.docker.swarm.service.name=${STACK_NAME}_${SVC_WEB}" | head -1) if [ -n "$wcid" ]; then echo "==> Warming city archives (detached) + IndexNow" docker exec -d "$wcid" sh -c 'python warm_cities.py --pace 2 >> /app/logs/warm-cities.log 2>&1' || true diff --git a/infra/deploy/stack/lb/Caddyfile.beta b/infra/deploy/stack/lb/Caddyfile.beta new file mode 100644 index 0000000..756d21d --- /dev/null +++ b/infra/deploy/stack/lb/Caddyfile.beta @@ -0,0 +1,34 @@ +# Beta's loopback LB bridge config — the beta counterpart of ./Caddyfile. +# +# Two differences from prod's, both load-bearing: +# +# 1. It proxies to beta-web / beta-frontend, not web / frontend. Beta's Swarm +# services carry that prefix so their short DNS names cannot collide with +# prod's on the shared overlay (see thermograph-beta-stack.yml). +# 2. deploy-stack.sh publishes this container on 127.0.0.1:8237 and :8180, +# not 8137/8180 — prod's LB already owns 8137/8080 on this host. The +# LISTEN ports below stay 8137/8080: those are inside the container, and +# the host mapping is what differs. Keeping the container ports identical +# to prod's means the two configs differ only in the upstream names. +# +# The host Caddy on vps2 terminates TLS for beta.thermograph.org and proxies to +# 127.0.0.1:8237 / :8180 — see deploy/Caddyfile. + +{ + auto_https off + admin off +} + +:8137 { + reverse_proxy beta-web:8137 { + # Fail fast to the client if the VIP has no healthy task; Swarm's own + # task healthchecks handle ejecting dead replicas from the VIP. + lb_try_duration 5s + } +} + +:8080 { + reverse_proxy beta-frontend:8080 { + lb_try_duration 5s + } +} diff --git a/infra/deploy/stack/thermograph-beta-stack.yml b/infra/deploy/stack/thermograph-beta-stack.yml new file mode 100644 index 0000000..1d1aca2 --- /dev/null +++ b/infra/deploy/stack/thermograph-beta-stack.yml @@ -0,0 +1,278 @@ +# Docker Swarm stack for BETA, co-resident with prod on vps2. +# +# Deployed by the same deploy/stack/deploy-stack.sh as prod, which picks this +# file (and beta's ports, env file, LB and DB role) out of deploy/env-topology.sh +# when THERMOGRAPH_ENV=beta. Beta moved here from its own box so that a beta +# green light is evidence about prod: same orchestrator, same host kernel, same +# Postgres build, same Caddy, same mail path, same mesh position. +# +# --------------------------------------------------------------------------- +# WHY THIS IS A SEPARATE FILE AND NOT AN OVERLAY ON thermograph-stack.yml +# --------------------------------------------------------------------------- +# `docker stack deploy` accepts multiple -c files and MERGES them. Merging can +# add and override, but it cannot REMOVE a service — and the single most +# important fact about beta is a removal: it has no `db`. It uses prod's. An +# overlay would therefore still create a second Postgres, which is the exact +# thing this design exists to avoid. The same goes for the two autoscalers, +# which beta deliberately does not run. +# +# The cost is a file that must be kept roughly in step with prod's by hand. +# Keep them in step for anything that affects whether the APP works (env vars, +# entrypoints, healthchecks, the migrate contract). Do NOT keep them in step on +# scale, replicas or resource limits — those differ on purpose (below). +# +# --------------------------------------------------------------------------- +# THE THREE THINGS THAT MAKE CO-RESIDENCY SAFE +# --------------------------------------------------------------------------- +# 1. SERVICE NAMES ARE PREFIXED (beta-web, not web). Swarm registers a service's +# short name as a DNS alias on every network it joins. Beta's tasks share the +# `data` network with prod's, so two services both called `web` would make +# `web` ambiguous — prod's frontend could resolve a beta task, and vice +# versa. The prefix removes the collision without touching prod's stack file. +# +# 2. THE DATABASE IS SHARED, THE DATA IS NOT. Beta connects to prod's `db` +# service as the role `thermograph_beta`, to the database `thermograph_beta`. +# That role owns only its own database (see deploy/db/provision-env-db.sh), +# so a beta deploy running an unmerged branch — or a migration that goes +# wrong — cannot read or write production data. One server is a capacity +# decision, not a trust decision. +# +# 3. NOTHING ELSE IS SHARED BY ACCIDENT. Separate checkout (/opt/thermograph-beta), +# separate rendered env file (/etc/thermograph-beta.env), separate stack env +# (/etc/thermograph/beta-stack.env), separate volumes, separate loopback +# ports (8237/8180 — prod owns 8137/8080), separate LB container, separate +# deploy lock and image-tag file. Every one of those is derived in +# env-topology.sh rather than repeated here by hand. +# +# --------------------------------------------------------------------------- +# WHAT BETA DELIBERATELY DOES NOT DO +# --------------------------------------------------------------------------- +# - No autoscaling: fixed 1 replica per service. Beta exists to answer "does +# this code work", not "does it scale"; a second replica would only add a +# variable prod's rehearsal doesn't need, on a host prod is also using. +# - No IndexNow ping and no city-archive warm (deploy-stack.sh gates both on +# TG_POST_DEPLOY). Pinging IndexNow from beta asks Bing/DuckDuckGo/Yandex to +# index beta.thermograph.org; the warm spends the shared upstream archive +# quota to fill a cache only a rehearsal reads. +# - No real mail and no Discord gateway. Both are governed by beta's vault +# (THERMOGRAPH_MAIL_BACKEND=console, THERMOGRAPH_DISCORD_BOT=0) rather than +# pinned here, so the operator can opt in with `sops edit` if a release ever +# genuinely needs to rehearse them. Discord in particular allows ONE gateway +# connection per bot token — beta and prod must never both hold one. + +services: + beta-web: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + # Beta's OWN role and OWN database on the shared instance. `db` resolves + # across the external `data` network to prod's db service. + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph_beta:${POSTGRES_PASSWORD}@db:5432/thermograph_beta + THERMOGRAPH_BASE: / + PORT: 8137 + THERMOGRAPH_SERVICE_ROLE: backend + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://beta-frontend:8080 + WORKERS: ${BETA_WEB_WORKERS:-2} + THERMOGRAPH_DATA_DIR: /state + # Never the notifier/scheduler — that is beta-worker's job, exactly as in + # prod, so the two files stay honest about which process owns what. + THERMOGRAPH_ROLE: web + # Migrations run as the one-shot task in deploy-stack.sh. + RUN_MIGRATIONS: "0" + # Overlay tasks reach the HOST's Postfix via the docker_gwbridge gateway. + # Same host, same Postfix as prod — but see the mail note in the header: + # beta's vault selects the console backend, so nothing is actually sent. + THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + THERMOGRAPH_LAKE_URL: http://beta-lake:8141 + volumes: + - appdata:/state + - applogs:/app/logs + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + networks: + - internal + - data + deploy: + replicas: 1 + # vps2 is the Swarm manager and every volume here is local to it. The + # desktop is a worker on this mesh and must never be scheduled the app. + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_WEB_CPUS:-2}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + + beta-worker: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph_beta:${POSTGRES_PASSWORD}@db:5432/thermograph_beta + THERMOGRAPH_BASE: / + PORT: 8137 + THERMOGRAPH_SERVICE_ROLE: backend + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://beta-frontend:8080 + WORKERS: "1" + THERMOGRAPH_DATA_DIR: /state + THERMOGRAPH_ROLE: worker + # The advisory lock is taken in beta's OWN database, so it can never + # contend with prod's worker despite the shared server. + THERMOGRAPH_SINGLETON_PG: "1" + RUN_MIGRATIONS: "0" + THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + THERMOGRAPH_LAKE_URL: http://beta-lake:8141 + volumes: + - appdata:/state + - applogs:/app/logs + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + networks: + - internal + - data + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_WORKER_CPUS:-1}" + restart_policy: + condition: on-failure + + beta-lake: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_ROLE: lake + PORT: 8141 + THERMOGRAPH_SERVICE_ROLE: backend + WORKERS: "1" + THERMOGRAPH_LAKE_CACHE: /state/lake-cache + volumes: + # Beta's own cache volume. Deliberately not prod's: they are read caches + # of the same bucket, but sharing a volume across two stacks would couple + # their lifecycles for no gain. + - lakecache:/state + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + # No `data` network: the lake reads object storage, never Postgres. + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_LAKE_CPUS:-1}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + + beta-daemon: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + # Same reasoning as prod's daemon: NOT env-entrypoint.sh, because that shim + # execs the image's own entrypoint (Alembic + uvicorn) and migrations belong + # to the one-shot task. Source the host-rendered env and exec the binary. + # + # Note the ordering consequence, which is load-bearing here: this sources + # the env file AFTER the `environment:` block is applied, so a key present + # in /etc/thermograph/beta-stack.env WINS over one set below. That is why + # THERMOGRAPH_DISCORD_BOT=0 lives in beta's vault and not in this file — a + # value set here would be silently overridden if the vault ever set one. + entrypoint: + - /bin/bash + - -c + - 'set -a; [ -f /host/thermograph.env ] && . /host/thermograph.env; set +a; exec /usr/local/bin/thermograph-daemon' + environment: + THERMOGRAPH_API_BASE_INTERNAL: http://beta-web:8137 + volumes: + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + # The image HEALTHCHECK curls /healthz on ${PORT}; the daemon serves + # nothing, so without this override Swarm restarts it forever. + healthcheck: + disable: true + networks: + - internal + deploy: + # EXACTLY 1, and in beta's case the Discord gateway is off entirely + # (vault: THERMOGRAPH_DISCORD_BOT=0) because prod's daemon holds the only + # permitted gateway connection for that bot token. + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "0.5" + memory: 128m + restart_policy: + condition: on-failure + update_config: + order: stop-first + failure_action: rollback + + beta-frontend: + image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + # REQUIRED: overriding `entrypoint:` with no `command:` drops the image's + # CMD entirely, and env-entrypoint.sh's fallback (`exec uvicorn app:app`) + # does not exist in this Go image — the task would exit 127 every deploy. + command: ["/usr/local/bin/thermograph-frontend"] + environment: + THERMOGRAPH_BASE: / + PORT: 8080 + THERMOGRAPH_SERVICE_ROLE: frontend + THERMOGRAPH_API_BASE_INTERNAL: http://beta-web:8137 + volumes: + - /opt/thermograph-beta/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/beta-stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${BETA_FRONTEND_CPUS:-1}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + +networks: + # Beta's own east-west network: beta-web <-> beta-frontend <-> beta-lake, and + # the loopback LB bridge joins it (hence attachable). Keeping this separate + # from prod's overlay means beta's ordinary traffic never touches it. + internal: + driver: overlay + attachable: true + + # Prod's overlay, joined ONLY to reach the shared `db` service. Declared + # external because prod's stack owns it: `docker stack deploy` of THIS file + # must never create, modify or (on `docker stack rm thermograph-beta`) remove + # the network prod's database is on. + # + # Consequence worth knowing before you tear anything down: `docker stack rm + # thermograph` would take this network with it and beta would lose its + # database link until prod is redeployed. + data: + external: true + name: thermograph_internal + +volumes: + # Beta's own, created by this stack under the thermograph-beta_ prefix. Unlike + # prod's (which are `external` because they were inherited from the compose + # era and hold live data), these can be recreated: beta's appdata is a parquet + # cache plus derived files, and its DATABASE — the part that matters — lives + # on the shared instance, not here. + appdata: {} + applogs: {} + lakecache: {} diff --git a/infra/deploy/swarm/README.md b/infra/deploy/swarm/README.md index 056d609..386cd0f 100644 --- a/infra/deploy/swarm/README.md +++ b/infra/deploy/swarm/README.md @@ -1,36 +1,41 @@ -# 3-node Docker Swarm (prod + beta + desktop), for hosting Forgejo +# 3-node Docker Swarm (vps2 + vps1 + desktop) -This Swarm cluster's only job is to run Forgejo (`deploy/forgejo/`) — it does -**not** orchestrate the Thermograph app itself, which stays on the -Terraform-managed `docker compose` deploys on prod/beta independently (see -`terraform/README.md`). Keeping those separate means nothing here can strand -or interfere with the app's already-working, single-writer Postgres/TimescaleDB -deploys. +This Swarm mesh's only **Swarm-scheduled** workload is Forgejo +(`deploy/forgejo/`), pinned to vps1. It does **not** orchestrate the +Thermograph app the same way — prod and beta each run as their own `docker +stack deploy` (`deploy/stack/thermograph-stack.yml` / +`thermograph-beta-stack.yml`) that happens to land on this same manager node +(vps2) because their volumes are local to it today. Keeping Forgejo's stack +and the app stacks conceptually separate means nothing here can strand or +interfere with the app's single-writer Postgres/TimescaleDB. This is the canonical topology from `thermograph-docs/runbooks/implementation-handoff.md` (Track B steps 2-3) — three nodes, -not two. An earlier revision of this doc/scripts covered just prod+beta; -the desktop (this LAN dev machine) joins too. +not two. The desktop (formerly the LAN dev machine) joins as a worker for flex +capacity and AI-model hosting; it hosts no Thermograph environment. **Nodes:** -- **manager** — prod, the new 48 GB / 12-core box (more headroom). -- **worker** — beta, the old VPS (`75.119.132.91`). -- **worker** — desktop, this LAN dev machine (also runs the Forgejo Actions - runner as a plain systemd service — see `deploy/forgejo/README.md` — not as - a Swarm-scheduled container). +- **manager** — vps2 (`169.58.46.181`), the box with headroom for prod's and + beta's Swarm stacks and their local volumes. +- **worker** — vps1 (`75.119.132.91`), pinned to run Forgejo + (`node.labels.role == forge`). Also runs Grafana/Loki/Alloy and the dev + environment, both outside this Swarm cluster (plain Docker/compose on the + same host's daemon, not Swarm-scheduled). +- **worker** — desktop (this machine), flex capacity plus AI-model hosting. No + Thermograph environment runs here. One manager, not more: Raft needs 3 nodes for real quorum-based HA, and this cluster only has 3 nodes total, so making even one more of them a manager would still fall short of real HA while adding split-brain risk. If the -manager (prod) goes down, the workers keep running whatever was already -scheduled on them (Forgejo, pinned to beta) but the cluster can't reschedule -anything until prod's back — acceptable for a small cluster whose only job is -CI/CD. +manager (vps2) goes down, the workers keep running whatever was already +scheduled on them (Forgejo, pinned to vps1) but the cluster can't reschedule +anything until vps2's back — acceptable for a small cluster whose only +Swarm-scheduled job is CI/CD. ## Order of operations -1. **Agent access first** (`deploy/provision-agent-access.sh`) on prod and - beta — everything below on those two boxes is run through that access. The +1. **Agent access first** (`deploy/provision-agent-access.sh`) on vps1 and + vps2 — everything below on those two boxes is run through that access. The desktop is wherever you're already working from; no separate access step needed there. 2. **WireGuard mesh** (`setup-wireguard.sh `) — run on @@ -38,19 +43,21 @@ CI/CD. the two-pass key-exchange dance (pubkeys aren't known until every node has run it once). Verify with `ping ` to each of the other two before continuing. -3. **Swarm init** (`init-swarm.sh `) on the manager (prod) only. +3. **Swarm init** (`init-swarm.sh `) on the manager (vps2) only. 4. **Swarm join** (`join-swarm.sh `) on **each** of the - two workers (beta, desktop) — same token for both. + two workers (vps1, desktop) — same token for both. 5. **Firewall lockdown** (`firewall-swarm.sh`) on **all three** nodes — closes 2377/7946/4789 to everything except the WireGuard interface. Do this *after* joining is confirmed working on all three, not before (locking the ports first would make the join itself fail). -6. **Label beta** (`label-forge-node.sh `) on the manager — - `docker node ls` shows each node's name/ID. Only beta gets `role=forge`; - the desktop and prod don't need a Swarm label for anything in this setup. +6. **Label vps1** (`label-forge-node.sh `) on the manager — + `docker node ls` shows each node's name/ID. Only vps1 gets `role=forge`; + the desktop and vps2 don't need a Swarm label for anything in this setup. 7. Deploy Forgejo: see `deploy/forgejo/README.md`. -8. Register the Actions runner **on the desktop** (not through Swarm): - `deploy/forgejo/register-lan-runner.sh`. +8. Register the Actions runner: `deploy/forgejo/register-lan-runner.sh`. See + that document (and `DEPLOY-DEV.md`) for where it actually runs today — its + own header comment predates the vps1/vps2 rename and still describes "the + desktop" as the canonical placement. ## Why WireGuard instead of relying on Swarm's built-in TLS alone @@ -65,19 +72,19 @@ than trusting the public internet (or the desktop's home network) directly. ## Verifying ```bash -# On the manager: +# On the manager (vps2): docker node ls # all three nodes Ready -docker node inspect --format '{{.Spec.Labels}}' # role:forge +docker node inspect --format '{{.Spec.Labels}}' # role:forge # From a FOURTH machine outside the mesh entirely, confirm the Swarm ports # are NOT reachable on either VPS's public IP (the desktop has no public IP # to check this way): -nc -zv -w2 2377 # should fail/timeout -nc -zvu -w2 4789 # should fail/timeout +nc -zv -w2 2377 # should fail/timeout +nc -zvu -w2 4789 # should fail/timeout ``` ## Adding a node label back out (undo) ```bash -docker node update --label-rm role +docker node update --label-rm role ``` diff --git a/infra/deploy/swarm/firewall-swarm.sh b/infra/deploy/swarm/firewall-swarm.sh index 0e0ee65..d5eb74f 100755 --- a/infra/deploy/swarm/firewall-swarm.sh +++ b/infra/deploy/swarm/firewall-swarm.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Locks the Swarm ports (2377 control, 7946 gossip, 4789 overlay VXLAN) to the # WireGuard interface only — they must never be reachable from the public -# internet. Run on BOTH boxes after joining the swarm. Existing app-facing -# rules (80/443, SSH, etc.) are untouched. +# internet. Run on ALL THREE nodes (vps2, vps1, desktop) after joining the +# swarm. Existing app-facing rules (80/443, SSH, etc.) are untouched. set -euo pipefail WG_IFACE="${WG_IFACE:-wg0}" @@ -34,8 +34,10 @@ if ! ufw status | grep -q "^Status: active"; then echo "ufw is currently INACTIVE on this node — 'ufw enable' switches its" echo "default policy to deny-incoming for EVERYTHING, not just the Swarm" echo "ports above. Before enabling, explicitly allow every port this node" - echo "already serves publicly (SSH at minimum; on beta specifically, also" - echo "80/tcp and 443/tcp for the live thermograph.org Caddy) — check" - echo "'ss -tlnp' for what's actually listening first. Enabling ufw without" - echo "doing this WILL drop live traffic the moment it activates." + echo "already serves publicly (SSH at minimum; on vps2 specifically, also" + echo "80/tcp and 443/tcp for the live thermograph.org/beta.thermograph.org" + echo "Caddy; on vps1, 80/tcp and 443/tcp for git.thermograph.org," + echo "dashboard.thermograph.org and emigriffith.dev) — check 'ss -tlnp' for" + echo "what's actually listening first. Enabling ufw without doing this WILL" + echo "drop live traffic the moment it activates." fi diff --git a/infra/deploy/swarm/init-swarm.sh b/infra/deploy/swarm/init-swarm.sh index 4d41861..2501678 100755 --- a/infra/deploy/swarm/init-swarm.sh +++ b/infra/deploy/swarm/init-swarm.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash -# Run ONCE, on the manager node only (prod — the new 48 GB box). Initializes -# the Swarm advertising the WireGuard address, so cluster traffic never -# touches the public interface. Run setup-wireguard.sh on all THREE nodes -# first (prod, beta, and the desktop — see +# Run ONCE, on the manager node only (vps2 — the box with headroom for prod's +# and beta's Swarm stacks and their local volumes). Initializes the Swarm +# advertising the WireGuard address, so cluster traffic never touches the +# public interface. Run setup-wireguard.sh on all THREE nodes first (vps2, +# vps1, and the desktop — see # docs/runbooks/implementation-handoff.md Track B steps 2-3). set -euo pipefail @@ -21,7 +22,7 @@ echo "==> docker swarm init, advertising ${MY_WG_IP} (the WireGuard address, not docker swarm init --advertise-addr "$MY_WG_IP" --listen-addr "${MY_WG_IP}:2377" echo -echo "==> Worker join command (run this on EACH of the two workers — beta and" +echo "==> Worker join command (run this on EACH of the two workers — vps1 and" echo " the desktop; the same token works for both):" TOKEN="$(docker swarm join-token -q worker)" echo " docker swarm join --token ${TOKEN} ${MY_WG_IP}:2377" diff --git a/infra/deploy/swarm/join-swarm.sh b/infra/deploy/swarm/join-swarm.sh index 2baf94e..5177028 100755 --- a/infra/deploy/swarm/join-swarm.sh +++ b/infra/deploy/swarm/join-swarm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run ONCE on EACH worker node (beta, and the desktop — not the manager, -# prod). Joins the Swarm initialized by init-swarm.sh, over the WireGuard +# Run ONCE on EACH worker node (vps1, and the desktop — not the manager, +# vps2). Joins the Swarm initialized by init-swarm.sh, over the WireGuard # tunnel. The join token is the same for both workers. set -euo pipefail diff --git a/infra/deploy/swarm/label-forge-node.sh b/infra/deploy/swarm/label-forge-node.sh index cf07fcf..1533820 100755 --- a/infra/deploy/swarm/label-forge-node.sh +++ b/infra/deploy/swarm/label-forge-node.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run ONCE, on the manager node, after both boxes have joined the swarm. -# Labels the worker (beta) so the Forgejo stack's placement constraint +# Run ONCE, on the manager node (vps2), after both boxes have joined the +# swarm. Labels the worker (vps1) so the Forgejo stack's placement constraint # (node.labels.role == forge) schedules there and nowhere else. set -euo pipefail diff --git a/infra/deploy/swarm/setup-wireguard.sh b/infra/deploy/swarm/setup-wireguard.sh index f81525c..29f2d4c 100755 --- a/infra/deploy/swarm/setup-wireguard.sh +++ b/infra/deploy/swarm/setup-wireguard.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Sets up this node's side of a full-mesh WireGuard tunnel across all of -# prod, beta, and the desktop (three nodes, not two — see +# vps2, vps1, and the desktop (three nodes, not two — see # docs/runbooks/implementation-handoff.md Track B step 2). Docker Swarm's # control plane (2377/tcp) is TLS-encrypted by default, but the overlay data # plane (VXLAN, 4789/udp) is NOT — and it should never face the public @@ -14,8 +14,8 @@ # --- Peer list format -------------------------------------------------- # A text file, one line per OTHER node (not including the one you're running # on), each line: -# 10.10.0.1 169.58.46.181 -# 10.10.0.2 75.119.132.91 +# 10.10.0.1 169.58.46.181 +# 10.10.0.2 75.119.132.91 # 10.10.0.3 # # First pass on any node: peer pubkeys you don't have yet are "-". Run this diff --git a/infra/deploy/thermograph.env.example b/infra/deploy/thermograph.env.example index 5c423fd..f35537f 100644 --- a/infra/deploy/thermograph.env.example +++ b/infra/deploy/thermograph.env.example @@ -46,8 +46,9 @@ POSTGRES_PASSWORD=change-me # an empty value is honored as-is and would break the fallback to the public API. #THERMOGRAPH_ARCHIVE_URL=http://open-meteo-api:8080/v1/archive -# Mark the session cookie Secure — required behind Caddy's HTTPS. Set to 1 in prod; -# leave unset only for plain-HTTP LAN dev (a Secure cookie is never sent over HTTP). +# Mark the session cookie Secure — required behind Caddy's HTTPS. Set to 1 for +# prod and beta (both TLS-fronted); leave unset only for dev's plain-HTTP +# mesh-only URL (a Secure cookie is never sent over HTTP). THERMOGRAPH_COOKIE_SECURE=1 # Pin these too (see their own sections below), so container restarts don't rotate @@ -58,11 +59,14 @@ THERMOGRAPH_COOKIE_SECURE=1 # Number of uvicorn worker processes. More than 1 stops a single slow upstream fetch # (e.g. a cache-miss weather lookup) from blocking every other request — the cause of -# past brief outages. Prod runs 4 (the compose app service also defaults to WORKERS=4); -# leave unset (defaults to 1) on a small box. Workers elect one leader for the -# subscription notifier via a lockfile (THERMOGRAPH_SINGLETON_LOCK, set by the compose -# app service to /app/data/notifier.lock) so its timer-driven upstream sweep runs once, -# not once per worker. ~200 MB RAM per worker. +# past brief outages. Prod's Swarm `web` service defaults to 4 (WEB_WORKERS, also the +# autoscaler's per-replica count); beta's defaults to 2 (BETA_WEB_WORKERS) — a +# rehearsal environment, not a decision that beta needs less headroom per se. The +# base compose file (dev) also defaults to 4. Leave unset (defaults to 1) on a small +# box. Workers elect one leader for the subscription notifier via a lockfile +# (THERMOGRAPH_SINGLETON_LOCK, set by the compose/stack app service to +# /app/data/notifier.lock) so its timer-driven upstream sweep runs once, not once per +# worker. ~200 MB RAM per worker. WORKERS=4 # THERMOGRAPH_SINGLETON_LOCK arbitrates workers on ONE host. Under multi-host Swarm, diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index 86f9c53..cfebe28 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -13,11 +13,21 @@ # by deploy-dev.sh. This overlay must NOT reintroduce `build:`; it only relaxes # resource caps and LAN-exposes a port, same as the monorepo overlay did. # -# Differences from the prod stack (unchanged intent from the monorepo overlay): -# 1. backend is published on ALL interfaces (0.0.0.0:8137), not loopback, so -# phones and other devices on the Wi-Fi can reach the dev server directly -- -# dev has no Caddy in front (prod does, which is why the base file binds -# 127.0.0.1 only). +# Differences from the prod stack: +# 1. backend is published on ${DEV_BIND_ADDR}, defaulting to LOOPBACK. +# +# This used to be a flat 0.0.0.0:8137, from when dev ran on the operator's +# LAN box and the point was for phones on the Wi-Fi to reach it. Dev now +# runs on vps1, a public VPS, where 0.0.0.0 would publish whatever +# unreviewed branch is in flight to the entire internet — with no Caddy, no +# TLS and no auth in front of it. +# +# So the default is 127.0.0.1 (safe anywhere, including a laptop running +# `make dev-up`), and deploy.sh sets DEV_BIND_ADDR to the environment's mesh +# address from deploy/env-topology.sh — 10.10.0.2 on vps1. Dev is then +# reachable at http://10.10.0.2:8137 from anything on the WireGuard mesh and +# from nowhere else. There is deliberately no dev DNS record and no Caddy +# site block for it. # 2. frontend's port publish is dropped entirely -- dev has no Caddy to reach it # directly, so it stays compose-internal-only, reached solely through # backend's own reverse-proxy fallback (THERMOGRAPH_FRONTEND_BASE_INTERNAL, @@ -27,18 +37,21 @@ # prod's backend=4 / frontend=2 / db=2 allocation. `!reset` drops the base # value (both the top-level `cpus:` and the Swarm-style `deploy.resources` # block the base file carries for parity). -# 4. backend/daemon/lake get a second env_file entry pointing at a copy of the -# render under $APP_DIR (deploy-dev.sh sets THERMOGRAPH_SECRETS_ENV_FILE_MIRROR -# to produce it). This box's Docker is the snap package, confined to $HOME -- -# it can't see /etc/thermograph.env at all (not a permissions error, it just -# doesn't exist as far as snap-confined Docker is concerned), so the base -# file's env_file entry silently loads nothing here. compose appends env_file -# lists across overlays (last-wins on duplicate keys), so this is additive: -# prod/beta never set the mirror var, so they only ever get the base entry. +# 4. backend/daemon/lake get a second env_file entry pointing at an OPTIONAL +# copy of the render under $APP_DIR (`required: false`, so it is a no-op +# when absent). It exists for snap-packaged Docker, which is confined to +# $HOME and cannot see /etc/thermograph.env at all — not a permissions +# error; the file simply does not exist as far as snap-confined Docker is +# concerned, so the base file's env_file entry silently loads nothing. +# deploy-dev.sh now produces the mirror ONLY when it detects snap Docker, +# which vps1 (ordinary apt Docker) is not — there, dev reads +# /etc/thermograph.env like beta and prod do, and no plaintext copy of the +# render is written into the checkout. compose appends env_file lists across +# overlays (last-wins on duplicate keys), so this stays additive. services: backend: ports: !override - - "8137:8137" + - "${DEV_BIND_ADDR:-127.0.0.1}:8137:8137" cpus: !reset null deploy: !reset null env_file: diff --git a/infra/ops/ICEBERG-HANDOFF.md b/infra/ops/ICEBERG-HANDOFF.md index d27810b..9b015cc 100644 --- a/infra/ops/ICEBERG-HANDOFF.md +++ b/infra/ops/ICEBERG-HANDOFF.md @@ -5,6 +5,17 @@ already shipped (`infra/ops/dbq.sh`) — **mirror it**. This document is the contract, the environment facts you'll trip over, and how the result gets verified. +**Historical note:** `infra/ops/iceberg.sh` now exists, originally built +against the environment facts below as they stood before the vps1/vps2 split +— `beta` at `75.119.132.91` and `prod` at `169.58.46.181`, each its own box. +`iceberg.sh` (and `dbq.sh`) have since been updated to resolve their SSH +target and host facts from `deploy/env-topology.sh` instead of a hardcoded +table, so they already reflect the current split (vps1 = `75.119.132.91` = +dev + Forgejo + Grafana; vps2 = `169.58.46.181` = prod AND beta). This +document is left as a historical record of the original design contract, not +a live description of where each environment's engine runs — see +`infra/ops/README.md` for the current behavior. + ## Definition of done `infra/ops/iceberg.sh` exists and behaves like `dbq.sh`: @@ -36,6 +47,9 @@ new network exposure and no interactive steps. ## Environment facts you need +**As originally written** (pre vps1/vps2 split; kept for history — see the +note at the top of this document): + - Monorepo `emi/thermograph` (domain dirs: `backend/ frontend/ infra/ observability/`). Ops tooling lives in `infra/ops/`. - Environments: **dev** = LAN compose stack on the dev machine; **beta** = @@ -53,6 +67,24 @@ new network exposure and no interactive steps. services. Untested; verify before relying on it. - WireGuard mesh: prod `10.10.0.1`, beta `10.10.0.2`, dev `10.10.0.3`. - `rclone` and `age` are already installed on prod and beta. + +**Current facts, post vps1/vps2 split** (the boxes and mesh IPs above did not +move — only which environment runs where, and what the boxes are called): + +- **vps2** = `169.58.46.181` = mesh `10.10.0.1` = the box called "prod" above. + Runs **both** prod and beta now, as two separate Docker Swarm stacks — beta + is a Swarm service too today, not a compose exception, so the tunnelling + constraint above applies equally to it: `ssh -L` is impossible for beta now + as well, not just prod. +- **vps1** = `75.119.132.91` = mesh `10.10.0.2` = the box called "beta" above. + Runs Forgejo, Grafana/Loki/Alloy, and **dev** (its own Postgres container, + reached over SSH like any fleet host — not the LAN-box-local case the + original facts describe). +- **desktop** = mesh `10.10.0.3` = "the dev machine" above. Hosts no + Thermograph environment any more; it's a Swarm worker for flex capacity and + AI-model hosting. The "dev machine is a Swarm worker in the prod cluster" + fact still holds structurally (the desktop is still a worker node), but + nothing about Iceberg or the app database routes through it any more. - Reference implementation to copy: **`infra/ops/dbq.sh`** + `infra/ops/README.md`. ## Storage — almost certainly your warehouse diff --git a/infra/ops/README.md b/infra/ops/README.md index f7aeb7d..0c19dd0 100644 --- a/infra/ops/README.md +++ b/infra/ops/README.md @@ -11,34 +11,56 @@ infra/ops/dbq.sh beta -c '\dt' echo "select 1" | infra/ops/dbq.sh prod -f - ``` -Environments: `dev` (LAN compose stack on the dev machine), `beta` -(75.119.132.91), `prod` (169.58.46.181). Extra args pass straight through to +Environments: `dev` (own compose stack + own Postgres container, on vps1, +`75.119.132.91`), `beta` and `prod` (both on vps2, `169.58.46.181` — separate +roles/databases on the ONE shared TimescaleDB instance there). Host, SSH +target and container filter all come from `deploy/env-topology.sh` now, never +a hardcoded table — `dbq.sh` SSHes to vps1 for `dev` and to vps2 for +`beta`/`prod`, and beta (which has no db container of its own) resolves +prod's shared `thermograph_db` task. Extra args pass straight through to `psql` (`-tA`, `-x`, `--csv`, `-f`, …); stdin is forwarded. +**Known gap:** beta has no read-only role of its own yet — `deploy/db/provision-env-db.sh` +only provisions the environment's *owning* role (`thermograph_beta`), so a +`dbq.sh beta` query connects as `thermograph_beta` (which can write) rather +than a read-only role, unlike `dev`/`prod`. See `dbq.sh`'s own header for the +tracked follow-up (a `thermograph_beta_ro` role). + ### Why it execs into the container instead of using a connection string None of the databases are exposed over TCP — each listens only on its private -docker network. Prod's is a Swarm **overlay** (`10.0.2.0/24`) that the host -cannot route to, so `ssh -L` works for beta but is *impossible* for prod; -publishing 5432 would mean new ufw rules plus a Swarm endpoint change on -production. Running `psql` inside the db container works identically in all -three environments with no ports, no tunnels, and no infra changes. +docker network. Prod's and beta's is a Swarm **overlay** (`thermograph_internal`) +that the vps2 host itself cannot route to, so `ssh -L` is impossible for +either of them — treat beta exactly like prod here, now that it's a Swarm +service too, not the compose-network exception it used to be. Publishing 5432 +would mean new ufw rules plus a Swarm endpoint change on a live environment. +Running `psql` inside the db container works identically in all three +environments with no ports, no tunnels, and no infra changes. The prod container name is a Swarm task name that changes on every redeploy, so it is resolved at call time via `docker ps --filter name=…`, never hardcoded. ### Safety -Queries connect as **`thermograph_ro`** — a `NOSUPERUSER` role granted only -`pg_read_all_data`. Read-only is enforced by Postgres, not by convention: +`dev` and `prod` connect as **`thermograph_ro`** — a `NOSUPERUSER` role granted +only `pg_read_all_data`. Read-only is enforced by Postgres, not by convention: ``` $ infra/ops/dbq.sh prod -c "create table t(i int)" ERROR: permission denied for schema public ``` +**`beta` is the exception, today:** `deploy/db/provision-env-db.sh` only ever +provisions the environment's *owning* role (`thermograph_beta` — +`NOSUPERUSER`/`NOCREATEDB`/`NOCREATEROLE`, but it owns its own database and +schema) and revokes `CONNECT` from `PUBLIC` on `thermograph_beta`, so +`thermograph_ro` was never granted `CONNECT` there. A `dbq.sh beta` query +therefore connects as `thermograph_beta` until a beta-scoped read-only role +exists (tracked in `dbq.sh`'s own header as a TODO) — it can write to beta's +database, though never to prod's. + The app's own `thermograph` role is a superuser and is deliberately **not** used -by this tool. If the role is ever missing (fresh database), recreate it with: +by this tool. If `thermograph_ro` is ever missing (fresh instance), recreate it with: ```sql CREATE ROLE thermograph_ro LOGIN; @@ -68,15 +90,18 @@ The lake is **one shared warehouse** in Contabo object storage same credentials — so unlike Postgres there is no per-env database to reach, and the environments differ only in *where the engine runs*: a throwaway `docker run` of a small DuckDB image (duckdb CLI + `httpfs`/`iceberg` -extensions pre-installed) on the target box — locally for LAN dev, over SSH -for beta/prod. The image is built on the host the first time it's needed from -a Dockerfile embedded in the script; its tag is a hash of that Dockerfile, so -editing the script rolls every host forward on the next call. +extensions pre-installed) on the target box — vps1 for `dev`, vps2 for +`beta`/`prod`, resolved from `deploy/env-topology.sh` rather than a hardcoded +table (beta and prod are two Swarm stacks on the same vps2 box now). The +image is built on the host the first time it's needed from a Dockerfile +embedded in the script; its tag is a hash of that Dockerfile, so editing the +script rolls every host forward on the next call. -A container per query means no daemon, no listening port, no tunnel (prod's -overlay cannot be tunnelled), no firewall or Swarm changes — `sudo ss -ltnp` -is identical before and after a query. It also never execs into an app -container and resolves no container names, so prod redeploys can't break it. +A container per query means no daemon, no listening port, no tunnel (beta's +and prod's Swarm overlay cannot be tunnelled), no firewall or Swarm changes — +`sudo ss -ltnp` is identical before and after a query. It also never execs +into an app container and resolves no container names, so a redeploy can't +break it. There is no catalog service to run or expose: an Iceberg table's current state is fully described by the newest metadata JSON under diff --git a/infra/ops/dbq.sh b/infra/ops/dbq.sh index c2c0cac..0837c4f 100755 --- a/infra/ops/dbq.sh +++ b/infra/ops/dbq.sh @@ -7,26 +7,43 @@ # echo "select 1" | infra/ops/dbq.sh prod -f - # # Why exec-into-the-container instead of a connection string: -# none of the databases are exposed over TCP. Each listens only on its private -# docker network -- and prod's is a Swarm *overlay* (10.0.2.0/24) that the host -# itself cannot route to, so `ssh -L` works for beta but is impossible for prod. -# Publishing 5432 would mean new firewall + Swarm endpoint changes on production. -# Running psql *inside* the db container works identically in all three -# environments with no ports, no tunnels and no infra changes -- locally for LAN -# dev, over SSH for beta/prod. +# none of the databases are exposed over TCP. Dev's listens only on its own +# private docker network on vps1; prod and beta share ONE instance on prod's +# Swarm *overlay* (10.0.2.0/24) on vps2, which the host itself cannot route +# to, so `ssh -L` is impossible for either. Publishing 5432 would mean new +# firewall + Swarm endpoint changes on production. Running psql *inside* the +# db container works identically in all three environments with no ports, no +# tunnels and no infra changes -- over SSH to vps1 for dev, to vps2 for +# beta/prod. # -# Safety: always connects as `thermograph_ro`, a NOSUPERUSER role granted only -# pg_read_all_data. Read-only is enforced by Postgres itself, not by convention, -# so a stray INSERT/DDL fails with "permission denied" even against prod. (The -# app's own `thermograph` role is a superuser -- deliberately not used here.) +# Host, SSH target and container filter all come from env-topology.sh, never +# a hardcoded table -- beta and prod are two Swarm stacks on the SAME box +# (vps2) now, and beta has no db container of its own: it reaches prod's +# `thermograph_db` task, the one shared TimescaleDB instance. +# +# Safety: every environment connects as a READ-ONLY role, never as the role the +# app itself uses -- a stray INSERT/DDL fails with "permission denied" even +# against prod. The role is the environment's own owning role with an `_ro` +# suffix, which resolves to the long-standing `thermograph_ro` for prod and dev +# and to `thermograph_beta_ro` for beta. +# +# That suffix convention is why beta needed no special case. It would have been +# easy to give beta one: `CONNECT` is revoked from `PUBLIC` on +# `thermograph_beta`, so prod's `thermograph_ro` cannot reach it, and the +# tempting shortcut was to let beta queries connect as `thermograph_beta` -- +# the role that OWNS beta's database. That would have quietly made ad-hoc +# queries read-write on beta alone. deploy/db/provision-env-db.sh provisions the +# `_ro` role for every environment instead. # # Any extra arguments are passed straight through to psql, so -tA, -c, -f, -x, # --csv etc. all work. Stdin is forwarded, so `-f -` reads piped SQL. set -euo pipefail +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../deploy/env-topology.sh" + KEYFILE="${THERMOGRAPH_AGENT_KEY:-$HOME/.ssh/thermograph_agent_ed25519}" -DB_USER="${THERMOGRAPH_DB_QUERY_USER:-thermograph_ro}" -DB_NAME="${THERMOGRAPH_DB_NAME:-thermograph}" usage() { sed -n '2,9p' "$0" | sed 's/^# \{0,1\}//' >&2 @@ -39,13 +56,33 @@ env_name="${1:-}" shift case "$env_name" in - dev) filter=thermograph-dev-db ; ssh_target= ;; - beta) filter=thermograph-db-1 ; ssh_target=agent@75.119.132.91 ;; - prod) filter=thermograph_db ; ssh_target=agent@169.58.46.181 ;; + dev|beta|prod) ;; -h|--help) usage ;; *) echo "dbq: unknown environment '$env_name' (want: dev|beta|prod)" >&2; exit 2 ;; esac +thermograph_topology "$env_name" + +ssh_target="$TG_SSH_TARGET" + +# Container filter: a Swarm task name (shared across prod and beta -- beta has +# no db task of its own) or a compose container name, per the environment's +# deploy mode. +if [ "$TG_DEPLOY_MODE" = stack ]; then + filter="$TG_DB_SERVICE" +else + filter="${TG_COMPOSE_PROJECT}-${TG_DB_SERVICE}" +fi + +DB_NAME="${THERMOGRAPH_DB_NAME:-$TG_DB_NAME}" +if [ -n "${THERMOGRAPH_DB_QUERY_USER:-}" ]; then + DB_USER="$THERMOGRAPH_DB_QUERY_USER" +else + # thermograph_ro for prod/dev, thermograph_beta_ro for beta -- provisioned by + # deploy/db/provision-env-db.sh. Never the owning role. + DB_USER="${TG_DB_USER}_ro" +fi + [ $# -gt 0 ] || usage # Quote the psql arguments so they survive the remote shell intact. diff --git a/infra/ops/iceberg.sh b/infra/ops/iceberg.sh index 693e54d..cfabf75 100755 --- a/infra/ops/iceberg.sh +++ b/infra/ops/iceberg.sh @@ -9,9 +9,12 @@ # Why an ephemeral container instead of a query service: the lake is one shared # warehouse in Contabo object storage (s3://era5-thermograph/iceberg), readable # from every environment, so the environments differ only in WHERE the engine -# runs -- a throwaway `docker run` of a small DuckDB image on the target box. -# No daemon, no listening port, no tunnel (prod's overlay cannot be tunnelled), -# and no container names to resolve, so prod redeploys cannot break it. +# runs -- a throwaway `docker run` of a small DuckDB image on the target box +# (vps1 for dev, vps2 for beta/prod -- the SSH target comes from +# env-topology.sh, never a hardcoded table; beta and prod are two Swarm stacks +# on the SAME vps2 box now). No daemon, no listening port, no tunnel (prod's +# overlay cannot be tunnelled), and no container names to resolve, so prod +# redeploys cannot break it. # # There is no catalog service: a table's current state is its newest metadata # JSON under //metadata/, resolved at call time and exposed @@ -34,6 +37,10 @@ # read as piped SQL. A single bare argument is treated as `-c` SQL. set -euo pipefail +SELF_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=infra/deploy/env-topology.sh +. "$SELF_DIR/../deploy/env-topology.sh" + KEYFILE="${THERMOGRAPH_AGENT_KEY:-$HOME/.ssh/thermograph_agent_ed25519}" usage() { @@ -47,13 +54,17 @@ env_name="${1:-}" shift case "$env_name" in - dev) ssh_target= ;; - beta) ssh_target=agent@75.119.132.91 ;; - prod) ssh_target=agent@169.58.46.181 ;; + dev|beta|prod) ;; -h|--help) usage ;; *) echo "iceberg: unknown environment '$env_name' (want: dev|beta|prod)" >&2; exit 2 ;; esac +# SSH target comes from env-topology.sh, not a hardcoded table: beta and prod +# are two Swarm stacks on the SAME box (vps2) now, so a stale beta entry here +# would silently point a beta query at the wrong host. +thermograph_topology "$env_name" +ssh_target="$TG_SSH_TARGET" + [ $# -gt 0 ] || usage # The engine image: duckdb CLI with the httpfs + iceberg extensions installed diff --git a/infra/terraform/README.md b/infra/terraform/README.md index 77a4df5..47717e5 100644 --- a/infra/terraform/README.md +++ b/infra/terraform/README.md @@ -10,27 +10,46 @@ scaffold" below) — scaffold-only today, no live resources until you opt in. ## What it manages -One reusable module (`modules/thermograph-host`) is instantiated per host via -`for_each` (`main.tf`'s `local.all_hosts`, merging `var.hosts` — SSH-managed, -already-existing boxes — with any `var.gcp_hosts` Terraform created itself). -This config manages **two VPS hosts** today: +One reusable module (`modules/thermograph-host`) is instantiated per +**(host, environment)** pair via `for_each` (`main.tf`'s `local.all_hosts`, +merging `var.hosts` — SSH-managed, already-existing boxes — with any +`var.gcp_hosts` Terraform created itself). `var.hosts` is keyed by HOST, not by +environment, because **vps2 alone runs two environments** (prod and beta) as +separate Docker Swarm stacks on the same box — see the `hosts` variable's file +header in `variables.tf` for why that stopped being "one entry = one +environment" and what still needs to change in `main.tf` to consume the new +shape (flagged there as a `TODO(cutover)`). This config manages **two VPS +hosts**, three environments, today: -| key | role | VPS | branch | domain | notes | -|--------|--------|-------------------------|-----------|-------------------|------------------------------------| -| `prod` | prod | NEW 48 GB / 12-core box (`169.58.46.181`) | `release` (of the APP repos — see `backend_image_tag` / `frontend_image_tag`) | `thermograph.org` | Caddy TLS; sized up (8/8/4/16g) | -| `beta` | beta | old box `75.119.132.91` | `main` (of the APP repo) | `beta.thermograph.org` | Caddy TLS; also hosts Forgejo | +| host | environment | VPS | branch | domain | notes | +|--------|-------------|--------------------------------------------|--------|------------------------|----------------------------------------------| +| `vps1` | `dev` | `75.119.132.91` | `dev` (of the APP repos — see `backend_image_tag` / `frontend_image_tag`) | none (mesh-only) | Also hosts Forgejo, Grafana/Loki/Alloy, emigriffith.dev — not Terraform-managed | +| `vps2` | `prod` | 48 GB / 12-core box (`169.58.46.181`) | `main` | `thermograph.org` | Caddy TLS (this module owns `/etc/caddy/Caddyfile`); sized up (8/8/4/16g) | +| `vps2` | `beta` | the SAME box as prod (`169.58.46.181`) | `main` | none (see below) | Separate Swarm stack + checkout, shares prod's TimescaleDB instance on a separate database/role | -Each host's own checkout on disk (`app_dir`, `git_branch`) is **this infra -repo**, not the app repo — the "branch" column above is which app-repo tag a -host is meant to track conceptually; the actual pinned versions are -`var.hosts[*].backend_image_tag` / `frontend_image_tag` (e.g. `"sha-<12 hex>"` each — -the app is two separately-published images, `emi/thermograph-backend/app` and -`emi/thermograph-frontend/app`), since the host has no app checkout to derive a tag -from. The LAN dev server is **out of scope here** — it -builds from source via the app repo's `deploy/deploy-dev.sh` (a self-hosted -Forgejo Actions runner), not Terraform. +Only ONE environment per host may set a `domain`: the module installs a full +`/etc/caddy/Caddyfile`, and a second `terraform apply` with its own `domain` set +would silently overwrite the first's site instead of adding to it (there is no +merge). Prod claims that slot on vps2 today, so beta's public reverse-proxy (if +and when `beta.thermograph.org` is exposed) has to be a site block in vps2's +shared, hand-maintained Caddy config instead — the same pattern +`deploy/forgejo/docker-stack.yml` already uses to put `git.thermograph.org` in +front of Forgejo's loopback port on vps1, alongside that box's own Caddy-fronted +`emigriffith.dev`. -Per host, over SSH provisioners, Terraform: +Each environment's own checkout on disk (`app_dir`, `git_branch`) is **this +infra repo**, not the app repo — the "branch" column above is which app-repo +tag an environment is meant to track conceptually; the actual pinned versions +are `var.hosts[*].environments[*].backend_image_tag` / `frontend_image_tag` +(e.g. `"sha-<12 hex>"` each — the app is two separately-published images, +`emi/thermograph-backend/app` and `emi/thermograph-frontend/app`), since the +host has no app checkout to derive a tag from. `app_dir` is required with no +default specifically so vps2's two environments can never collide on the same +checkout path (prod: `/opt/thermograph`, beta: `/opt/thermograph-beta`). The +LAN-laptop dev rehearsal (`make dev-up`) is **out of scope here** — it's a +local-only compose overlay, not this Terraform-managed `dev` environment. + +Per (host, environment), over SSH provisioners, Terraform: - installs Docker + the compose plugin if missing; - configures a `ufw` firewall (22/80/443 always; on a host with **no** domain it also @@ -104,7 +123,11 @@ terraform plan terraform apply ``` -Target one host with `-target='module.host["beta"]'` if you want to apply to just one. +Target one host with `-target='module.host["prod"]'` if you want to apply to just one +(the module's `for_each` key changes once `main.tf` is updated to flatten `var.hosts`'s +new per-host `environments` map — see the `TODO(cutover)` on the `hosts` variable in +`variables.tf` — at which point the key for beta becomes something like +`module.host["vps2-beta"]`, not `module.host["beta"]`). ## Secrets @@ -150,11 +173,15 @@ Terraform brings the stack up — don't let Terraform recreate containers mid-mi ## Assumptions / notes -- **beta has no public domain by default.** With `compose_files = ["docker-compose.yml"]` - the app binds `127.0.0.1:8137` (loopback), so opening the port in `ufw` alone does not - expose it. Reach beta via an SSH tunnel, or set `domain = "beta.thermograph.org"` (adds - Caddy TLS) — or add the `0.0.0.0`-publishing dev overlay to `compose_files` — to make - it reachable. `COOKIE_SECURE` is auto-set to `0` when there's no domain (a Secure +- **beta has no public domain here, and can't without a different Caddy strategy.** + With `compose_files = ["docker-compose.yml"]` the app binds `127.0.0.1:8137` + (loopback), so opening the port in `ufw` alone does not expose it — and setting + `domain` on beta the way prod does would make this module overwrite the SAME + `/etc/caddy/Caddyfile` prod's apply just installed (see the table above). Reach + beta via an SSH tunnel, or front it with a site block in vps2's own + hand-maintained Caddy config (outside Terraform), the same way Forgejo's + `git.thermograph.org` reaches Forgejo's loopback port on vps1. + `COOKIE_SECURE` is auto-set to `0` when there's no domain (a Secure cookie is never sent over plain HTTP) and `1` behind Caddy TLS. - The rendered Caddyfile only reverse-proxies the app. The repo's `deploy/Caddyfile` additionally serves the `emigriffith.dev` portfolio and legacy redirects; those are diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf index b5a2171..68e2ac2 100644 --- a/infra/terraform/main.tf +++ b/infra/terraform/main.tf @@ -44,13 +44,39 @@ module "gcp_vm" { } locals { - # Every host this config manages, SSH-only (var.hosts) plus GCP-created (whose - # `host` is filled in from the VM Terraform just created) — one unified map so a - # single `module.host` for_each below handles both without duplicating any - # provisioning logic. GCP hosts always use a named size tier (simpler than + # ONE MODULE INSTANCE PER (HOST, ENVIRONMENT) PAIR, not per host. + # + # var.hosts is keyed by machine (vps1, vps2) and each machine carries an + # `environments` map, because vps2 runs prod AND beta. The module below + # provisions an ENVIRONMENT — a checkout, a branch, image tags, a Caddy site, + # container sizing — so it needs one instance per environment, with the two + # instances on vps2 sharing that machine's SSH identity. + # + # Flattened to keys like "vps2-prod" and "vps2-beta". Keeping them distinct + # module instances is what makes `app_dir` (required, no default in + # variables.tf) do its job: two environments on one box get two explicitly + # different checkouts, so neither apply can `git reset --hard` the other's + # tree. + ssh_environments = merge([ + for hname, h in var.hosts : { + for ename, e in h.environments : + "${hname}-${ename}" => merge(e, { + host = h.host + ssh_user = h.ssh_user + ssh_private_key_path = h.ssh_private_key_path + }) + } + ]...) + + # Every environment this config manages, SSH-only (flattened above) plus + # GCP-created (whose `host` is filled in from the VM Terraform just created) — + # one unified map so a single `module.host` for_each below handles both without + # duplicating any provisioning logic. GCP hosts stay one-entry-per-machine: + # nothing creates two environments on a created VM today, and the variable + # keeps its flat shape. They always use a named size tier (simpler than # exposing the four raw sizing fields on that variable too). all_hosts = merge( - var.hosts, + local.ssh_environments, { for name, h in var.gcp_hosts : name => merge(h, { host = module.gcp_vm[name].external_ip diff --git a/infra/terraform/modules/thermograph-host/variables.tf b/infra/terraform/modules/thermograph-host/variables.tf index c41f85b..ceac2a6 100644 --- a/infra/terraform/modules/thermograph-host/variables.tf +++ b/infra/terraform/modules/thermograph-host/variables.tf @@ -21,7 +21,7 @@ variable "ssh_private_key_path" { } variable "role" { - description = "\"prod\" | \"dev\" — informational." + description = "\"prod\" | \"beta\" | \"dev\" — informational. One module instance is one (host, environment) pair, so vps2 (which runs both prod and beta) gets two instances, each with its own role/app_dir/image tags -- see the root module's `hosts` variable." type = string } diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf index 2336b73..0a5cba8 100644 --- a/infra/terraform/outputs.tf +++ b/infra/terraform/outputs.tf @@ -4,7 +4,7 @@ output "prod_url" { } output "hosts" { - description = "Per-host summary: the address managed and its role." + description = "Per-environment summary, keyed \"-\" (vps2 carries two): the machine managed and the role running on it." value = { for name, mod in module.host : name => { host = mod.host diff --git a/infra/terraform/terraform.tfvars.example b/infra/terraform/terraform.tfvars.example index a2ea7ed..7b80ef5 100644 --- a/infra/terraform/terraform.tfvars.example +++ b/infra/terraform/terraform.tfvars.example @@ -1,8 +1,8 @@ -# Copy to terraform.tfvars and fill in real IPs + credentials. +# Copy to terraform.tfvars and fill in real credentials. # cp terraform.tfvars.example terraform.tfvars # terraform.tfvars is gitignored (repo_url may carry a credential, and om_rclone_conf # always does — both land in local state). NEVER commit real values. -# + # App secrets (POSTGRES_PASSWORD, THERMOGRAPH_AUTH_SECRET, VAPID keys, # REGISTRY_TOKEN, Discord/SMTP creds, ...) are NOT here anymore -- they live in # the SOPS+age vault (../deploy/secrets/*.yaml), rendered at deploy time. See @@ -11,54 +11,99 @@ # --------------------------------------------------------------------------------- # Hosts # --------------------------------------------------------------------------------- -# Two VPS hosts. (The `dev` branch deploys to the LAN dev server via the app repo's -# deploy/deploy-dev.sh — that box is NOT managed by Terraform.) +# Two VPS boxes, keyed by HOST (vps1, vps2) — NOT by environment, because vps2 +# alone carries two (prod AND beta). Each host lists its SSH identity once, then +# an `environments` map for whatever runs on it. (The `dev` branch deploys to +# vps1 the same way in reality, but the LAN-laptop `make dev-up` rehearsal is NOT +# managed by Terraform at all.) hosts = { - # Production: the NEW 48 GB / 12-core VPS serving thermograph.org. - prod = { - host = "REPLACE_WITH_NEW_VPS_IP" # <-- the new prod VPS IP/hostname - ssh_user = "agent" - ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" - role = "prod" - git_branch = "main" # this INFRA repo's branch -- see *_image_tag for the app versions - backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" # e.g. "sha-abcdef012345" -- from thermograph-backend build-push.yml - frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" # e.g. "sha-012345abcdef" -- from thermograph-frontend build-push.yml - domain = "thermograph.org" # Caddy TLS in front, app on loopback - compose_files = ["docker-compose.yml"] - app_dir = "/opt/thermograph" - # "large" is the named size tier for this box (locals.sizes in main.tf) — same - # numbers as hand-picking workers=8/app_cpus=8/db_cpus=4/db_memory="16g" below, - # via the shortcut. The Postgres internal budget scales from db_memory - # automatically (deploy/db/init/20-tuning.sh); no separate tuning edit. - size = "large" - # Self-host the ERA5 archive here: layers docker-compose.openmeteo.yml and - # provisions the rclone mount of the object-storage bucket (om_* vars below). - openmeteo = true - om_data_dir = "/mnt/om-archive" - } - - # Beta / testing: the OLD VPS, repurposed. - beta = { + # vps1: Forgejo (git+CI+registry), Grafana/Loki/Alloy, emigriffith.dev, and the + # `dev` environment. One environment today; the shape still nests it so a + # second one (there is none planned) would never have to fight this host's + # SSH identity. + vps1 = { host = "75.119.132.91" ssh_user = "agent" ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" - role = "beta" - git_branch = "main" - backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" - frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" - # No public domain by default: no Caddy/TLS, firewall opens the app port. NOTE: - # with compose_files = ["docker-compose.yml"] the app binds 127.0.0.1 only, so - # until you either set a domain (e.g. "beta.thermograph.org", which fronts it with - # Caddy) or add the 0.0.0.0-publishing dev overlay, reach it via an SSH tunnel. - domain = "" - compose_files = ["docker-compose.yml"] - app_dir = "/opt/thermograph" - # Explicit numbers, not a size tier — both styles work on any host; a tier is - # purely an opt-in shortcut (see prod's `size = "large"` above). - workers = 4 - app_cpus = 4 - db_cpus = 2 - db_memory = "8g" + + environments = { + dev = { + role = "dev" + git_branch = "dev" # the only environment that tracks `dev`; beta/prod track `main` + backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" + frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" + # Mesh-only: no public domain, no Caddy/TLS. Reachable at + # http://10.10.0.2:8137 from the WireGuard mesh only. + domain = "" + compose_files = ["docker-compose.yml"] + app_dir = "/opt/thermograph-dev" + workers = 4 + app_cpus = 4 + db_cpus = 2 + db_memory = "8g" + } + } + } + + # vps2: prod (thermograph.org) AND beta (beta.thermograph.org) as two separate + # Swarm stacks on the SAME 48 GB / 12-core box — plus Centralis, Postfix and + # the backups (not Terraform-managed). ONE shared TimescaleDB instance serves + # both app_dirs below, on separate databases/roles (deploy/db/provision-env-db.sh); + # each environment's db_cpus/db_memory here sizes only that environment's own + # app-container caps, not a second database. + vps2 = { + host = "169.58.46.181" + ssh_user = "agent" + ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" + + environments = { + prod = { + role = "prod" + git_branch = "main" # this INFRA repo's branch -- see *_image_tag for the app versions + backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" # e.g. "sha-abcdef012345" -- from thermograph-backend build-push.yml + frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" # e.g. "sha-012345abcdef" -- from thermograph-frontend build-push.yml + domain = "thermograph.org" # Caddy TLS in front, app on loopback + compose_files = ["docker-compose.yml"] + # MUST differ from beta's app_dir below -- see variables.tf's file header. + app_dir = "/opt/thermograph" + # "large" is the named size tier for this box (locals.sizes in main.tf) — same + # numbers as hand-picking workers=8/app_cpus=8/db_cpus=4/db_memory="16g" below, + # via the shortcut. The Postgres internal budget scales from db_memory + # automatically (deploy/db/init/20-tuning.sh); no separate tuning edit. + size = "large" + # Self-host the ERA5 archive here: layers docker-compose.openmeteo.yml and + # provisions the rclone mount of the object-storage bucket (om_* vars below). + openmeteo = true + om_data_dir = "/mnt/om-archive" + } + + beta = { + role = "beta" + git_branch = "main" + backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" + frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" + # "" ON PURPOSE, unlike prod above: modules/thermograph-host installs a + # FULL /etc/caddy/Caddyfile per environment with a domain set, and only + # ONE environment on a box can own that file. Prod already claims it on + # this host, so beta.thermograph.org's public reverse-proxy has to be a + # site block in vps2's shared, hand-maintained Caddy instead (the same + # pattern deploy/forgejo/docker-stack.yml uses for git.thermograph.org + # on vps1) -- not something this module can render for a second + # environment on the same host. + domain = "" + compose_files = ["docker-compose.yml"] + # MUST differ from prod's app_dir above -- a SECOND checkout on the same + # box, so a `git reset --hard` in one deploy can never yank the tree out + # from under the other's running deploy. + app_dir = "/opt/thermograph-beta" + # Explicit numbers, not a size tier — both styles work on any environment; a + # tier is purely an opt-in shortcut (see prod's `size = "large"` above). + workers = 4 + app_cpus = 4 + db_cpus = 2 + db_memory = "8g" + } + } } # UAT: an ephemeral, single-node environment — same images/topology shape as @@ -68,20 +113,27 @@ hosts = { # host = "REPLACE_WITH_UAT_VM_IP" # ssh_user = "agent" # ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" - # role = "uat" - # git_branch = "main" - # backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" - # frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" - # domain = "" - # compose_files = ["docker-compose.yml"] - # app_dir = "/opt/thermograph" - # size = "nano" + # environments = { + # uat = { + # role = "uat" + # git_branch = "main" + # backend_image_tag = "REPLACE_WITH_BACKEND_IMAGE_TAG" + # frontend_image_tag = "REPLACE_WITH_FRONTEND_IMAGE_TAG" + # domain = "" + # compose_files = ["docker-compose.yml"] + # app_dir = "/opt/thermograph" + # size = "nano" + # } + # } # } } -# GCP-created hosts — SCAFFOLD ONLY, empty by default. Populate an entry to have -# Terraform actually create a GCP Compute Engine VM (see modules/gcp-host); until -# then no google_* resource is planned and no GCP credentials are needed. Example: +# GCP-created hosts — SCAFFOLD ONLY, empty by default, and still keyed one entry +# per ENVIRONMENT (see variables.tf's TODO(cutover) note on gcp_hosts — these are +# hypothetical single-purpose boxes, not vps1/vps2, so they don't need the +# host/environment nesting above). Populate an entry to have Terraform actually +# create a GCP Compute Engine VM (see modules/gcp-host); until then no google_* +# resource is planned and no GCP credentials are needed. Example: # gcp_hosts = { # gcp-uat = { # project = "REPLACE_WITH_GCP_PROJECT_ID" @@ -108,7 +160,7 @@ hosts = { # repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/emi/thermograph-infra.git" # --------------------------------------------------------------------------------- -# Self-hosted Open-Meteo archive (only used by hosts with openmeteo = true) -------- +# Self-hosted Open-Meteo archive (only used by environments with openmeteo = true) # --------------------------------------------------------------------------------- # The ERA5 .om archive lives in an object-storage bucket, rclone-mounted on the host. # om_rclone_conf holds bucket credentials (sensitive; lands in state — keep out of git). diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 8386d75..d6c187a 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -1,69 +1,109 @@ # --------------------------------------------------------------------------------- # Hosts # --------------------------------------------------------------------------------- -# One entry per VPS. The same module is instantiated for each (see main.tf's -# for_each). This config manages TWO hosts: prod (new 48 GB / 12-core VPS, branch -# `release`, thermograph.org) and beta (the old VPS 75.119.132.91, branch `main`, -# no public domain by default). The `dev` branch deploys to the LAN dev server via -# deploy/deploy-dev.sh (a self-hosted runner, in the app repo) and is NOT managed here. +# Keyed by HOST (vps1, vps2), NOT by environment: vps2 alone runs TWO environments +# (prod AND beta) as separate Swarm stacks on the SAME box, so "one entry = one +# environment" broke the moment that became true — two entries that both defaulted +# `app_dir` to "/opt/thermograph" would let a prod apply and a beta apply stomp the +# identical checkout. Each host now carries its SSH identity ONCE (host IP, +# ssh_user, key) plus an `environments` map of everything that varies per +# environment running on it — checkout path, git branch, image tags, domain, +# sizing. `app_dir` is REQUIRED with no default for exactly this reason: two +# environments on one host must be given two explicitly DIFFERENT paths (see +# terraform.tfvars.example), so their checkouts — and a `git reset --hard` in one +# of them — can never collide. # -# A host with `domain = ""` gets no Caddy/TLS: the app port is opened on the firewall -# and the app is reached directly (beta does this by default; set a domain to front -# it with Caddy TLS). +# vps1 (75.119.132.91): Forgejo (git+CI+registry), Grafana/Loki/Alloy, +# emigriffith.dev, and the `dev` environment (its own Postgres, mesh-only, no +# public DNS/Caddy site). vps2 (169.58.46.181): `prod` (thermograph.org) and +# `beta` (beta.thermograph.org) — Centralis, Postfix and the backups also live +# here — sharing the ONE TimescaleDB instance both app environments use, on +# separate databases/roles (deploy/db/provision-env-db.sh); this variable's +# per-environment db_cpus/db_memory size only that environment's app-container +# caps, not a second database (see deploy/db/init/20-tuning.sh). The LAN dev +# branch (`make dev-up`, a laptop-local rehearsal) deploys nowhere via Terraform +# and is NOT managed here. # -# Resource sizing (workers / app_cpus / db_cpus / db_memory) defaults to the historical -# 4 / 4 / 2 / 8g budget (right for beta). The prod box is 48 GB / 12 cores — raise these -# there (the example uses app_cpus 8, db_cpus 4, db_memory "16g"). The Postgres internal -# budget scales from db_memory automatically (deploy/db/init/20-tuning.sh), so no -# separate tuning edit is needed to exploit the RAM. +# An environment with `domain = ""` gets no Caddy/TLS: its app port is opened on +# the firewall and it's reached directly. +# +# Resource sizing (workers / app_cpus / db_cpus / db_memory), per environment, +# defaults to the historical 4 / 4 / 2 / 8g budget (right for beta). vps2 is 48 GB +# / 12 cores — raise these for prod (the example uses app_cpus 8, db_cpus 4, +# db_memory "16g", via the "large" size tier). The Postgres internal budget scales +# from db_memory automatically (deploy/db/init/20-tuning.sh), so no separate +# tuning edit is needed to exploit the RAM. +# +# TODO(cutover): main.tf's `module.host` for_each (via `local.all_hosts`) still +# expects ONE flat entry per key, reading `each.value.host`/`.role`/`.app_dir`/… +# directly. It needs a flattening step ahead of that for_each — one derived entry +# per (host, environment) pair, keyed e.g. "-" ("vps2-prod" and +# "vps2-beta" as two separate module instances sharing vps2's host/ssh_user/ +# ssh_private_key_path but each with its own app_dir/role/image tags/sizing). +# This variable's new shape is not yet wired into main.tf's module block. variable "hosts" { - description = "Map of hosts to manage, keyed by a short name (e.g. \"prod\", \"beta\")." + description = "Map of VPS boxes to manage, keyed by a short host name (\"vps1\", \"vps2\") — NOT by environment. SSH identity is per-host; everything environment-specific lives in that host's `environments` map." type = map(object({ host = string # IP or hostname to SSH to ssh_user = optional(string, "deploy") # SSH login user ssh_private_key_path = string # path to the private key for that user - role = string # "prod" | "beta" (informational + outputs) - git_branch = string # this INFRA repo's branch the checkout is reset to - # Which app images to run, e.g. "sha-<12 hex>" (each matching build-push.yml's tag - # for the commit that app repo built) or a semver tag on a release push. The app is - # TWO separately-published images now — emi/thermograph-backend/app and - # emi/thermograph-frontend/app — pinned independently. Required, no default: the - # host's checkout is this infra repo, not the app repos, so there's no "current - # commit" to derive a tag from; both must be explicit. Bump these (via a normal - # tfvars edit + apply) whenever an app repo ships a commit you want this host - # running; the infra repo's own git_branch is independent and rarely needs to change. - backend_image_tag = string - frontend_image_tag = string - domain = optional(string, "") # public domain; "" => no Caddy/TLS - compose_files = optional(list(string), ["docker-compose.yml"]) - app_dir = optional(string, "/opt/thermograph") # checkout path on the host - workers = optional(number, 4) # uvicorn workers (WORKERS) - app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) - db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) - db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) - # A named size tier (see locals.sizes in main.tf: nano/small/medium/large) — - # when set, overrides the four fields above with the tier's preset. Leave - # null (default) to keep hand-picking workers/app_cpus/db_cpus/db_memory, - # as prod/beta already do below. - size = optional(string, null) - # The floating tag matches today's behavior everywhere until you pin it. Pin to - # an exact minor (SELECT extversion FROM pg_extension WHERE extname='timescaledb' - # on the live DB) before any host of this stack could ever replicate with - # another — a floating tag risks mismatched extension minors, which blocks a - # physical replica (hop-1 cutover runbook hazard #7). Use the SAME tag everywhere. - timescaledb_tag = optional(string, "latest-pg18") - # Self-host the ERA5 archive (docker-compose.openmeteo.yml + a host rclone mount - # of the object-storage bucket). Only the self-hosting host (prod) sets true. - openmeteo = optional(bool, false) - om_data_dir = optional(string, "/mnt/om-archive") # host rclone mount point (OM_DATA_DIR) + # One entry per environment running on this host, keyed by environment name + # ("prod" | "beta" | "dev"). A host normally carries one; vps2 carries two. + environments = map(object({ + role = string # "prod" | "beta" | "dev" (informational + outputs; should match the map key above) + git_branch = string # this INFRA repo's branch the checkout is reset to + # Which app images to run, e.g. "sha-<12 hex>" (each matching build-push.yml's tag + # for the commit that app repo built) or a semver tag on a release push. The app is + # TWO separately-published images now — emi/thermograph-backend/app and + # emi/thermograph-frontend/app — pinned independently. Required, no default: the + # host's checkout is this infra repo, not the app repos, so there's no "current + # commit" to derive a tag from; both must be explicit. Bump these (via a normal + # tfvars edit + apply) whenever an app repo ships a commit you want this + # environment running; the infra repo's own git_branch is independent and rarely + # needs to change. + backend_image_tag = string + frontend_image_tag = string + domain = optional(string, "") # public domain for this environment; "" => no Caddy/TLS + compose_files = optional(list(string), ["docker-compose.yml"]) + # REQUIRED, no default — see the file header: two environments on one host + # must never be able to default to the same checkout path. + app_dir = string # checkout path on the host + workers = optional(number, 4) # uvicorn workers (WORKERS) + app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) + db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) + db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) + # A named size tier (see locals.sizes in main.tf: nano/small/medium/large) — + # when set, overrides the four fields above with the tier's preset. Leave + # null (default) to keep hand-picking workers/app_cpus/db_cpus/db_memory, + # as prod/beta already do below. + size = optional(string, null) + # The floating tag matches today's behavior everywhere until you pin it. Pin to + # an exact minor (SELECT extversion FROM pg_extension WHERE extname='timescaledb' + # on the live DB) before any host of this stack could ever replicate with + # another — a floating tag risks mismatched extension minors, which blocks a + # physical replica (hop-1 cutover runbook hazard #7). Use the SAME tag everywhere. + timescaledb_tag = optional(string, "latest-pg18") + # Self-host the ERA5 archive (docker-compose.openmeteo.yml + a host rclone mount + # of the object-storage bucket). Only the self-hosting environment (prod) sets true. + openmeteo = optional(bool, false) + om_data_dir = optional(string, "/mnt/om-archive") # host rclone mount point (OM_DATA_DIR) + })) })) } -# GCP-created hosts, keyed the same way as `hosts`. Default {} => zero GCP resources -# planned and the google provider is never actually invoked (see versions.tf and -# modules/gcp-host). Populate an entry to have Terraform create the VM itself; its -# output IP then feeds into the SAME thermograph-host module every SSH-managed host -# uses (main.tf), so provisioning logic is never duplicated between providers. +# GCP-created hosts, keyed the same way `hosts` used to be — one entry per +# environment, since these are hypothetical single-purpose boxes (e.g. a future +# UAT VM), not vps1/vps2, so they don't need the host/environment split above. +# Default {} => zero GCP resources planned and the google provider is never +# actually invoked (see versions.tf and modules/gcp-host). Populate an entry to +# have Terraform create the VM itself; its output IP then feeds into the SAME +# thermograph-host module every SSH-managed host uses (main.tf), so provisioning +# logic is never duplicated between providers. +# TODO(cutover): main.tf's `local.all_hosts = merge(var.hosts, ...)` currently +# merges this flat shape with `var.hosts` into one map for `module.host`'s +# for_each. Once `var.hosts` is nested (above), that merge needs to target the +# same flattened (host, environment) shape this variable already has — e.g. by +# treating each `gcp_hosts` entry as its own single-environment host. variable "gcp_hosts" { description = "Map of hosts for Terraform to CREATE on GCP (Compute Engine), keyed the same way as `hosts`. Empty by default -- no live GCP resources exist yet; this is a scaffold for future use. See modules/gcp-host." type = map(object({ diff --git a/observability/.env.example b/observability/.env.example index 73b9f35..ae61a4d 100644 --- a/observability/.env.example +++ b/observability/.env.example @@ -1,7 +1,10 @@ -# Copy to .env on the monitoring host (beta) before `docker compose up`. +# Copy to .env on the monitoring host (vps1) before `docker compose up`. # .env is gitignored — never commit real secrets. +# +# vps1 is NOT the beta environment — that runs on vps2 now, as its own Swarm +# stack. Don't confuse the two when reading older docs that say "beta" here. -# The public hostname beta's Caddy serves Grafana at (see caddy-grafana.conf). +# The public hostname vps1's Caddy serves Grafana at (see caddy-grafana.conf). GRAFANA_DOMAIN=dashboard.thermograph.org # Break-glass local admin (Google SSO below is the primary login). Keep this @@ -22,8 +25,8 @@ GOOGLE_CLIENT_SECRET= # --- Alerting (Discord) ---------------------------------------------------------- # Where every alert in grafana/provisioning/alerting/ is delivered: a webhook on # the private #ops-alerts channel (Owners category) of the Thermograph.org -# Discord. NOT email — beta's Grafana relays SMTP through prod's Postfix, so -# email alerts die exactly when prod does. +# Discord. NOT email — vps1's Grafana relays SMTP through vps2's Postfix, so +# email alerts die exactly when prod (on vps2) does. # # To mint one: Discord -> #ops-alerts -> Edit Channel -> Integrations -> # Webhooks -> New Webhook -> Copy Webhook URL. Treat it like a password; anyone diff --git a/observability/CLAUDE.md b/observability/CLAUDE.md index 1485024..4d6db89 100644 --- a/observability/CLAUDE.md +++ b/observability/CLAUDE.md @@ -1,28 +1,37 @@ # observability/ — agent instructions -The **logging stack** for the fleet: Loki + Grafana on beta, with a Grafana Alloy -agent on every node (prod, beta, dev) shipping container and app logs over the -WireGuard mesh. Grafana is fronted by beta's Caddy at -**`dashboard.thermograph.org`** (Google SSO, pre-provisioned users only) — use -that hostname everywhere, never `grafana.thermograph.org`. +The **logging stack** for the fleet: Loki + Grafana on **vps1** — the monitoring +host, mesh `10.10.0.2`, and NOT the beta *environment* (that runs on vps2 now; +any doc that still says "beta" meaning this box is stale) — with a Grafana +Alloy agent on every node shipping container and app logs over the WireGuard +mesh. vps1 also hosts Forgejo, the portfolio site, and the **dev** environment +(mesh-only, no public DNS). vps2 runs **prod and beta** as two Swarm stacks. +Grafana is fronted by vps1's Caddy at **`dashboard.thermograph.org`** (Google +SSO, pre-provisioned users only) — use that hostname everywhere, never +`grafana.thermograph.org`. This is operational config, not application code: there is **no build** and no -deploy automation. It ships by hand — `docker compose up -d` on beta for +deploy automation. It ships by hand — `docker compose up -d` on vps1 for Loki+Grafana, and `alloy/docker-compose.agent.yml` on each node. Read the root `CLAUDE.md` first; the branch model there applies here too. ## Layout -- `docker-compose.yml` — the Loki + Grafana stack (beta). +- `docker-compose.yml` — the Loki + Grafana stack (vps1). - `loki/config.yml` — mesh-only, filesystem storage. - `grafana/provisioning/` — datasource + dashboard provider, auto-loaded at startup. `grafana/dashboards/*.json` — the dashboards. - `grafana/provisioning/alerting/` — alert rules, the Discord contact point, and the notification policy. -- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` — the per-node shipper. - Node name comes from `ALLOY_NODE`, set per host. -- `caddy-grafana.conf` — the beta Caddy vhost, kept here for reference. -- `.env.example` — copy to `.env` on beta. `.env` is gitignored; never commit +- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` (+ + `alloy/docker-compose.agent.beta.yml`, a vps2-only overlay adding beta's + `applogs` mount) — the per-node shipper. `ALLOY_NODE` is the MACHINE + (`vps1`|`vps2`|`desktop`) — that's a semantic change from when the fleet had + one environment per node; `host` (`prod`|`beta`|`dev`), the ENVIRONMENT + label every dashboard/alert slices on, is now derived per source instead, + since vps2 alone carries two of them. +- `caddy-grafana.conf` — the vps1 Caddy vhost, kept here for reference. +- `.env.example` — copy to `.env` on vps1. `.env` is gitignored; never commit OAuth secrets or the admin password. ## Rules @@ -40,8 +49,8 @@ Loki+Grafana, and `alloy/docker-compose.agent.yml` on each node. Read the root literal Discord webhook URL in the repo is a hard failure. A rule pointing at a missing refId is valid YAML, provisions cleanly, and then never fires — that silent no-op is what this check exists to prevent. -- **Alerts go to Discord `#ops-alerts`, never email.** Beta's Grafana relays SMTP - through prod's Postfix, so email dies exactly when prod does. +- **Alerts go to Discord `#ops-alerts`, never email.** vps1's Grafana relays + SMTP through vps2's Postfix, so email dies exactly when prod (on vps2) does. - **Every rule is LogQL** — there is no Prometheus anywhere in the fleet. Thresholds were derived from real Loki data and the working is in the comments beside each rule; re-derive before changing a number rather than guessing. diff --git a/observability/README.md b/observability/README.md index d0f9af5..e2ac68c 100644 --- a/observability/README.md +++ b/observability/README.md @@ -7,41 +7,61 @@ over the private WireGuard mesh. This replaces the old in-repo persistent, queryable, fleet-wide log store and UI. ``` - prod (10.10.0.1) beta (10.10.0.2) desktop / LAN dev (10.10.0.3) - ┌────────────┐ ┌──────────────────┐ ┌────────────┐ - │ Alloy agent│──wg0──┐ │ Loki ◀── Alloy │ ┌──wg0│ Alloy agent│ - └────────────┘ └───▶│ Grafana (Caddy) │◀──┘ └────────────┘ - docker+caddy+app └──────────────────┘ docker+caddy+app - logs central store + UI logs + vps1 (10.10.0.2) vps2 (10.10.0.1) desktop (10.10.0.3) + ┌───────────────────────┐ ┌───────────────────────┐ ┌──────────────┐ + │ Loki ◀───── Alloy │◀──wg0───┤ Alloy agent │◀─wg0──┤ Alloy agent │ + │ Grafana (Caddy) │ │ prod + beta (Swarm) │ │ (no app │ + │ dev (compose) │◀──wg0───┤ Postfix, backups │ │ stack) │ + │ Forgejo, portfolio │ └───────────────────────┘ └──────────────┘ + └───────────────────────┘ docker+caddy+app logs ships no app logs + central store + UI + dev ``` +- **vps1** is the monitoring host: Loki, Grafana, Forgejo, the portfolio site, + and the **dev** environment (its own compose project, mesh-only, no public + DNS). It does **not** run the beta *environment* — that now lives on vps2 — so + any older doc that says "beta" meaning *this box* is stale and needs reading + as vps1. +- **vps2** runs **prod and beta** as two separate Docker Swarm stacks + (`thermograph`, `thermograph-beta`), plus Postfix and backups. One shared + TimescaleDB serves both, on separate roles/databases. +- **desktop** hosts no Thermograph environment now (AI-model hosting + flex + Swarm-worker capacity instead), so it ships no app logs. - **Loki** stores logs (filesystem, 30-day retention). Listens on the mesh IP `10.10.0.2:3100` — never public. -- **Grafana** is the UI, fronted by beta's Caddy at `dashboard.thermograph.org`, +- **Grafana** is the UI, fronted by vps1's Caddy at `dashboard.thermograph.org`, login via **Google SSO** (with a break-glass local admin); Loki is not exposed. - **Alloy** runs on each node and ships three sources to Loki: every Docker container's stdout/stderr, Caddy's host access logs, and the app's structured JSON logs (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase` - become labels). Every line is tagged with its node (`host = prod|beta|dev`). + become labels). Every line now carries two distinct labels: `host` + (`prod`|`beta`|`dev`) is the ENVIRONMENT, derived *per source* (a container's + stack prefix, the Caddy access-log filename, or which `applogs` volume a JSON + line came out of) — because vps2 alone carries two environments, and one + node-wide value would have stamped every beta container and log line as + `prod`. `node` (`vps1`|`vps2`|`desktop`) is the MACHINE, stamped externally on + everything an agent ships. See `alloy/config.alloy` for exactly how `host` is + derived per source. ## Layout | Path | What | |------|------| -| `docker-compose.yml` | the central Loki + Grafana stack (runs on beta) | +| `docker-compose.yml` | the central Loki + Grafana stack (runs on vps1) | | `loki/config.yml` | Loki config (filesystem, retention) | | `grafana/provisioning/` | auto-wired Loki datasource + dashboard provider | | `grafana/provisioning/alerting/` | contact point, notification policy and alert rules — see [Alerting](#alerting) | | `grafana/dashboards/thermograph-logs.json` | the fleet-logs dashboard | | `alloy/config.alloy` | the per-node collector config | | `alloy/docker-compose.agent.yml` | runs the Alloy agent on a node | -| `caddy-grafana.conf` | beta Caddy site block for the Grafana UI | +| `alloy/docker-compose.agent.beta.yml` | overlay adding beta's `applogs` mount — vps2 only | +| `caddy-grafana.conf` | vps1 Caddy site block for the Grafana UI | ## Deploy > ### ⚠️ Merging is not deploying > -> `/opt/observability` on beta (and prod) is a checkout of the **archived** +> `/opt/observability` on vps1 (and vps2) is a checkout of the **archived** > `emi/thermograph-observability` repo. It can never `git pull` again — the > content now lives here, in the mono repo, under `observability/`. A previous > observability PR merged and had **zero live effect** for exactly this reason. @@ -49,22 +69,23 @@ persistent, queryable, fleet-wide log store and UI. > Until someone re-points those checkouts at the mono repo, every change here > ships by hand: back up the live file with a UTC-timestamped suffix, `scp` the > new one up, `sudo cp` it into place, restart the specific container. Use -> `sudo docker restart ` — **not `docker compose`**, which on prod -> demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to interpolate from. -> (Beta's stack does have an `.env`, so compose works there — and is required +> `sudo docker restart ` — **not `docker compose`**, which on vps2 +> demands a `GF_SECURITY_ADMIN_PASSWORD` it has no `.env` to interpolate from +> (vps2 only runs the Alloy agent — there's no Grafana there to configure). +> (vps1's stack does have an `.env`, so compose works there — and is required > when a change adds a new environment variable, since `docker restart` alone > will not pick one up.) -### 1. Central stack (on beta) +### 1. Central stack (on vps1) ```bash -# on beta (75.119.132.91): +# on vps1 (75.119.132.91): git clone thermograph-observability && cd thermograph-observability cp .env.example .env # set GF_SECURITY_ADMIN_PASSWORD docker compose up -d # Loki on 10.10.0.2:3100 + 127.0.0.1:3100; Grafana on 127.0.0.1:3000 ``` -Expose the Grafana UI: point `dashboard.thermograph.org` at beta, append +Expose the Grafana UI: point `dashboard.thermograph.org` at vps1, append `caddy-grafana.conf` to `/etc/caddy/Caddyfile`, `systemctl reload caddy`. ### Google SSO (Grafana + Forgejo) @@ -80,7 +101,7 @@ OAuth 2.0 Client (type: *Web application*) and register both redirect URIs: (`allow_sign_up=false`), so provision your email as an admin once: ```bash -# on beta, after the stack is up (uses the break-glass admin creds from .env): +# on vps1, after the stack is up (uses the break-glass admin creds from .env): curl -s -u admin:"$GF_SECURITY_ADMIN_PASSWORD" -H 'Content-Type: application/json' \ -X POST http://127.0.0.1:3000/api/admin/users \ -d '{"name":"you","login":"you@gmail.com","email":"you@gmail.com","password":"'"$(openssl rand -base64 24)"'"}' @@ -97,28 +118,50 @@ docker exec -u git forgejo admin auth add-oauth \ --auto-discover-url https://accounts.google.com/.well-known/openid-configuration ``` -### 2. Alloy agent (on every node — prod, beta, desktop) +### 2. Alloy agent (on vps1 and vps2) + +`ALLOY_NODE` names the MACHINE now, not the environment — vps2 alone runs two +environments (prod and beta), so a node name can no longer stand in for a +`host` value. `ALLOY_ENV` supplies the default environment label for anything +on that node not attributable to a specific stack (Forgejo, Grafana, the +portfolio site). `APPLOGS_VOLUME` is the node's primary `applogs` volume; vps2 +additionally needs `BETA_APPLOGS_VOLUME` and a second compose file, since it +carries both environments' structured logs side by side. + +Every node pushes to Loki over the mesh IP — a containerized agent can't reach +the host's `127.0.0.1`, but it can reach vps1's wg0 address at +`10.10.0.2:3100`. ```bash cd thermograph-observability/alloy -# Every node (beta included) pushes to Loki over the mesh IP — a containerized -# agent can't reach the host's 127.0.0.1, but it can reach beta's wg0 address: -# beta: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=beta -# prod: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=prod -# desktop: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=dev -ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + +# vps1 (dev + Forgejo + the monitoring stack itself): +ALLOY_NODE=vps1 ALLOY_ENV=dev APPLOGS_VOLUME=thermograph-dev_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ docker compose -f docker-compose.agent.yml up -d + +# vps2 (prod + beta) — note the SECOND file, adding beta's log volume: +ALLOY_NODE=vps2 ALLOY_ENV=prod APPLOGS_VOLUME=thermograph_applogs \ +BETA_APPLOGS_VOLUME=thermograph-beta_applogs \ +LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ + docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d ``` -On the LAN dev node the app's log volume is `thermograph-dev_applogs` (not -`thermograph_applogs`) — edit the two references in `docker-compose.agent.yml` -there. The mesh must already be up (see the main repo's `deploy/swarm/` / -`INFRA.md`); Swarm is **not** required — the agents talk plain HTTP over wg0. +The mesh must already be up (see the main repo's `deploy/swarm/` / `INFRA.md`); +Swarm is **not** required for the agent itself — it talks plain HTTP over wg0. + + ## Verify ```bash -# From any mesh node, confirm Loki is receiving from every host: +# From any mesh node, confirm Loki is receiving from every environment +# (host = prod|beta|dev; a separate node/vps1|vps2|desktop label exists too): curl -s 'http://10.10.0.2:3100/loki/api/v1/label/host/values' # -> {"data":["beta","dev","prod"]} # Then open Grafana → the "Thermograph — Fleet Logs" dashboard. ``` @@ -130,6 +173,13 @@ log volume by service, app error rate by tag, upstream 429 count, Caddy 5xx count, a per-node notifier-liveness indicator (from the heartbeat log), recent app errors, and a live all-container tail. Edit it in Grafana and re-export the JSON here to version a change. + + ## Alerting Provisioned from `grafana/provisioning/alerting/`, mounted read-only into Grafana @@ -146,8 +196,8 @@ Discord contact point. Deliberately **not email**: - Grafana's factory default pointed at the literal string `` — a paging path that had never delivered a message to anyone. -- beta's Grafana relays SMTP through **prod's** Postfix at `10.10.0.1:25` (see - `docker-compose.override.yml` on beta). Email alerts therefore travel through +- vps1's Grafana relays SMTP through **vps2's** Postfix at `10.10.0.1:25` (see + `docker-compose.override.yml` on vps1). Email alerts therefore travel through the box most likely to be on fire, and vanish exactly when they matter. - Discord is off-estate, works when prod is dead, and reaches a phone. @@ -157,7 +207,7 @@ Discord contact point. Deliberately **not email**: `#notes` / `#staff-chat` are for humans. The webhook URL is a secret and is **not in this repo**: it comes from -`DISCORD_ALERT_WEBHOOK_URL` in beta's gitignored `.env`, which compose passes to +`DISCORD_ALERT_WEBHOOK_URL` in vps1's gitignored `.env`, which compose passes to Grafana and which Grafana interpolates when reading the provisioning file. Grafana refuses to start if it is unset. @@ -187,7 +237,7 @@ value is its absence**: nothing can report its own death, so if `#ops-alerts` ha been silent for more than a day, alerting is broken rather than the estate healthy. Set `isPaused: true` on it to opt out of that guarantee. -### Deploying an alerting change to beta +### Deploying an alerting change to vps1 Grafana already mounts `./grafana/provisioning` read-only, so new files under `alerting/` need no compose change — but the **first** deploy does, because it @@ -196,25 +246,26 @@ new environment variable. ```bash # run from the mono-repo root -BETA="agent@75.119.132.91" +VPS1="agent@75.119.132.91" # the monitoring host — NOT the beta environment, + # which now runs on vps2; don't repoint this at it. SSH="ssh -i ~/.ssh/thermograph_agent_ed25519" TS=$(date -u +%Y%m%dT%H%M%SZ) -# 1. put the secret in beta's .env (once), then confirm it took: -$SSH $BETA "grep -q '^DISCORD_ALERT_WEBHOOK_URL=' /opt/observability/.env && echo present || echo MISSING" +# 1. put the secret in vps1's .env (once), then confirm it took: +$SSH $VPS1 "grep -q '^DISCORD_ALERT_WEBHOOK_URL=' /opt/observability/.env && echo present || echo MISSING" # 2. back up the live compose file, stage the new files, move them into place: -$SSH $BETA "sudo cp -a /opt/observability/docker-compose.yml /opt/observability/docker-compose.yml.$TS \ +$SSH $VPS1 "sudo cp -a /opt/observability/docker-compose.yml /opt/observability/docker-compose.yml.$TS \ && rm -rf /tmp/obs-stage && mkdir -p /tmp/obs-stage/alerting" -scp -i ~/.ssh/thermograph_agent_ed25519 observability/docker-compose.yml $BETA:/tmp/obs-stage/ -scp -i ~/.ssh/thermograph_agent_ed25519 observability/grafana/provisioning/alerting/*.yml $BETA:/tmp/obs-stage/alerting/ -$SSH $BETA "sudo mkdir -p /opt/observability/grafana/provisioning/alerting \ +scp -i ~/.ssh/thermograph_agent_ed25519 observability/docker-compose.yml $VPS1:/tmp/obs-stage/ +scp -i ~/.ssh/thermograph_agent_ed25519 observability/grafana/provisioning/alerting/*.yml $VPS1:/tmp/obs-stage/alerting/ +$SSH $VPS1 "sudo mkdir -p /opt/observability/grafana/provisioning/alerting \ && sudo cp /tmp/obs-stage/docker-compose.yml /opt/observability/ \ && sudo cp /tmp/obs-stage/alerting/*.yml /opt/observability/grafana/provisioning/alerting/" # 3. recreate Grafana so it reads the new env var AND the new provisioning. # `docker restart` is NOT enough here — it will not pick up a new env var. -$SSH $BETA "cd /opt/observability && docker compose up -d grafana" +$SSH $VPS1 "cd /opt/observability && docker compose up -d grafana" ``` Subsequent rule-only changes are just step 2 plus @@ -227,13 +278,14 @@ new environment variable is involved. falls back to its default (email → ``) root policy — i.e. no alerting, which is where this started. -Alerting lives on **beta only** — prod and dev run Alloy agents, not Grafana, so -nothing in `alerting/` ever ships to them. +Alerting lives on **vps1 only** — vps2 (prod + beta) and dev only run Alloy +agents, not Grafana, so nothing in `alerting/` ever ships to them, even though +dev happens to share a box with vps1's Grafana. ### Verify alerting ```bash -# on beta, after deploying and restarting Grafana: +# on vps1, after deploying and restarting Grafana: GFP=$(grep '^GF_SECURITY_ADMIN_PASSWORD=' /opt/observability/.env | cut -d= -f2-) WH=$(grep '^DISCORD_ALERT_WEBHOOK_URL=' /opt/observability/.env | cut -d= -f2-) # 1. all 12 rules loaded and healthy (health should be "ok", never "error"): @@ -270,7 +322,7 @@ delete a receiver a policy still references). Either use the UI — Alerting → Contact points → `grafana-default-email` → Delete — or: ```bash -# on beta. Reads the running Alertmanager config, drops the one receiver, +# on vps1. Reads the running Alertmanager config, drops the one receiver, # writes it back. Verified against Grafana 11.6.1 (returns 202). GFP=$(grep '^GF_SECURITY_ADMIN_PASSWORD=' /opt/observability/.env | cut -d= -f2-) AM=localhost:3000/api/alertmanager/grafana/config/api/v1/alerts @@ -289,7 +341,7 @@ This is cosmetic — leaving it costs nothing but a confusing row in the UI. - **Loki is mesh-only** by design; only Grafana (with auth) is public. - Alloy reads the Docker socket read-only to discover containers. -- Retention is 30 days on beta's disk — bump `loki/config.yml` if you want more. +- Retention is 30 days on vps1's disk — bump `loki/config.yml` if you want more. - The app emits a `tag=heartbeat` line for the subscription notifier every ~15m; the dashboard's "Notifier alive?" panel turns red if a node misses it. Extend the same `audit.log_heartbeat()` to any future background daemon. @@ -319,6 +371,6 @@ This is cosmetic — leaving it costs nothing but a confusing row in the UI. is for, but it is a *manual* dead-man's switch: it relies on somebody noticing that the daily message stopped. A real external watchdog (an uptime pinger hitting a Grafana endpoint from off-estate) would close this properly. -- **beta runs a `docker-compose.override.yml` that is not in this repo**, wiring - Grafana's SMTP to prod's Postfix. It should be mirrored into the tracked +- **vps1 runs a `docker-compose.override.yml` that is not in this repo**, wiring + Grafana's SMTP to vps2's Postfix. It should be mirrored into the tracked compose file or deleted; alerting no longer depends on it. diff --git a/observability/alloy/config.alloy b/observability/alloy/config.alloy index 3279da2..2efa071 100644 --- a/observability/alloy/config.alloy +++ b/observability/alloy/config.alloy @@ -1,15 +1,39 @@ -// Grafana Alloy — the log collector that runs on EVERY node (prod, beta, LAN -// dev). It gathers three sources and ships them to the central Loki on beta: +// Grafana Alloy — the log collector that runs on EVERY node (vps1, vps2). It +// gathers three sources and ships them to the central Loki on vps1: // -// 1. Every Docker container's stdout/stderr (app, db, and on beta forgejo) -// via the Docker socket. +// 1. Every Docker container's stdout/stderr (app, db, and on vps1 forgejo + +// the monitoring stack itself) via the Docker socket. // 2. Caddy's host access logs (/var/log/caddy/*.log) — the reverse proxy runs // on the host, not in a container, so its logs aren't in Docker. // 3. The app's structured JSON logs (errors/access/audit *.jsonl) from the -// `applogs` Docker volume, parsed so `level`/`tag`/`phase` become labels. +// `applogs` Docker volumes, parsed so `level`/`tag`/`phase` become labels. +// +// --------------------------------------------------------------------------- +// `host` IS THE ENVIRONMENT, AND IT IS NO LONGER THE NODE +// --------------------------------------------------------------------------- +// Every query, dashboard and alert in this estate slices on host="prod" | +// "beta" | "dev". That label used to be an EXTERNAL label — one value stamped +// on everything the agent shipped — which was exact while one node ran exactly +// one environment. +// +// vps2 now runs prod AND beta. A single external label there would have marked +// every beta container, every beta access log and every beta JSON line as +// `prod`: not a cosmetic problem, but wrong data feeding the prod alert rules, +// and beta's own alerts (NotifierHeartbeatMissingBeta) firing on prod's +// traffic or never firing at all. +// +// So `host` is now derived PER SOURCE — from the container's stack prefix, from +// the access-log filename, from which log volume a JSON line came out of — and +// ALLOY_ENV supplies the fallback for anything on the node that isn't +// environment-specific (Forgejo, Grafana, Loki, the portfolio site). The node +// itself is still labelled, as `node`, because "which machine" is a real +// question too — it is just a different question from "which environment". // // Per-node settings come from the environment (see docker-compose.agent.yml): -// ALLOY_NODE — this node's name label (prod | beta | dev) +// ALLOY_NODE — the MACHINE this agent runs on (vps1 | vps2 | desktop) +// ALLOY_ENV — the default environment label for logs on this node that +// aren't attributable to a specific stack (vps1 -> dev, +// vps2 -> prod) // LOKI_URL — where to push (http://10.10.0.2:3100/loki/api/v1/push over wg0) livedebugging { enabled = false } @@ -43,9 +67,48 @@ discovery.relabel "containers" { source_labels = ["__meta_docker_container_label_com_docker_compose_service"] target_label = "service" } + + // --- environment attribution ------------------------------------------- + // Relabel rules are applied in order and a later write to the same target + // wins, so this is "default, then override with anything more specific". + // + // The regexes are fully anchored (Prometheus relabel semantics), which is + // what keeps `/thermograph_web.1.x` and `/thermograph-beta_beta-web.1.x` + // from matching each other: prod's stack prefix ends in an underscore, + // beta's in `-beta_`. + rule { + source_labels = ["__meta_docker_container_name"] + regex = ".*" + target_label = "host" + replacement = sys.env("ALLOY_ENV") + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/thermograph_.*" + target_label = "host" + replacement = "prod" + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/thermograph-beta_.*" + target_label = "host" + replacement = "beta" + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/thermograph-dev[-_].*" + target_label = "host" + replacement = "dev" + } + + // Drop noise/duplicate containers, per environment. Beta's equivalents are + // listed explicitly: its LB is `thermograph-beta-lb` (which does NOT match + // `thermograph-lb.*`) and its worker is `thermograph-beta_beta-worker`, so + // omitting them would have quietly re-admitted exactly the healthz-poll + // noise the prod entries exist to exclude. rule { source_labels = ["container"] - regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph_worker).*" + regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph-beta-lb|thermograph_worker|thermograph-beta_beta-worker).*" action = "drop" } } @@ -94,11 +157,48 @@ loki.process "caddy" { expression = "(?i)(semrushbot|claudebot|ahrefsbot|yandexbot|bytespider|mj12bot|petalbot)" drop_counter_reason = "crawler" } + + // Attribute each access log to an environment BY FILENAME. On vps2 one + // Caddy fronts both thermograph.org and beta.thermograph.org, so a single + // node-wide label would file every beta request under prod. + // + // By filename rather than by parsing `request.host` out of the JSON: the + // site block that writes the file is the same place the hostname is + // declared (see infra/deploy/Caddyfile.vps2), one label costs nothing to + // evaluate, and it keeps working for a non-JSON log format. + stage.static_labels { + values = { host = sys.env("ALLOY_ENV") } + } + stage.match { + selector = "{filename=~\".*/beta[.]log\"}" + + stage.static_labels { + values = { host = "beta" } + } + } + stage.match { + selector = "{filename=~\".*/thermograph[.]log\"}" + + stage.static_labels { + values = { host = "prod" } + } + } } // --- 3. App structured JSON logs (errors / access / audit) ----------------------- -// Mounted read-only from the app's `applogs` volume at /applogs (see the agent -// compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable. +// Mounted read-only from the app's `applogs` volumes (see the agent compose). +// Lift `level`/`tag`/`phase` out of the JSON so they're queryable. +// +// TWO MOUNTS, NOT ONE. Each environment has its own applogs volume, and on vps2 +// both exist side by side: +// +// /applogs this node's primary environment (vps2 -> prod, vps1 -> dev) +// /applogs-beta beta's, mounted on vps2 only +// +// A single mount was structurally unable to carry both: whichever volume the +// agent bound, the other environment's structured logs would never reach Loki +// at all — silently, since a missing file source is not an error. Every alert +// built on the JSON stream (the notifier heartbeats in particular) reads this. local.file_match "app_jsonl" { path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }] sync_period = "1m" @@ -122,7 +222,38 @@ loki.process "app_jsonl" { } // A record with no explicit level: an error-folder line is an error, else info. stage.static_labels { - values = { source = "app" } + values = { source = "app", host = sys.env("ALLOY_ENV") } + } + stage.labels { + values = { level = "", tag = "", phase = "" } + } +} + +// Beta's structured logs. Absent on vps1 (the mount simply isn't there), where +// this matches nothing and costs nothing. +local.file_match "app_jsonl_beta" { + path_targets = [{ __path__ = "/applogs-beta/**/*.jsonl", job = "app-json" }] + sync_period = "1m" +} + +loki.source.file "app_jsonl_beta" { + targets = local.file_match.app_jsonl_beta.targets + forward_to = [loki.process.app_jsonl_beta.receiver] + + file_watch { + min_poll_frequency = "2s" + max_poll_frequency = "10s" + } +} + +loki.process "app_jsonl_beta" { + forward_to = [loki.write.central.receiver] + + stage.json { + expressions = { level = "level", tag = "tag", phase = "phase", status = "status" } + } + stage.static_labels { + values = { source = "app", host = "beta" } } stage.labels { values = { level = "", tag = "", phase = "" } @@ -134,7 +265,9 @@ loki.write "central" { endpoint { url = sys.env("LOKI_URL") } - // Every line from this node is stamped with its node name, so one Grafana - // view can slice prod vs beta vs dev. - external_labels = { host = sys.env("ALLOY_NODE") } + // `node` — the MACHINE, stamped on everything this agent ships. `host` is + // deliberately NOT here: it is the ENVIRONMENT, set per source above, + // because vps2 carries two of them and an external label cannot vary per + // stream. Setting host here would override that work on every line. + external_labels = { node = sys.env("ALLOY_NODE") } } diff --git a/observability/alloy/docker-compose.agent.beta.yml b/observability/alloy/docker-compose.agent.beta.yml new file mode 100644 index 0000000..3308e95 --- /dev/null +++ b/observability/alloy/docker-compose.agent.beta.yml @@ -0,0 +1,24 @@ +# Overlay for vps2 ONLY — the node that runs two environments. +# +# docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d +# +# Adds beta's applogs volume at /applogs-beta, which config.alloy reads as a +# separate source and labels host="beta". Everything else — the Docker socket, +# the Caddy logs, the primary /applogs mount — comes from the base file. +# +# This is an overlay rather than another variable in the base file because the +# /applogs-beta source is labelled beta UNCONDITIONALLY. On a node where beta +# does not run, there is no correct volume to point it at: aiming it at the +# node's own logs would republish them as beta, manufacturing data for an +# environment that isn't there. "Absent" is the only honest configuration, and +# a missing overlay expresses that exactly. + +services: + alloy: + volumes: + - beta_applogs:/applogs-beta:ro + +volumes: + beta_applogs: + external: true + name: ${BETA_APPLOGS_VOLUME:?set BETA_APPLOGS_VOLUME (thermograph-beta_applogs)} diff --git a/observability/alloy/docker-compose.agent.yml b/observability/alloy/docker-compose.agent.yml index 341f9ca..9bab056 100644 --- a/observability/alloy/docker-compose.agent.yml +++ b/observability/alloy/docker-compose.agent.yml @@ -1,22 +1,41 @@ -# The Alloy log-collector agent. Runs on EACH node (prod, beta, LAN dev) — one -# per node — reading that node's Docker containers, Caddy logs, and the app's -# JSON logs, and shipping them to the central Loki on beta over the mesh. +# The Alloy log-collector agent. Runs on EACH node (vps1, vps2) — one per node +# — reading that node's Docker containers, Caddy logs, and the app's JSON logs, +# and shipping them to the central Loki on vps1 over the mesh. # # Per-node config comes from the environment (an .env file next to this, or # exported before `up`): -# ALLOY_NODE this node's label: prod | beta | dev +# ALLOY_NODE the MACHINE: vps1 | vps2 | desktop +# ALLOY_ENV the default ENVIRONMENT label for logs on this node that +# aren't attributable to a specific stack (Forgejo, Grafana, +# the portfolio site): vps1 -> dev, vps2 -> prod +# APPLOGS_VOLUME this node's primary applogs volume # LOKI_URL push endpoint. All nodes push over the mesh: # http://10.10.0.2:3100/loki/api/v1/push # -# Deploy on a node: -# ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ +# Deploy on vps1 (dev + Forgejo + the monitoring stack itself): +# ALLOY_NODE=vps1 ALLOY_ENV=dev APPLOGS_VOLUME=thermograph-dev_applogs \ +# LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ # docker compose -f docker-compose.agent.yml up -d # -# The app's JSON logs are read from its `applogs` Docker volume, named -# `_applogs`. prod and beta run the app under the compose project -# `thermograph`, so the external volume below is `thermograph_applogs`. The LAN -# dev stack uses the project `thermograph-dev` — on that node, change the two -# `thermograph_applogs` references below to `thermograph-dev_applogs`. +# Deploy on vps2 (prod + beta) — note the SECOND file, which adds beta's log +# volume: +# ALLOY_NODE=vps2 ALLOY_ENV=prod APPLOGS_VOLUME=thermograph_applogs \ +# BETA_APPLOGS_VOLUME=thermograph-beta_applogs \ +# LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \ +# docker compose -f docker-compose.agent.yml -f docker-compose.agent.beta.yml up -d +# +# WHY THE VOLUME NAME IS A VARIABLE NOW, AND WHY BETA IS AN OVERLAY. The name +# used to be the literal `thermograph_applogs`, with a comment telling you to +# hand-edit two lines on the dev node. That was survivable while one node ran +# one environment. vps2 runs two, each with its own applogs volume, and a single +# hardcoded mount there would ship one environment's structured logs and +# silently drop the other's. +# +# Beta's mount is a separate overlay file rather than a second variable in this +# one because the second mount is labelled host="beta" unconditionally. Pointing +# it at dev's volume on vps1 just to satisfy compose would ship every dev log +# line a second time labelled as beta — inventing an environment's worth of +# fake data on a box beta does not run on. services: alloy: @@ -27,7 +46,8 @@ services: - --storage.path=/var/lib/alloy/data - --server.http.listen-addr=0.0.0.0:12345 environment: - ALLOY_NODE: ${ALLOY_NODE:?set ALLOY_NODE (prod|beta|dev)} + ALLOY_NODE: ${ALLOY_NODE:?set ALLOY_NODE (vps1|vps2|desktop) — the MACHINE} + ALLOY_ENV: ${ALLOY_ENV:?set ALLOY_ENV (dev|beta|prod) — this node's default environment label} LOKI_URL: ${LOKI_URL:?set LOKI_URL} volumes: - ./config.alloy:/etc/alloy/config.alloy:ro @@ -36,13 +56,16 @@ services: # Caddy's host access logs (runs on the host, not a container). - /var/log/caddy:/var/log/caddy:ro # The app's structured JSON logs, straight off its named volume. - - thermograph_applogs:/applogs:ro + - applogs:/applogs:ro - alloy_data:/var/lib/alloy/data restart: unless-stopped volumes: # The app's existing log volume (created by the app's own stack). external:true - # means compose references it, never creates or deletes it. - thermograph_applogs: + # means compose references it, never creates or deletes it. The NAME varies by + # node — see the header — while the mount path stays /applogs, which is what + # lets one config.alloy serve every node. + applogs: external: true + name: ${APPLOGS_VOLUME:?set APPLOGS_VOLUME (e.g. thermograph_applogs on vps2, thermograph-dev_applogs on vps1)} alloy_data: {} diff --git a/observability/caddy-grafana.conf b/observability/caddy-grafana.conf index 787b816..420805c 100644 --- a/observability/caddy-grafana.conf +++ b/observability/caddy-grafana.conf @@ -1,10 +1,11 @@ -# Append to beta's /etc/caddy/Caddyfile (beta is the monitoring host), then -# `systemctl reload caddy`. Grafana publishes to 127.0.0.1:3000; this fronts it -# with TLS, same pattern as beta's other site blocks. Login is Google SSO (with a -# break-glass local admin); Loki (the log store, on 10.10.0.2:3100) is NOT proxied -# here and stays mesh-only. +# Append to vps1's /etc/caddy/Caddyfile (vps1 is the monitoring host — NOT the +# beta environment, which runs on vps2 now), then `systemctl reload caddy`. +# Grafana publishes to 127.0.0.1:3000; this fronts it with TLS, same pattern as +# vps1's other site blocks. Login is Google SSO (with a break-glass local +# admin); Loki (the log store, on 10.10.0.2:3100) is NOT proxied here and stays +# mesh-only. # -# dashboard.thermograph.org's A record must already point at beta (75.119.132.91). +# dashboard.thermograph.org's A record must already point at vps1 (75.119.132.91). dashboard.thermograph.org { encode zstd gzip diff --git a/observability/docker-compose.yml b/observability/docker-compose.yml index 06b3bc5..36516b7 100644 --- a/observability/docker-compose.yml +++ b/observability/docker-compose.yml @@ -1,11 +1,14 @@ # The central observability stack: Loki (log store) + Grafana (UI). Runs on ONE -# node — the monitoring host, `beta` (75.119.132.91 / mesh 10.10.0.2) — because -# beta is an always-on VPS with a public Caddy already fronting it. Every node's -# Alloy agent (see alloy/) ships logs here; agents on prod and the desktop reach -# Loki over the WireGuard mesh, so Loki must listen on the mesh interface. +# node — the monitoring host, `vps1` (75.119.132.91 / mesh 10.10.0.2) — because +# vps1 is an always-on VPS with a public Caddy already fronting it. This is NOT +# where the beta *environment* lives (that's vps2 now, alongside prod, as two +# Swarm stacks) — don't let "beta" in older docs send you looking for Grafana +# there. Every node's Alloy agent (see alloy/) ships logs here; agents on vps2 +# and desktop reach Loki over the WireGuard mesh, so Loki must listen on the +# mesh interface. # -# Deploy (on beta): docker compose up -d -# Grafana is proxied by beta's Caddy at dashboard.thermograph.org (see README); +# Deploy (on vps1): docker compose up -d +# Grafana is proxied by vps1's Caddy at dashboard.thermograph.org (see README); # Loki is NOT public — it binds the mesh IP only, reachable to agents over wg0. services: @@ -19,7 +22,7 @@ services: # Mesh-only: agents on prod (10.10.0.1) and desktop (10.10.0.3) push here # over wg0. Never published on the public interface. - "10.10.0.2:3100:3100" - # Also localhost, so the beta-local Alloy agent and curl checks can reach it. + # Also localhost, so the vps1-local Alloy agent and curl checks can reach it. - "127.0.0.1:3100:3100" restart: unless-stopped @@ -61,7 +64,7 @@ services: # --- Alerting -------------------------------------------------------------- # The Discord webhook that grafana/provisioning/alerting/contact-points.yml # interpolates as $DISCORD_ALERT_WEBHOOK_URL. It is a secret (holding it is - # enough to post in the channel), so it lives only in beta's .env. + # enough to post in the channel), so it lives only in vps1's .env. # Required, not defaulted: a Grafana that comes up with an empty webhook # looks perfectly healthy and pages nobody, which is the failure mode this # whole config exists to end. Better to refuse to start. @@ -71,7 +74,7 @@ services: - ./grafana/dashboards:/var/lib/grafana/dashboards:ro - grafana_data:/var/lib/grafana ports: - # Host-local; beta's Caddy reverse-proxies to it. Not public directly. + # Host-local; vps1's Caddy reverse-proxies to it. Not public directly. - "127.0.0.1:3000:3000" restart: unless-stopped diff --git a/observability/grafana/provisioning/alerting/contact-points.yml b/observability/grafana/provisioning/alerting/contact-points.yml index bdddbe2..a24535d 100644 --- a/observability/grafana/provisioning/alerting/contact-points.yml +++ b/observability/grafana/provisioning/alerting/contact-points.yml @@ -3,21 +3,22 @@ # # WHY NOT EMAIL. The Grafana factory default routes to the literal string # "", i.e. nowhere. Fixing it by pointing at a real mailbox -# would still be wrong: beta's Grafana relays SMTP through prod's Postfix at -# 10.10.0.1:25 (see the live docker-compose.override.yml on beta), so email +# would still be wrong: vps1's Grafana (the monitoring host — not the beta +# environment, which runs on vps2) relays SMTP through vps2's Postfix at +# 10.10.0.1:25 (see the live docker-compose.override.yml on vps1), so email # alerts travel *through the box most likely to be on fire* and are silently # lost whenever Postfix is down — which is exactly when you need them. Discord # is off-estate: it works when prod is dead, and it works from a phone. # # THE WEBHOOK URL IS A SECRET. Anyone holding it can post into the channel, so # it is NOT in this repo. It is read from the Grafana process environment, which -# docker-compose.yml feeds from beta's gitignored .env (see .env.example). +# docker-compose.yml feeds from vps1's gitignored .env (see .env.example). # Grafana expands $VAR / $__env{VAR} when it reads provisioning files; if the # variable is unset the contact point ends up with a literal "$DISCORD_..." # string and every notification fails — see README "Verify alerting". # # To rotate: make a new webhook on the #ops-alerts channel, replace the value in -# beta's .env, `docker restart observability-grafana-1`, delete the old webhook +# vps1's .env, `docker restart observability-grafana-1`, delete the old webhook # in Discord. apiVersion: 1 diff --git a/observability/grafana/provisioning/alerting/rules-prod.yml b/observability/grafana/provisioning/alerting/rules-prod.yml index f3f2a3b..470348a 100644 --- a/observability/grafana/provisioning/alerting/rules-prod.yml +++ b/observability/grafana/provisioning/alerting/rules-prod.yml @@ -33,7 +33,11 @@ # label — only `container`, which includes a per-task suffix that changes on # every redeploy (thermograph_web.1.). So prod services are matched as # container=~"thermograph_.+" with job="docker". Do not use `service=` -# here; it only works on the compose hosts (beta, dev). +# here; it only works on dev, now the ONLY compose host left — beta is Swarm +# too (stack `thermograph-beta`, since both prod and beta run on vps2), so a +# beta-targeted rule needs its own regex against beta's prefixed container +# names (thermograph-beta_beta-web.1. etc.), not a bare `service=` +# match either. # # --------------------------------------------------------------------------- # NOT ALERTABLE, ON PURPOSE @@ -69,6 +73,10 @@ groups: # DATA: 15-minute windows over 24h ranged 35 -> 301 lines, minimum 35, # and never once reached zero — including *during* the outage (72), when # Caddy was happily logging 503s. Zero is unambiguous. + # + # The description names vps1, not "beta": the mesh path that matters here + # is the one to the MONITORING host, which is vps1. Beta is on vps2 and + # has nothing to do with whether these logs arrive. - uid: tg_prod_edge_dark title: ProdEdgeDark condition: C @@ -81,7 +89,8 @@ groups: Nothing has reached thermograph.org's reverse proxy in 15 minutes (healthy floor is 35 lines/15m, and it stayed at 72 even during the 04:20Z outage). Either prod is down, Caddy is down, prod's Alloy - agent stopped shipping, or the WireGuard mesh to beta is broken. + agent stopped shipping, or the WireGuard mesh to vps1 (where Loki + runs) is broken. This rule also fires on Loki NoData/errors, so it doubles as the "we have lost observability" alarm — if it is the only thing firing, suspect the pipeline before suspecting the app. @@ -397,9 +406,10 @@ groups: # means two consecutive empty evaluations, so roughly 25 minutes of real # silence before it pages — about one and a half missed beats. # - # dev is deliberately excluded: the LAN box had zero heartbeats for the - # first 10.5h of the backtest window because it was simply off. Alerting - # on dev would mean an alert that is firing more often than not. + # dev is deliberately excluded: the dev box (mesh-only, on vps1) had zero + # heartbeats for the first 10.5h of the backtest window because it was + # simply off. Alerting on dev would mean an alert that is firing more + # often than not. # # MUST stay filtered to daemon="subscription-notifier". The selector was # originally bare `tag="heartbeat"`, which was unambiguous while the @@ -833,10 +843,10 @@ groups: # -- 12 ---------------------------------------------------------------- # Nothing in this file can tell you that Grafana itself has stopped, or - # that beta is down, or that the Discord webhook was revoked — a monitoring - # system cannot report its own death. So this rule fires permanently and - # by design, throttled by the notification policy to exactly one Discord - # message per 24 hours. + # that vps1 (the monitoring host) is down, or that the Discord webhook was + # revoked — a monitoring system cannot report its own death. So this rule + # fires permanently and by design, throttled by the notification policy + # to exactly one Discord message per 24 hours. # # HOW TO USE IT: if #ops-alerts has been silent for more than a day, the # alerting pipeline is broken, not the estate healthy. That one daily @@ -844,6 +854,7 @@ groups: # rules are still capable of reaching you. # # To silence it (and lose that guarantee), set isPaused: true. + # - uid: tg_alerting_watchdog title: AlertingWatchdog condition: C @@ -854,13 +865,20 @@ groups: summary: 'Alerting pipeline is alive (daily heartbeat — not a fault)' description: >- This is the dead-man's switch and it is supposed to be firing. It - proves Grafana on beta is evaluating rules, Loki is answering, and + proves Grafana on vps1 is evaluating rules, Loki is answering, and the Discord webhook still works. Seeing it once a day is correct. NOT seeing it for over a day means alerting itself is down and no - other rule can reach you — check observability-grafana-1 on beta. + other rule can reach you — check observability-grafana-1 on vps1. labels: severity: watchdog - host: beta + # `node`, not `host`: every other rule's `host` names an ENVIRONMENT + # (prod/beta/dev), and this rule is about the alerting pipeline + # itself, which is a machine. It used to be filed `host: beta` back + # when the monitoring box and the beta environment were the same + # machine — they are not any more, and that label would now read as a + # claim about beta. Routing is unaffected: notification-policies.yml + # matches on `severity` only and uses `host` just for grouping. + node: vps1 class: meta isPaused: false data: diff --git a/observability/loki/config.yml b/observability/loki/config.yml index b2dfb46..95bad61 100644 --- a/observability/loki/config.yml +++ b/observability/loki/config.yml @@ -1,8 +1,9 @@ -# Loki — the single log store for the whole Thermograph fleet (prod, beta, LAN -# dev). Every node's Alloy agent pushes here over the WireGuard mesh; Grafana -# reads from here. Single-binary, filesystem-backed: no object storage, no -# clustering — right for a three-node hobby fleet, and everything lives on one -# volume you can back up or blow away. +# Loki — the single log store for the whole Thermograph fleet: prod and beta +# (both on vps2, as separate Swarm stacks) and dev (vps1, mesh-only, its own +# compose project). Every node's Alloy agent pushes here over the WireGuard +# mesh; Grafana reads from here. Single-binary, filesystem-backed: no object +# storage, no clustering — right for a three-node hobby fleet, and everything +# lives on one volume you can back up or blow away. auth_enabled: false server: -- 2.45.2 From 7583258509de22b0faace75301fce509522be8e1 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 07:05:09 +0000 Subject: [PATCH 07/37] infra-sync: refuse to render dev's vault onto a host that is not dev (#107) --- .forgejo/workflows/infra-sync.yml | 25 ++++++++++++++++++++++- infra/deploy/RUNBOOK-vps1-vps2-cutover.md | 14 ++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/infra-sync.yml b/.forgejo/workflows/infra-sync.yml index 8f86697..67cd055 100644 --- a/.forgejo/workflows/infra-sync.yml +++ b/.forgejo/workflows/infra-sync.yml @@ -61,6 +61,30 @@ jobs: cd /opt/thermograph-dev git fetch --prune origin dev git reset --hard origin/dev + echo "synced to $(git log --oneline -1)" + + # RENDER ONLY IF THIS HOST IS ACTUALLY DEV. + # + # dev's env file is /etc/thermograph.env — the same path beta uses, + # and during the vps1/vps2 cutover beta is STILL ON VPS1. Rendering + # unconditionally would overwrite the live env file of the + # environment currently serving beta.thermograph.org with dev's + # credentials: beta's running containers would survive (env_file is + # read at container start) and then come up wrong on its next + # deploy, pointed at dev's database password. + # + # The host marker is the box's own statement of which environment it + # is, and it is the right authority for "is it safe to write this + # file here". It flips to `dev` in provision-dev.sh, which is run + # only after beta has vacated. Until then this skips loudly rather + # than failing: the checkout sync above is still useful and correct. + marker=$(cat /etc/thermograph/secrets-env 2>/dev/null || true) + if [ "$marker" != dev ]; then + echo "!! /etc/thermograph/secrets-env says '${marker:-}', not 'dev'." + echo "!! Skipping the secrets render — this host is not dev yet, and" + echo "!! /etc/thermograph.env here belongs to '${marker:-another environment}'." + exit 0 + fi if [ -f infra/deploy/render-secrets.sh ]; then . infra/deploy/render-secrets.sh # SKIP_COMMON is dev's standing rule, not a preference: common.yaml @@ -69,7 +93,6 @@ jobs: THERMOGRAPH_SECRETS_SKIP_COMMON=1 \ render_thermograph_secrets /opt/thermograph-dev/infra dev /etc/thermograph.env fi - echo "synced to $(git log --oneline -1)" sync-beta: if: github.ref_name == 'main' diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md index 72934a5..b1874da 100644 --- a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -278,7 +278,19 @@ sudo bash /opt/thermograph-dev/infra/deploy/provision-dev.sh # clones if absen ``` The script writes `/etc/thermograph/secrets-env` = `dev` and removes any -`deploy-mode` marker. Then deploy: +`deploy-mode` marker. + +**That marker flip is load-bearing, and it is why this step comes after step 9.** +dev's env file is `/etc/thermograph.env` — the very path beta uses while beta is +still on vps1. `infra-sync`'s `sync-dev` job refuses to render unless the marker +already says `dev`, so until you run this script it syncs the checkout and skips +the render with a loud message. Flip the marker before beta has vacated and the +next dev push overwrites beta's live env file with dev's credentials; beta's +running containers survive it and then come up wrong on their next deploy. + +(The checkout itself is safe to create at any time — only the render is gated.) + +Then deploy: ``` cd /opt/thermograph-dev -- 2.45.2 From 5c41ee4dadbec5bc4cac2aff457b272f6e113f5b Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 07:17:20 +0000 Subject: [PATCH 08/37] provision-env-db: revoke PUBLIC connect on the owner database too (#108) --- infra/deploy/RUNBOOK-vps1-vps2-cutover.md | 23 +++++++++++-- infra/deploy/db/provision-env-db.sh | 40 +++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md index b1874da..fdb379a 100644 --- a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -149,8 +149,16 @@ it. That guard is why the superuser check above should never fail; run it anyway. If the first command SUCCEEDS, stop. `CONNECT` was not revoked and beta would be -able to read production data; re-run the provisioning script and re-check before -going further. +able to open a session against the production database; re-run the provisioning +script and re-check before going further. + +This is not hypothetical — it happened on the live cutover. The first version of +the script revoked `PUBLIC` connect on the *guest's* database only, and a fresh +Postgres database carries `=Tc` (TEMP **and** CONNECT) for `PUBLIC`, so +`thermograph_beta` could still reach `thermograph`. The script now hardens the +owner's database too, granting the owner's read-only role an explicit `CONNECT` +first so the revoke cannot strip `ops/dbq.sh` of its access. The check above is +what caught it. **Rollback:** `DROP DATABASE thermograph_beta; DROP ROLE thermograph_beta;` — prod is unaffected either way. @@ -239,12 +247,23 @@ serving; nothing has moved yet. ``` ssh agent@169.58.46.181 sudo caddy validate --config /etc/caddy/Caddyfile # BEFORE reloading +# `validate` runs as root and CREATES any log file the config names, owned by +# root — which the caddy user then cannot open, and the reload fails. Fix the +# ownership before reloading, or the first reload rejects the whole config: +sudo chown caddy:caddy /var/log/caddy/beta.log sudo systemctl reload caddy ``` Validate before reload, always: a malformed Caddyfile takes `thermograph.org` down with it, and prod is on this box now. +The `chown` is not optional and it bit this cutover. `caddy validate` as root +left `/var/log/caddy/beta.log` as `root:root`, so the reload failed with +`permission denied` on the log writer. (The root-owned `centralis.log` sitting +in that directory is the same mistake, made earlier.) Note what did NOT happen: +Caddy rejects a bad config atomically and keeps serving the old one, so +`thermograph.org` stayed up throughout — verify it anyway. + Caddy cannot issue the certificate until DNS moves (step 7), so expect the beta hostname to fail TLS until then. That is fine and expected. diff --git a/infra/deploy/db/provision-env-db.sh b/infra/deploy/db/provision-env-db.sh index 334612c..3695027 100755 --- a/infra/deploy/db/provision-env-db.sh +++ b/infra/deploy/db/provision-env-db.sh @@ -156,6 +156,46 @@ ALTER DEFAULT PRIVILEGES FOR ROLE :"role" IN SCHEMA public GRANT SELECT ON SEQUENCES TO :"rorole"; SQL +# --- harden the OWNER environment's database too ------------------------------- +# +# Revoking CONNECT from PUBLIC on the guest's own database is only half the job, +# and the missing half is the half that matters. A fresh Postgres database +# carries `=Tc` for PUBLIC — TEMP *and* CONNECT — so the moment a second role +# exists on the instance, that role can connect to every database that still has +# the default. Provisioning beta and stopping here left `thermograph_beta` able +# to open a session against prod's database; caught live by the runbook's own +# isolation check, which is why that check exists. +# +# Order matters: grant the owner's read-only role an EXPLICIT connect first, so +# the revoke below cannot strip the ad-hoc query path (ops/dbq.sh) of its access. +# The owner's app role is the database owner and a superuser, so it is unaffected +# either way. +OWNER_DB=$( + # shellcheck disable=SC1090 # sourced above; re-resolving prod in a subshell + thermograph_topology prod >/dev/null 2>&1 + printf '%s' "$TG_DB_NAME" +) +OWNER_RO=$( + thermograph_topology prod >/dev/null 2>&1 + printf '%s_ro' "$TG_DB_USER" +) +# Restore this run's environment — thermograph_topology overwrote the TG_* vars. +thermograph_topology "$ENV_NAME" + +if [ "$OWNER_DB" != "$TG_DB_NAME" ]; then + echo "==> Hardening the owner database '${OWNER_DB}' against PUBLIC connects" + docker exec -i "$DB_CID" \ + psql -v ON_ERROR_STOP=1 -U thermograph -d postgres \ + -v ownerdb="$OWNER_DB" -v ownerro="$OWNER_RO" <<'SQL' +-- Only if that read-only role actually exists (it predates this script). +SELECT format('GRANT CONNECT ON DATABASE %I TO %I', :'ownerdb', :'ownerro') +WHERE EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'ownerro') +\gexec + +REVOKE CONNECT ON DATABASE :"ownerdb" FROM PUBLIC; +SQL +fi + echo "==> OK: ${TG_DB_USER} owns ${TG_DB_NAME} (timescaledb enabled, PUBLIC revoked)" echo " Verify isolation:" echo " docker exec $DB_CID psql -U ${TG_DB_USER} -d thermograph -c 'select 1' # must FAIL" -- 2.45.2 From 64986189e1f9ea051958227bc3657a93aebab840 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 07:28:43 +0000 Subject: [PATCH 09/37] deploy.sh: honour TG_POST_DEPLOY on the compose path too (#109) --- infra/deploy/deploy.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index 98fd9de..9efeaf7 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -417,7 +417,21 @@ docker images --format '{{.Repository}}:{{.Tag}}' \ # Post-deploy warm/IndexNow only make sense once the backend is (re)deployed -- # they exec inside the backend container. Skip them on a frontend-only roll. -if [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; then +# +# TG_POST_DEPLOY gates them per ENVIRONMENT (env-topology.sh): prod does both, +# beta and dev do neither. Both are wrong outside prod. The warm spends the +# shared upstream archive quota to fill a cache nobody serves from, and the +# IndexNow ping announces URLs to Bing/DuckDuckGo/Yandex — from dev that means +# submitting a mesh address no crawler can reach. +# +# This gate existed in deploy/stack/deploy-stack.sh (beta, prod) but was missed +# here, on the compose path, which is the ONE environment where the default is +# wrong: dev. Standing dev up on vps1 duly warmed 1000 cities and tried to +# submit 14,007 URLs to IndexNow before this was fixed. +# +# Defaults to 1 when unset so a checkout predating env-topology.sh keeps prod's +# historical behaviour. +if { [ "$SERVICE" = backend ] || [ "$SERVICE" = all ]; } && [ "${TG_POST_DEPLOY:-1}" = 1 ]; then warm_city_archives ping_indexnow fi -- 2.45.2 From 4122f4353b2fc6c15b523ad6bc02fa27cc92789c Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 07:49:43 +0000 Subject: [PATCH 10/37] dev: serve at dev.thermograph.org behind basic auth, bound to loopback (#111) --- infra/deploy/Caddyfile.vps1 | 54 ++++++++++++++++++++--- infra/deploy/RUNBOOK-vps1-vps2-cutover.md | 11 ++--- infra/deploy/env-topology.sh | 16 ++++--- infra/deploy/provision-dev.sh | 15 ++++--- infra/docker-compose.dev.yml | 12 ++--- 5 files changed, 77 insertions(+), 31 deletions(-) diff --git a/infra/deploy/Caddyfile.vps1 b/infra/deploy/Caddyfile.vps1 index 113e5db..77bbbac 100644 --- a/infra/deploy/Caddyfile.vps1 +++ b/infra/deploy/Caddyfile.vps1 @@ -5,13 +5,7 @@ # git.thermograph.org -> Forgejo (see deploy/forgejo/caddy-git.conf) # dashboard.thermograph.org -> Grafana (see observability/caddy-grafana.conf) # -# and hosts the **dev** environment, which is deliberately ABSENT from this -# file. Dev has no site block, no DNS record and no certificate: it publishes on -# the WireGuard address only (http://10.10.0.2:8137) and is reachable from the -# mesh and nowhere else. Dev runs whatever branch is in flight, on the same box -# as Forgejo and its CI — putting it behind a public hostname would be handing -# the internet an unreviewed build. If you ever need it reachable from a phone, -# join the phone to the mesh; do not add a site block here. +# and hosts the **dev** environment at dev.thermograph.org (below). # # beta.thermograph.org is NOT here either — beta moved to vps2, next to prod. # That is the one edit in this file most likely to be made by mistake: a "beta" @@ -54,6 +48,52 @@ http://75.119.132.91 { redir https://emigriffith.dev{uri} permanent } +# Dev. Publicly resolvable, deliberately NOT publicly usable. +# +# Dev runs whatever branch is in flight — including unreviewed ones — on the +# same box as Forgejo and its CI runner. Three things make that acceptable, and +# all three are load-bearing: +# +# 1. The stack binds LOOPBACK (deploy/env-topology.sh sets DEV_BIND_ADDR to +# 127.0.0.1), so this site block is the only route in. Not even the mesh +# reaches the app directly. +# 2. basic auth, so a stumbled-upon URL is not a running build. +# 3. X-Robots-Tag: noindex, so dev never competes with thermograph.org in a +# search index the way a second copy of the same content otherwise would. +# +# Only :8137 is proxied. The dev overlay does not publish the frontend at all +# (docker-compose.dev.yml `ports: !reset null`) — the backend reverse-proxies to +# it internally, so one upstream serves the whole site here, unlike vps2 where +# Caddy path-splits across two ports. +# +# The credential below is a bcrypt hash, not a password. Rotate with: +# caddy hash-password --plaintext '' +dev.thermograph.org { + encode zstd gzip + + header { + X-Robots-Tag "noindex, nofollow" + } + + basic_auth { + dev $2a$14$LU5sNyxbop3HOsjhXB6ZrOQiveqhbcYVE6.Wi0bydAv4QhNpj4HMC + } + + reverse_proxy 127.0.0.1:8137 { + health_uri /healthz + health_interval 15s + health_timeout 3s + health_status 2xx + } + + log { + output file /var/log/caddy/dev.log { + roll_size 20MiB + roll_keep 5 + } + } +} + # Optional: redirect www -> apex. Add the www CNAME/A record first, then # uncomment. # www.emigriffith.dev { diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md index fdb379a..408464b 100644 --- a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -317,13 +317,14 @@ SERVICE=all BACKEND_IMAGE_TAG= FRONTEND_IMAGE_TAG= \ infra/deploy/deploy-dev.sh ``` -**Verify it is mesh-only.** This is the security-relevant check of the whole -cutover — vps1 is a public box and dev runs unreviewed branches: +**Verify Caddy is the only way in.** This is the security-relevant check of the +whole cutover — vps1 is a public box and dev runs unreviewed branches: ``` -ss -ltnp | grep 8137 # MUST show 10.10.0.2:8137, never 0.0.0.0:8137 -curl -fsS -o /dev/null -w '%{http_code}\n' http://10.10.0.2:8137/healthz # from the mesh -curl --max-time 5 http://75.119.132.91:8137/healthz # MUST fail/refuse +ss -ltnp | grep 8137 # MUST show 127.0.0.1:8137, never 0.0.0.0 +curl --max-time 5 http://75.119.132.91:8137/healthz # MUST fail/refuse +curl -o /dev/null -w '%{http_code}\n' https://dev.thermograph.org/ # 401 without creds +curl -o /dev/null -w '%{http_code}\n' -u dev: https://dev.thermograph.org/healthz # 200 ``` Also confirm dev did **not** get the fleet's shared production credentials — diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh index 77b6c82..7e4ae38 100644 --- a/infra/deploy/env-topology.sh +++ b/infra/deploy/env-topology.sh @@ -176,12 +176,16 @@ thermograph_topology() { # instance lives on vps2 and holds real user data. TG_DB_NAME=thermograph TG_DB_USER=thermograph - # Mesh-only. vps1 is a public VPS, so the dev overlay must not publish on - # 0.0.0.0 the way it did on the operator's LAN box — dev runs whatever - # branch is in flight, including unreviewed ones. Reachable at - # http://10.10.0.2:8137 from anything on the WireGuard mesh, and from - # nowhere else. There is deliberately no dev DNS record and no Caddy site. - TG_BIND_ADDR=10.10.0.2 + # LOOPBACK ONLY. vps1 is a public VPS and dev runs whatever branch is in + # flight, including unreviewed ones, so the stack must never publish on + # 0.0.0.0 the way it did on the operator's LAN box. + # + # It was briefly mesh-only (10.10.0.2). It is now fronted by Caddy at + # dev.thermograph.org — TLS, HTTP basic auth, and X-Robots-Tag: noindex + # (see deploy/Caddyfile.vps1) — so the only reachable path is through that + # site block, and binding loopback is what guarantees it: nothing can + # reach the app without passing the password, not even from the mesh. + TG_BIND_ADDR=127.0.0.1 # Dev NEVER layers common.yaml — that file is the internet-facing fleet's # shared production credential set (both S3 keypairs, the VAPID signing # key, REGISTRY_TOKEN, the metrics token), and dev runs whatever branch is diff --git a/infra/deploy/provision-dev.sh b/infra/deploy/provision-dev.sh index 2ef56a8..1a2c776 100755 --- a/infra/deploy/provision-dev.sh +++ b/infra/deploy/provision-dev.sh @@ -13,9 +13,10 @@ # - It renders dev.yaml ALONE, never layering common.yaml (the fleet's shared # production credentials). vps1 also runs Forgejo and its CI, and dev runs # whatever branch is in flight — see the long note in render-secrets.sh. -# - It is MESH-ONLY: published on 10.10.0.2:8137 (wg0), never 0.0.0.0. No DNS -# record, no Caddy site, no TLS. Anything that can reach it is already on -# the WireGuard mesh. +# - It binds LOOPBACK, never 0.0.0.0. Caddy fronts it at dev.thermograph.org +# with TLS, basic auth and X-Robots-Tag: noindex (deploy/Caddyfile.vps1). +# Binding loopback is what makes that guard total rather than decorative: +# the site block is the only route in. # # Idempotent; re-run it after changing the branch or repointing the remote. set -euo pipefail @@ -70,10 +71,10 @@ Deploy it: SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- \\ $APP_DIR/infra/deploy/deploy-dev.sh -Reach it (mesh only): - http://${TG_BIND_ADDR}:8137 +Reach it: + https://dev.thermograph.org (basic auth; see deploy/Caddyfile.vps1) -Confirm it is NOT publicly exposed — this must show ${TG_BIND_ADDR}:8137 and -never 0.0.0.0:8137: +Confirm the app itself is NOT directly exposed — this must show +${TG_BIND_ADDR}:8137 and never 0.0.0.0:8137, so Caddy is the only way in: ss -ltnp | grep 8137 EOF diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index cfebe28..1f04222 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -22,12 +22,12 @@ # unreviewed branch is in flight to the entire internet — with no Caddy, no # TLS and no auth in front of it. # -# So the default is 127.0.0.1 (safe anywhere, including a laptop running -# `make dev-up`), and deploy.sh sets DEV_BIND_ADDR to the environment's mesh -# address from deploy/env-topology.sh — 10.10.0.2 on vps1. Dev is then -# reachable at http://10.10.0.2:8137 from anything on the WireGuard mesh and -# from nowhere else. There is deliberately no dev DNS record and no Caddy -# site block for it. +# So the default is 127.0.0.1 — safe anywhere, including a laptop running +# `make dev-up` — and on vps1 deploy/env-topology.sh keeps it there. Caddy +# fronts the stack at dev.thermograph.org with TLS, HTTP basic auth and +# X-Robots-Tag: noindex (deploy/Caddyfile.vps1). Loopback is what makes that +# guard total: the site block is the only route in, so nothing reaches the +# app without the password — not even from the WireGuard mesh. # 2. frontend's port publish is dropped entirely -- dev has no Caddy to reach it # directly, so it stays compose-internal-only, reached solely through # backend's own reverse-proxy fallback (THERMOGRAPH_FRONTEND_BASE_INTERNAL, -- 2.45.2 From 801c7671151970c5aeef02d92dac2d5d1f5de883 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 08:28:18 +0000 Subject: [PATCH 11/37] dev: leave /healthz outside the basic-auth boundary (#113) --- infra/deploy/Caddyfile.vps1 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/infra/deploy/Caddyfile.vps1 b/infra/deploy/Caddyfile.vps1 index 77bbbac..ebd6ac2 100644 --- a/infra/deploy/Caddyfile.vps1 +++ b/infra/deploy/Caddyfile.vps1 @@ -75,7 +75,14 @@ dev.thermograph.org { X-Robots-Tag "noindex, nofollow" } - basic_auth { + # /healthz is deliberately OUTSIDE the auth boundary. It returns liveness and + # nothing else — no data, no version, no configuration — and Centralis polls + # it to report dev in `fleet_status`. Behind basic auth that poll gets a 401 + # and dev reads as permanently down, which is how a monitoring blind spot + # gets created in the name of security. + @protected not path /healthz + + basic_auth @protected { dev $2a$14$LU5sNyxbop3HOsjhXB6ZrOQiveqhbcYVE6.Wi0bydAv4QhNpj4HMC } -- 2.45.2 From 8572c4aa6db353b8c1d75a7534c90edbe690c00d Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:16:55 +0000 Subject: [PATCH 12/37] accounts: sign in with Discord, sharing the existing link callback (#115) --- backend/accounts/models.py | 10 +- backend/accounts/schemas.py | 3 + backend/accounts/users.py | 20 ++ .../versions/0003_user_discord_only.py | 50 +++ backend/notifications/discord_link.py | 304 ++++++++++++++---- .../tests/notifications/test_discord_login.py | 251 +++++++++++++++ frontend/static/account.js | 76 ++++- frontend/static/style.css | 24 ++ infra/deploy/thermograph.env.example | 17 +- 9 files changed, 682 insertions(+), 73 deletions(-) create mode 100644 backend/alembic/versions/0003_user_discord_only.py create mode 100644 backend/tests/notifications/test_discord_login.py diff --git a/backend/accounts/models.py b/backend/accounts/models.py index 40b6df4..bc3dc54 100644 --- a/backend/accounts/models.py +++ b/backend/accounts/models.py @@ -34,13 +34,19 @@ class User(SQLAlchemyBaseUserTableUUID, Base): # is_superuser, is_verified. Optional extras: display_name: Mapped[str | None] = mapped_column(String(120), nullable=True) # Linked Discord account id (OAuth2 identify) — the key DM alerts reach the user - # by. NB: create_all only makes this on a fresh DB; an existing prod accounts.db - # needs the manual migration in deploy/migrations/ (no Alembic in this project). + # by, and the identity "Sign in with Discord" resolves an account from. Unique, + # so one Discord account can never own two Thermograph accounts. discord_id: Mapped[str | None] = mapped_column(String(32), unique=True, nullable=True) # Whether to also deliver alerts as a Discord DM. Set True on linking (an active # opt-in); the user can mute it while staying linked. Same migration caveat. discord_dm: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false()) + # True when the account was created by "Sign in with Discord" and so has no + # password its owner has ever seen (hashed_password is NOT NULL, so the row + # carries a generated one nobody knows). Load-bearing: unlinking Discord from + # such an account would remove its only way in, so discord_link.py refuses. + discord_only: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, + server_default=false()) class AccessToken(SQLAlchemyBaseAccessTokenTableUUID, Base): diff --git a/backend/accounts/schemas.py b/backend/accounts/schemas.py index 0898910..0715e12 100644 --- a/backend/accounts/schemas.py +++ b/backend/accounts/schemas.py @@ -24,6 +24,9 @@ class UserRead(schemas.BaseUser[uuid.UUID]): # flow (discord_link.py), not editable through the profile. discord_id: str | None = None discord_dm: bool = False + # True when Discord is the account's only credential, so the UI can explain + # why "Unlink Discord" is refused rather than just failing. + discord_only: bool = False class UserCreate(schemas.BaseUserCreate): diff --git a/backend/accounts/users.py b/backend/accounts/users.py index 8967126..a86a4d7 100644 --- a/backend/accounts/users.py +++ b/backend/accounts/users.py @@ -140,3 +140,23 @@ fastapi_users = FastAPIUsers[User, uuid.UUID](get_user_manager, [auth_backend]) # Dependencies handlers use to require / peek at the logged-in user. current_active_user = fastapi_users.current_user(active=True) current_user_optional = fastapi_users.current_user(active=True, optional=True) + + +async def attach_session(response, user, user_manager, access_token_db, request=None): + """Log `user` in by copying a fresh session cookie onto `response`. + + The library's routers own this for password login, but an OAuth callback has to + end in a *redirect* the browser follows, not the 204 the login router returns. + So mint the session the normal way and lift the Set-Cookie header off the + library's response rather than re-deriving the cookie's name/TTL/flags here — + duplicating those would let the two logins drift into issuing different cookies. + """ + strategy = get_database_strategy(access_token_db) + issued = await auth_backend.login(strategy, user) + for key, value in issued.raw_headers: + if key.lower() == b"set-cookie": + response.raw_headers.append((key, value)) + # The routers call this hook; a hand-rolled login has to as well, or Discord + # sign-ins go missing from the audit log that every password login appears in. + await user_manager.on_after_login(user, request, response) + return response diff --git a/backend/alembic/versions/0003_user_discord_only.py b/backend/alembic/versions/0003_user_discord_only.py new file mode 100644 index 0000000..7a84e6a --- /dev/null +++ b/backend/alembic/versions/0003_user_discord_only.py @@ -0,0 +1,50 @@ +"""user.discord_only — mark accounts whose only credential is Discord + +"Sign in with Discord" (notifications/discord_link.py) creates accounts for people +who never chose a password. ``hashed_password`` is NOT NULL, so those rows carry a +generated one nobody has ever seen; without a flag there is no way to tell such an +account apart from a password account that merely linked Discord later. That +distinction is load-bearing: unlinking Discord from a Discord-only account would +delete its only way in, and no reset-password router is mounted to recover it. + +Conditional on purpose. Revision 0001 builds the schema with +``Base.metadata.create_all``, which reads the *current* models.py — so a database +created after this column was added to the ORM already has it, and an +unconditional ``add_column`` would fail there with a duplicate-column error. Only +a database stamped before this revision is missing it. + +Revision ID: 0003_user_discord_only +Revises: 0002_climate_hypertables +Create Date: 2026-07-26 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0003_user_discord_only" +down_revision = "0002_climate_hypertables" +branch_labels = None +depends_on = None + + +def _has_column() -> bool: + cols = sa.inspect(op.get_bind()).get_columns("user") + return any(c["name"] == "discord_only" for c in cols) + + +def upgrade() -> None: + if _has_column(): + return + # server_default (not just a Python-side default) so the backfill of existing + # rows happens in the same statement: every account that predates Discord + # login has a password, so False is correct for all of them. + op.add_column( + "user", + sa.Column("discord_only", sa.Boolean(), nullable=False, + server_default=sa.false()), + ) + + +def downgrade() -> None: + if not _has_column(): + return + op.drop_column("user", "discord_only") diff --git a/backend/notifications/discord_link.py b/backend/notifications/discord_link.py index c4feffd..272d81c 100644 --- a/backend/notifications/discord_link.py +++ b/backend/notifications/discord_link.py @@ -1,16 +1,33 @@ -"""Link a Thermograph account to a Discord account via OAuth2 (identify scope). +"""Discord as an identity: sign in with it, and connect it to an account. -The Discord user id we store here is the durable key a later feature (DM alerts) -uses to reach the person. The flow is the standard authorization-code grant: +Two flows over one authorization-code grant, told apart by the signed `state`: - /link/start -> redirect the logged-in user to Discord's consent screen - /link/callback -> exchange the code, read the Discord user id, store it - /unlink -> forget it + link — /discord/link/start an already-signed-in user attaches a Discord + account to the session they're in + login — /discord/login/start an anonymous visitor authenticates, creating or + resolving a Thermograph account from the Discord + identity + +**Both come back to /discord/link/callback.** Discord only honours redirect URIs +pre-registered in the developer portal, so a second callback path would mean an +operator has to add one there before login could work anywhere. Sharing the +registered one keeps this deployable with no portal change; the flows separate on +the state's purpose, which is inside the HMAC and so can't be flipped by hand. `state` is signed with the app's auth secret (stdlib hmac, no new dependency) and -carries the Thermograph user id, so the callback can't be replayed or bound to a -different account. Everything requires an active session, so linking always acts on -the person who is actually logged in. +carries the purpose plus, for a link, the Thermograph user id — so a callback can't +be replayed, bound to a different account, or replayed *as the other flow*. + +How a login resolves to an account, in order: + + 1. A user already carrying this `discord_id` — the durable key, no email needed. + 2. Otherwise the Discord email, but **only if Discord reports it verified**. + That flag is the sole evidence the person owns the address; matching an + existing Thermograph account on an unverified one would hand that account to + whoever typed the address into Discord. Unverified is refused outright. + 3. No account for that email: create one. It has no password its owner has ever + seen, so it is flagged `discord_only` and unlink is refused (see `unlink`) — + otherwise unlinking would delete the only way back in. """ from __future__ import annotations @@ -23,15 +40,23 @@ import time import urllib.parse import httpx -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import RedirectResponse from pydantic import BaseModel -from sqlalchemy import update +from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from accounts.db import get_async_session from accounts.models import User -from accounts.users import SECRET, current_active_user +from accounts.users import ( + SECRET, + attach_session, + current_active_user, + current_user_optional, + get_access_token_db, + get_user_manager, +) +from core import audit router = APIRouter(tags=["discord"]) @@ -43,6 +68,11 @@ _AUTHORIZE = "https://discord.com/api/oauth2/authorize" _TOKEN = "https://discord.com/api/oauth2/token" _ME = "https://discord.com/api/users/@me" +# Linking only needs the account id. Signing in also needs the address to match or +# create a Thermograph account by, which is a separate scope Discord prompts for. +_SCOPE_LINK = "identify" +_SCOPE_LOGIN = "identify email" + # How long a start->callback round trip may take before the signed state expires. _STATE_TTL = 600 _TIMEOUT = httpx.Timeout(10.0) @@ -61,15 +91,22 @@ def _unb64(s: str) -> bytes: return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4)) -def _sign_state(uid: str) -> str: - payload = _b64(json.dumps({"u": uid, "t": int(time.time())}).encode()) +def _sign_state(uid: str | None, purpose: str = "link") -> str: + payload = _b64(json.dumps( + {"u": uid or "", "t": int(time.time()), "p": purpose} + ).encode()) mac = hmac.new(SECRET.encode(), payload.encode(), hashlib.sha256).hexdigest()[:32] return f"{payload}.{mac}" -def _verify_state(state: str) -> str | None: - """The Thermograph user id the state was minted for, or None if it's missing, - tampered, or older than the TTL.""" +def _verify_state(state: str, purpose: str = "link") -> str | None: + """The Thermograph user id this state was minted for, or None if it's missing, + tampered, expired, or was minted for a *different* purpose. + + A login state has no user id yet, so it verifies as ``""`` — falsy, but + distinct from the None that means "reject". Callers must test against None, + not truthiness, or a valid login state reads as a failure. + """ try: payload, mac = state.split(".", 1) except (ValueError, AttributeError): @@ -83,7 +120,11 @@ def _verify_state(state: str) -> str | None: return None if time.time() - float(data.get("t", 0)) > _STATE_TTL: return None - return data.get("u") + # States minted before the purpose field existed are links; they age out in + # _STATE_TTL, so this fallback only spans a deploy. + if data.get("p", "link") != purpose: + return None + return data.get("u") or "" def _redirect_uri(request: Request) -> str: @@ -94,52 +135,30 @@ def _redirect_uri(request: Request) -> str: return f"{proto}://{host}{BASE}/api/v2/discord/link/callback" -def _account_redirect(status: str) -> RedirectResponse: - # Back to the alerts page, which surfaces the link state; 303 so the browser - # issues a GET after the callback. - return RedirectResponse(url=f"{BASE}/subscriptions?discord={status}", status_code=303) - - -# --- routes ------------------------------------------------------------------ -@router.get("/discord/config") -async def discord_config(): - """Whether Discord account-linking is configured on this server. The account - menu uses it to show the "Link Discord" entry only when linking will actually - work — so an unconfigured server surfaces no dead-end Discord UI, and enabling - it later (setting the OAuth env vars) makes the entry appear on its own.""" - return {"enabled": enabled()} - - -@router.get("/discord/link/start") -async def link_start(request: Request, user=Depends(current_active_user)): - if not enabled(): - return _account_redirect("unavailable") +def _authorize_redirect(request: Request, scope: str, state: str) -> RedirectResponse: params = { "client_id": CLIENT_ID, "redirect_uri": _redirect_uri(request), "response_type": "code", - "scope": "identify", - "state": _sign_state(str(user.id)), + "scope": scope, + "state": state, "prompt": "consent", } - return RedirectResponse(url=f"{_AUTHORIZE}?{urllib.parse.urlencode(params)}", status_code=303) + return RedirectResponse(url=f"{_AUTHORIZE}?{urllib.parse.urlencode(params)}", + status_code=303) -@router.get("/discord/link/callback") -async def link_callback( - request: Request, - user=Depends(current_active_user), - session: AsyncSession = Depends(get_async_session), -): - if not enabled(): - return _account_redirect("unavailable") - # User declined on Discord's screen, or the state doesn't check out. - code = request.query_params.get("code") - state = request.query_params.get("state", "") - if not code: - return _account_redirect("cancelled") - if _verify_state(state) != str(user.id): - return _account_redirect("error") +def _account_redirect(status: str) -> RedirectResponse: + # Back to the alerts page, which surfaces the outcome as a toast; 303 so the + # browser issues a GET after the callback. The frontend serves this page at + # /alerts — the route is named for the feature, not for subscriptions.html. + return RedirectResponse(url=f"{BASE}/alerts?discord={status}", status_code=303) + + +# --- Discord API -------------------------------------------------------------- +async def _fetch_profile(request: Request, code: str) -> dict | None: + """Trade the authorization code for the Discord user object, or None if any + step fails. Callers treat None as a graceful error, never a crash.""" try: async with httpx.AsyncClient(timeout=_TIMEOUT) as client: tok = await client.post(_TOKEN, data={ @@ -150,16 +169,121 @@ async def link_callback( "redirect_uri": _redirect_uri(request), }, headers={"Content-Type": "application/x-www-form-urlencoded"}) if tok.status_code >= 300: - return _account_redirect("error") + return None access = tok.json().get("access_token") me = await client.get(_ME, headers={"Authorization": f"Bearer {access}"}) if me.status_code >= 300: - return _account_redirect("error") - discord_id = str(me.json().get("id") or "") + return None + profile = me.json() except Exception: # noqa: BLE001 - any network/parse failure is a graceful error + return None + return profile if isinstance(profile, dict) else None + + +async def _user_by_discord_id(session: AsyncSession, discord_id: str) -> User | None: + return ( + await session.execute(select(User).where(User.discord_id == discord_id)) + ).scalar_one_or_none() + + +async def _create_from_discord(user_manager, email: str, discord_id: str) -> User: + """Create an account for a Discord identity that matched none of ours. + + Deliberately goes around ``UserManager.create()``: there is no password to + validate, and its ``on_after_register`` mails a confirmation link for an + address Discord has already verified. The generated password exists only + because ``hashed_password`` is NOT NULL — nobody ever sees it, which is what + ``discord_only`` records. + """ + password = user_manager.password_helper.generate() + user = await user_manager.user_db.create({ + "email": email, + "hashed_password": user_manager.password_helper.hash(password), + "is_active": True, + "is_superuser": False, + "is_verified": True, + "discord_id": discord_id, + "discord_dm": True, + "discord_only": True, + }) + audit.log_activity("auth.register", {"user_id": str(user.id), "via": "discord"}) + return user + + +# --- routes ------------------------------------------------------------------ +@router.get("/discord/config") +async def discord_config(): + """Whether Discord is configured on this server. The account menu and the sign-in + modal use it to offer Discord only when it will actually work — so an + unconfigured server surfaces no dead-end Discord UI, and enabling it later + (setting the OAuth env vars) makes both entries appear on their own.""" + return {"enabled": enabled()} + + +@router.get("/discord/link/start") +async def link_start(request: Request, user=Depends(current_active_user)): + if not enabled(): + return _account_redirect("unavailable") + return _authorize_redirect(request, _SCOPE_LINK, _sign_state(str(user.id), "link")) + + +@router.get("/discord/login/start") +async def login_start(request: Request, user=Depends(current_user_optional)): + """Sign in with Discord. Unauthenticated by design — this is how you get a + session, not something you do with one.""" + if not enabled(): + return _account_redirect("unavailable") + # Someone already signed in who lands here wants to connect, not to be + # switched into whichever account the Discord identity resolves to. + if user is not None: + return _authorize_redirect(request, _SCOPE_LINK, _sign_state(str(user.id), "link")) + return _authorize_redirect(request, _SCOPE_LOGIN, _sign_state(None, "login")) + + +@router.get("/discord/link/callback") +async def link_callback( + request: Request, + user=Depends(current_user_optional), + session: AsyncSession = Depends(get_async_session), + user_manager=Depends(get_user_manager), + access_token_db=Depends(get_access_token_db), +): + """The single redirect target both flows return to (see the module docstring).""" + if not enabled(): + return _account_redirect("unavailable") + # User declined on Discord's screen, or the state doesn't check out. + code = request.query_params.get("code") + state = request.query_params.get("state", "") + if not code: + return _account_redirect("cancelled") + + # Which flow minted this state? The purpose is inside the HMAC, so a state + # can only verify under the one it was signed for. + uid = _verify_state(state, "link") + if uid is not None: + return await _finish_link(request, code, uid, user, session) + if _verify_state(state, "login") is not None: + return await _finish_login(request, code, session, user_manager, access_token_db) + return _account_redirect("error") + + +async def _finish_link(request: Request, code: str, uid: str, user, + session: AsyncSession) -> RedirectResponse: + # Linking always acts on the person actually logged in, and only on the one + # the state was minted for. + if user is None or str(user.id) != uid: return _account_redirect("error") + profile = await _fetch_profile(request, code) + if profile is None: + return _account_redirect("error") + discord_id = str(profile.get("id") or "") if not discord_id: return _account_redirect("error") + # discord_id is UNIQUE, so without this check the UPDATE would raise an + # IntegrityError and 500 instead of explaining itself. + owner = await _user_by_discord_id(session, discord_id) + if owner is not None and owner.id != user.id: + return _account_redirect("taken") # Linking is an active opt-in, so turn DM alerts on; the user can mute them # (POST /discord/dm) while staying linked. await session.execute( @@ -169,11 +293,73 @@ async def link_callback( return _account_redirect("linked") +async def _finish_login(request: Request, code: str, session: AsyncSession, + user_manager, access_token_db) -> RedirectResponse: + """Resolve the Discord identity to an account and sign in as it. + + Deliberately ignores any session already present. A login state carries no + user id — it can't, there is no user yet — so it is replayable against whoever + happens to be signed in. Treating it as a link would therefore be a forced- + linking takeover: an attacker who gets a victim to open this callback with a + code from the *attacker's* Discord would attach their identity to the victim's + account, then sign in as them. Only /discord/link/start, whose state is bound + to a specific user id, may link. The worst a replayed login state can do is + sign someone into the attacker's own account. + """ + profile = await _fetch_profile(request, code) + if profile is None: + return _account_redirect("error") + discord_id = str(profile.get("id") or "") + if not discord_id: + return _account_redirect("error") + + created = False + user = await _user_by_discord_id(session, discord_id) + if user is None: + # No account carries this Discord id, so fall back to the email — the only + # other identity Discord gives us, and only when Discord vouches for it. + email = str(profile.get("email") or "").strip().lower() + if not email: + return _account_redirect("noemail") + if profile.get("verified") is not True: + return _account_redirect("unverified") + user = await user_manager.user_db.get_by_email(email) + if user is None: + user = await _create_from_discord(user_manager, email, discord_id) + created = True + elif user.discord_id: + # That address belongs to an account already tied to a different + # Discord identity; connecting would silently move it. + return _account_redirect("mismatch") + elif not user.is_active: + # Checked before the link, not just before the session: a deactivated + # account must not quietly acquire a Discord identity on the way out. + return _account_redirect("inactive") + else: + await session.execute( + update(User).where(User.id == user.id) + .values(discord_id=discord_id, discord_dm=True) + ) + await session.commit() + if not user.is_active: + return _account_redirect("inactive") + response = _account_redirect("created" if created else "signedin") + return await attach_session(response, user, user_manager, access_token_db, request) + + @router.post("/discord/unlink", status_code=204) async def unlink( user=Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): + if user.discord_only: + # The account has no password anyone has ever seen, and no reset-password + # route is mounted to set one — unlinking here is unrecoverable. + raise HTTPException( + status_code=409, + detail="This account was created with Discord and has no password, " + "so unlinking would leave no way to sign in.", + ) await session.execute( update(User).where(User.id == user.id).values(discord_id=None, discord_dm=False) ) diff --git a/backend/tests/notifications/test_discord_login.py b/backend/tests/notifications/test_discord_login.py new file mode 100644 index 0000000..cc3d4f9 --- /dev/null +++ b/backend/tests/notifications/test_discord_login.py @@ -0,0 +1,251 @@ +"""Sign in with Discord: account resolution, session issuance, and the guards that +keep the login flow from becoming an account-takeover path. Discord HTTP is mocked; +reuses the throwaway accounts DB from conftest.""" +import pytest +from fastapi.testclient import TestClient + +from web import app as appmod +from accounts import db +from notifications import discord_link as dl + +V2 = "/thermograph/api/v2" +PW = "supersecret123" + + +@pytest.fixture(scope="module", autouse=True) +def _tables(): + db.Base.metadata.create_all(db.sync_engine) + + +@pytest.fixture(autouse=True) +def _configured(monkeypatch): + monkeypatch.setattr(dl, "CLIENT_ID", "app123") + monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") + + +class _Resp: + def __init__(self, code, data): self.status_code = code; self._data = data + def json(self): return self._data + + +def _mock_discord(monkeypatch, profile): + """Stand in for httpx.AsyncClient: token exchange, then /users/@me -> profile.""" + class _Client: + def __init__(self, *a, **k): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, url, **k): return _Resp(200, {"access_token": "tok"}) + async def get(self, url, **k): return _Resp(200, profile) + monkeypatch.setattr(dl.httpx, "AsyncClient", _Client) + + +def _register(client, email): + r = client.post(f"{V2}/auth/register", json={"email": email, "password": PW}) + assert r.status_code in (201, 400) + + +def _login(client, email): + _register(client, email) + assert client.post(f"{V2}/auth/login", + data={"username": email, "password": PW}).status_code == 204 + + +def _callback(client, state): + return client.get(f"{V2}/discord/link/callback?code=abc&state={state}", + follow_redirects=False) + + +# --- start ------------------------------------------------------------------- + +def test_login_start_needs_no_session_and_asks_for_email(): + c = TestClient(appmod.app) + r = c.get(f"{V2}/discord/login/start", follow_redirects=False) + assert r.status_code == 303 + loc = r.headers["location"] + assert loc.startswith("https://discord.com/api/oauth2/authorize") + # The email scope is what makes account resolution possible at all. + assert "scope=identify+email" in loc and "state=" in loc + + +def test_login_start_while_signed_in_links_instead_of_switching_account(): + """Someone already signed in wants to connect Discord, not be switched into + whichever account the Discord identity happens to resolve to.""" + c = TestClient(appmod.app) + _login(c, "already-in@example.com") + r = c.get(f"{V2}/discord/login/start", follow_redirects=False) + assert r.status_code == 303 + loc = r.headers["location"] + assert "scope=identify&" in loc or loc.endswith("scope=identify") + state = loc.split("state=")[1].split("&")[0] + uid = c.get(f"{V2}/users/me").json()["id"] + assert dl._verify_state(state, "link") == uid + + +def test_login_start_when_unconfigured_bounces_back(monkeypatch): + monkeypatch.setattr(dl, "CLIENT_ID", "") + monkeypatch.setattr(dl, "CLIENT_SECRET", "") + c = TestClient(appmod.app) + r = c.get(f"{V2}/discord/login/start", follow_redirects=False) + assert r.status_code == 303 and "discord=unavailable" in r.headers["location"] + + +# --- state purposes are not interchangeable ---------------------------------- + +def test_state_purposes_do_not_cross_over(): + login_state = dl._sign_state(None, "login") + link_state = dl._sign_state("user-123", "link") + # A login state carries no user id, but verifies as "" — distinct from reject. + assert dl._verify_state(login_state, "login") == "" + assert dl._verify_state(login_state, "link") is None + assert dl._verify_state(link_state, "link") == "user-123" + assert dl._verify_state(link_state, "login") is None + + +def test_a_login_state_cannot_link_the_signed_in_account(monkeypatch): + """Forced-linking guard. A login state carries no user id, so it is replayable + against whoever is signed in; if the callback treated it as a link, an attacker + could attach their own Discord identity to a victim's account and then sign in + as them. It must complete as a plain login instead.""" + _mock_discord(monkeypatch, {"id": "d-crossover", "email": "attacker@example.com", + "verified": True}) + c = TestClient(appmod.app) + _login(c, "crossover@example.com") + # A state whose purpose says login, but naming the signed-in user — the shape + # a forced-link attempt would take. + r = _callback(c, dl._sign_state(c.get(f"{V2}/users/me").json()["id"], "login")) + # Resolved as a login into the Discord identity's own account, not as a link. + assert "discord=created" in r.headers["location"] + assert c.get(f"{V2}/users/me").json()["email"] == "attacker@example.com" + # The account that had been signed in is untouched. + victim = TestClient(appmod.app) + _login(victim, "crossover@example.com") + assert victim.get(f"{V2}/users/me").json()["discord_id"] is None + + +# --- login resolves an account ----------------------------------------------- + +def test_login_creates_an_account_and_signs_in(monkeypatch): + _mock_discord(monkeypatch, {"id": "d-new", "email": "New.User@Example.com", + "verified": True}) + c = TestClient(appmod.app) + r = _callback(c, dl._sign_state(None, "login")) + assert r.status_code == 303 and "discord=created" in r.headers["location"] + me = c.get(f"{V2}/users/me") + assert me.status_code == 200 + body = me.json() + assert body["email"] == "new.user@example.com" # normalised + assert body["discord_id"] == "d-new" + assert body["is_verified"] is True # Discord vouched for it + assert body["discord_only"] is True + assert body["discord_dm"] is True + + +def test_login_recognises_a_previously_linked_account(monkeypatch): + """The discord_id is the durable key: it resolves the account with no email.""" + _mock_discord(monkeypatch, {"id": "d-returning", "email": "link-me@example.com", + "verified": True}) + linker = TestClient(appmod.app) + _login(linker, "link-me@example.com") + uid = linker.get(f"{V2}/users/me").json()["id"] + assert "discord=linked" in _callback(linker, dl._sign_state(uid, "link") + ).headers["location"] + # A fresh client, no session, and Discord now returns no email at all. + _mock_discord(monkeypatch, {"id": "d-returning"}) + c = TestClient(appmod.app) + r = _callback(c, dl._sign_state(None, "login")) + assert r.status_code == 303 and "discord=signedin" in r.headers["location"] + assert c.get(f"{V2}/users/me").json()["id"] == uid + + +def test_login_connects_a_verified_email_to_an_existing_account(monkeypatch): + """This is the 'connect the account with that' path: an existing password + account picks up the Discord id and is signed in.""" + c = TestClient(appmod.app) + _register(c, "existing@example.com") + _mock_discord(monkeypatch, {"id": "d-existing", "email": "existing@example.com", + "verified": True}) + fresh = TestClient(appmod.app) + r = _callback(fresh, dl._sign_state(None, "login")) + assert r.status_code == 303 and "discord=signedin" in r.headers["location"] + body = fresh.get(f"{V2}/users/me").json() + assert body["email"] == "existing@example.com" + assert body["discord_id"] == "d-existing" + # It kept its password, so it is not Discord-only and may unlink. + assert body["discord_only"] is False + + +def test_login_refuses_an_unverified_discord_email(monkeypatch): + """Discord's verified flag is the only evidence the person owns the address. + Without it, this would hand over any account whose email someone typed in.""" + c = TestClient(appmod.app) + _register(c, "victim@example.com") + _mock_discord(monkeypatch, {"id": "d-attacker", "email": "victim@example.com", + "verified": False}) + fresh = TestClient(appmod.app) + r = _callback(fresh, dl._sign_state(None, "login")) + assert r.status_code == 303 and "discord=unverified" in r.headers["location"] + assert fresh.get(f"{V2}/users/me").status_code == 401 # no session issued + # And the targeted account is untouched. + _login(c, "victim@example.com") + assert c.get(f"{V2}/users/me").json()["discord_id"] is None + + +def test_login_without_an_email_cannot_create_an_account(monkeypatch): + _mock_discord(monkeypatch, {"id": "d-noemail"}) + c = TestClient(appmod.app) + r = _callback(c, dl._sign_state(None, "login")) + assert r.status_code == 303 and "discord=noemail" in r.headers["location"] + assert c.get(f"{V2}/users/me").status_code == 401 + + +def test_login_will_not_move_an_account_between_discord_identities(monkeypatch): + """The email matches an account that is already linked to a different Discord + id — signing in would silently re-point it.""" + owner = TestClient(appmod.app) + _login(owner, "owned@example.com") + uid = owner.get(f"{V2}/users/me").json()["id"] + _mock_discord(monkeypatch, {"id": "d-owner", "email": "owned@example.com", + "verified": True}) + assert "discord=linked" in _callback(owner, dl._sign_state(uid, "link") + ).headers["location"] + # A second Discord account claiming the same address. + _mock_discord(monkeypatch, {"id": "d-interloper", "email": "owned@example.com", + "verified": True}) + fresh = TestClient(appmod.app) + r = _callback(fresh, dl._sign_state(None, "login")) + assert r.status_code == 303 and "discord=mismatch" in r.headers["location"] + assert fresh.get(f"{V2}/users/me").status_code == 401 + assert owner.get(f"{V2}/users/me").json()["discord_id"] == "d-owner" + + +# --- guards on the connected account ----------------------------------------- + +def test_linking_a_discord_account_someone_else_owns_is_refused(monkeypatch): + first = TestClient(appmod.app) + _login(first, "first-owner@example.com") + _mock_discord(monkeypatch, {"id": "d-contested", "email": "first-owner@example.com", + "verified": True}) + uid = first.get(f"{V2}/users/me").json()["id"] + assert "discord=linked" in _callback(first, dl._sign_state(uid, "link") + ).headers["location"] + # A different Thermograph account tries to claim the same Discord identity. + second = TestClient(appmod.app) + _login(second, "second-owner@example.com") + uid2 = second.get(f"{V2}/users/me").json()["id"] + r = _callback(second, dl._sign_state(uid2, "link")) + assert r.status_code == 303 and "discord=taken" in r.headers["location"] + assert second.get(f"{V2}/users/me").json()["discord_id"] is None + + +def test_a_discord_only_account_cannot_unlink_itself_into_a_lockout(monkeypatch): + _mock_discord(monkeypatch, {"id": "d-onlyway", "email": "onlyway@example.com", + "verified": True}) + c = TestClient(appmod.app) + assert "discord=created" in _callback(c, dl._sign_state(None, "login") + ).headers["location"] + r = c.post(f"{V2}/discord/unlink") + assert r.status_code == 409 + assert "no password" in r.json()["detail"] + assert c.get(f"{V2}/users/me").json()["discord_id"] == "d-onlyway" + # Muting DMs is still fine — that is not a lockout. + assert c.post(f"{V2}/discord/dm", json={"enabled": False}).status_code == 204 diff --git a/frontend/static/account.js b/frontend/static/account.js index 1d17376..f3eb0c6 100644 --- a/frontend/static/account.js +++ b/frontend/static/account.js @@ -10,7 +10,7 @@ // units.js is (imported by each page's entry module). let currentUser = null; // {id, email, display_name} or null -let discordEnabled = false; // is Discord linking configured on the server? +let discordEnabled = false; // is Discord configured on the server? let discordChecked = false; // have we asked yet? (once per page load) const authCbs = []; // notified on login/logout so pages can re-gate @@ -89,10 +89,10 @@ async function refreshUser() { const res = await apiFetch(uv("users/me")); currentUser = res.ok ? await res.json() : null; } catch (e) { currentUser = null; } - // Learn once whether Discord linking is configured, so the account menu only - // offers "Link Discord" when it will actually work (checked lazily, and only - // for a signed-in user — the menu never shows it to anyone else). - if (currentUser && !discordChecked) { + // Learn once whether Discord is configured, so we only offer "Link Discord" and + // "Continue with Discord" when they'll actually work. Asked regardless of sign-in + // state: the sign-in modal needs the answer precisely when nobody is signed in. + if (!discordChecked) { discordChecked = true; try { const r = await apiFetch(uv("discord/config")); @@ -126,6 +126,10 @@ export async function logout() { // --- auth modal -------------------------------------------------------------- let modal = null, mode = "login"; +// Discord's brand mark ("Clyde"), inline like the other icons here so it needs no +// extra request and inherits currentColor. +const DISCORD_IC = ``; + function buildModal() { modal = document.createElement("div"); modal.className = "mp-overlay acct-overlay"; @@ -145,6 +149,12 @@ function buildModal() { +

`; @@ -173,6 +183,9 @@ function setMode(m) { modal.querySelector(".acct-switch").innerHTML = isLogin ? 'Need an account? ' : 'Already have an account? '; + // One label for both modes: Discord signs you in or creates the account, + // whichever applies, so making the user pick first would be a false choice. + modal.querySelector(".acct-alt").hidden = !discordEnabled; showError(""); } @@ -371,7 +384,11 @@ function renderHeader() { My alerts ${currentUser.discord_id ? `` - + '' + // A Discord-created account has no password, so Discord is its only + // way in and the server refuses to unlink it — don't offer the button. + + (currentUser.discord_only + ? 'Signed in with Discord' + : '') : (discordEnabled ? `Link Discord` : "")} @@ -403,7 +420,17 @@ function renderHeader() { const unlinkBtn = el.querySelector(".acct-discord-unlink"); if (unlinkBtn) unlinkBtn.addEventListener("click", async () => { unlinkBtn.disabled = true; - try { await apiFetch(uv("discord/unlink"), { method: "POST" }); } catch (e) {} + try { + const res = await apiFetch(uv("discord/unlink"), { method: "POST" }); + // 409: the account was created with Discord and has no password, so + // unlinking would lock it out. Say why instead of appearing to do nothing. + if (res.status === 409) { + const body = await res.json().catch(() => null); + showToast((body && body.detail) || "Discord can't be unlinked from this account.", true); + unlinkBtn.disabled = false; + return; + } + } catch (e) {} await refreshUser(); emitAuth(); // repaint the popover in its unlinked state }); @@ -452,6 +479,30 @@ function showToast(msg, isError = false) { toastTimer = setTimeout(() => { el.hidden = true; }, 6000); } +// --- Discord OAuth outcomes -------------------------------------------------- +// The link/login callback can only talk back to us through a redirect, so it +// lands on ?discord=. Each status maps to [message, isError]; anything +// unrecognised is treated as a generic failure rather than shown raw. +const DISCORD_STATUS = { + linked: ["Discord connected.", false], + signedin: ["Signed in with Discord.", false], + created: ["Account created with Discord — welcome to Thermograph.", false], + cancelled: ["Discord sign-in cancelled.", false], + taken: ["That Discord account is already connected to another Thermograph account.", true], + mismatch: ["That email belongs to an account connected to a different Discord account. " + + "Sign in with your password to change it.", true], + noemail: ["Discord didn't share an email address, so there's no account to sign in to.", true], + unverified: ["Verify your email address with Discord first, then try again.", true], + inactive: ["That account is deactivated.", true], + unavailable: ["Discord sign-in isn't available on this server.", true], +}; + +function showDiscordStatus(status) { + const [msg, isError] = DISCORD_STATUS[status] + || ["Something went wrong with Discord. Please try again.", true]; + showToast(msg, isError); +} + // --- boot -------------------------------------------------------------------- (async function initAccount() { ensureAcctEl(); @@ -462,10 +513,19 @@ function showToast(msg, isError = false) { const params = new URLSearchParams(location.search); const token = params.get("verify_token"); - if (token) { + // Both of these are one-shot handoffs from a redirect; strip them together so a + // reload can't replay either, and so the two never fight over the URL. + const discordStatus = params.get("discord"); + if (token || discordStatus) { params.delete("verify_token"); + params.delete("discord"); const clean = location.pathname + (params.toString() ? `?${params}` : "") + location.hash; history.replaceState(null, "", clean); + } + // refreshUser() above already ran, and the callback's redirect carried the new + // session cookie, so the header is painted signed-in before this toast lands. + if (discordStatus) showDiscordStatus(discordStatus); + if (token) { try { await verifyEmail(token); await refreshUser(); diff --git a/frontend/static/style.css b/frontend/static/style.css index c794f15..f50112b 100644 --- a/frontend/static/style.css +++ b/frontend/static/style.css @@ -6,6 +6,9 @@ --text: #e7ecf2; --muted: #9aa6b2; --accent: #f0803c; + /* Discord's brand blurple. Fixed in both schemes on purpose: it identifies the + provider, so it is their colour rather than one of ours to re-theme. */ + --discord: #5865f2; /* temperature grades: 9-step diverging cold -> green -> hot (colorblind-safe). rec-* are the "Near Record" danger tiers — deliberately dark/saturated. */ @@ -422,6 +425,27 @@ main { max-width: 1200px; margin: 0 auto; padding: 20px 24px 60px; } color: var(--text); border: 1px solid var(--rec-hot, #d64545); } .acct-error[hidden] { display: none; } +/* "or / Continue with Discord" under the submit button. Hidden unless the server + reports Discord configured (account.js sets [hidden]). */ +.acct-alt { display: flex; flex-direction: column; gap: 10px; } +.acct-alt[hidden] { display: none; } +.acct-or { + display: flex; align-items: center; gap: 10px; + font-size: 12px; color: var(--muted); text-transform: uppercase; letter-spacing: .06em; +} +.acct-or::before, .acct-or::after { + content: ""; flex: 1; height: 1px; background: var(--border); +} +.acct-discord-login { + display: flex; align-items: center; justify-content: center; gap: 9px; + padding: 12px 16px; min-height: 44px; border-radius: 10px; + background: var(--discord); color: #fff; font-weight: 700; font-size: 15px; + text-decoration: none; border: 1px solid transparent; +} +.acct-discord-login:hover { filter: brightness(1.08); } +.acct-discord-login svg { width: 20px; height: 15px; flex: none; } + +.acct-pop-note { display: block; padding: 9px 12px; font-size: 13px; } .acct-switch { margin: 0; font-size: 13px; color: var(--muted); text-align: center; } .acct-switch button { background: none; border: 0; color: var(--accent); cursor: pointer; diff --git a/infra/deploy/thermograph.env.example b/infra/deploy/thermograph.env.example index f35537f..ef158f8 100644 --- a/infra/deploy/thermograph.env.example +++ b/infra/deploy/thermograph.env.example @@ -210,10 +210,19 @@ THERMOGRAPH_BASE_URL=https://thermograph.org # Discord allows ONE gateway connection per bot token, so enable this in exactly # one environment — prod, the only vault with the token. #THERMOGRAPH_DISCORD_BOT= -# Account linking (OAuth2 "identify"): lets a signed-in user connect their Discord -# account, storing their Discord user id for DM alerts. CLIENT_ID is the same App -# ID above. The CLIENT_SECRET is from OAuth2 -> Client Secret (a credential). In the -# portal, add the redirect: https://thermograph.org/api/v2/discord/link/callback +# Discord as an identity (notifications/discord_link.py). One pair of credentials +# gates BOTH flows, and setting them makes both appear in the UI at once: +# - account linking (scope "identify") — a signed-in user connects their Discord +# account, storing their Discord user id for DM alerts; +# - sign in with Discord (scope "identify email") — an anonymous visitor +# authenticates, resolving or creating an account from the Discord identity. +# CLIENT_ID is the same App ID above. The CLIENT_SECRET is from OAuth2 -> Client +# Secret (a credential). In the portal, add the redirect: +# https://thermograph.org/api/v2/discord/link/callback +# Both flows come back to that one path (the purpose is carried in the signed +# state), so enabling sign-in needs no new redirect registered. Scopes are asked +# for per request, not registered, so the email scope needs no portal change +# either. An environment without these two vars simply shows no Discord UI. #THERMOGRAPH_DISCORD_CLIENT_SECRET= # Gateway bot (opt-in): holds a live websocket so the bot replies to messages that # @mention it (or DM it) with a city grade — e.g. "@Thermograph Phoenix". Reuses -- 2.45.2 From 67284dfe2e6938d1f5a98bd32043bd5c0613b3ee Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:27:40 +0000 Subject: [PATCH 13/37] bookmarks: account-synced saved locations API --- backend/accounts/api_accounts.py | 181 ++++++++++++++++++++++++++++++- 1 file changed, 178 insertions(+), 3 deletions(-) diff --git a/backend/accounts/api_accounts.py b/backend/accounts/api_accounts.py index 96f3276..6e83350 100644 --- a/backend/accounts/api_accounts.py +++ b/backend/accounts/api_accounts.py @@ -1,4 +1,4 @@ -"""Authenticated API for subscriptions and notifications. +"""Authenticated API for subscriptions, notifications, and bookmarks. All routes require a logged-in user (the fastapi-users cookie session) and are scoped to that user — a row that isn't theirs reads as 404, never 403, so the API @@ -7,7 +7,7 @@ doesn't leak which ids exist. Mounted under {BASE}/api/v2 alongside the rest of import os import time -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from fastapi.concurrency import run_in_threadpool from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -17,8 +17,15 @@ from data import climate from data import grid from notifications import push from accounts.db import get_async_session, get_read_session -from accounts.models import Notification, PushSubscription, Subscription +from accounts.models import Bookmark, Notification, PushSubscription, Subscription from accounts.schemas import ( + BOOKMARK_MAX_PER_USER, + BookmarkImportIn, + BookmarkImportOut, + BookmarkIn, + BookmarkList, + BookmarkOut, + BookmarkPatch, NotificationList, NotificationOut, PushSubscriptionIn, @@ -323,3 +330,171 @@ async def push_test( # `failed` (delivery rejected — e.g. VAPID key mismatch) is surfaced so the client # can tell "request accepted" apart from "actually delivered". return {"devices": len(rows), "sent": sent, "pruned": pruned, "failed": failed} + + +# --- bookmarks ----------------------------------------------------------------- +async def _owned_bookmark(session: AsyncSession, bookmark_id: int, user) -> Bookmark: + bookmark = ( + await session.execute( + select(Bookmark).where( + Bookmark.id == bookmark_id, Bookmark.user_id == user.id + ) + ) + ).scalar_one_or_none() + if bookmark is None: + raise HTTPException(status_code=404, detail="Bookmark not found.") + return bookmark + + +@router.get("/bookmarks", response_model=BookmarkList) +async def list_bookmarks( + user=Depends(current_active_user), + session: AsyncSession = Depends(get_read_session), # pure read → read-only engine +): + rows = ( + await session.execute( + select(Bookmark) + .where(Bookmark.user_id == user.id) + .order_by(Bookmark.created_at) + ) + ).scalars().all() + return BookmarkList(bookmarks=[BookmarkOut.model_validate(r) for r in rows]) + + +@router.post("/bookmarks", response_model=BookmarkOut, status_code=201) +async def create_bookmark( + body: BookmarkIn, + response: Response, + user=Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + """Create a bookmark, or — if one already exists for this (user, cell) — + rename it in place and return 200. Idempotent upsert rather than a 409, so + re-bookmarking a place you already saved is never an error.""" + cell = grid.snap(body.lat, body.lon) + cell_id = cell["id"] + + existing = ( + await session.execute( + select(Bookmark).where( + Bookmark.user_id == user.id, Bookmark.cell_id == cell_id + ) + ) + ).scalar_one_or_none() + if existing is not None: + existing.label = body.label + await session.commit() + await session.refresh(existing) + audit.log_activity("bookmark.update", {"user_id": str(user.id), + "bookmark_id": existing.id}) + response.status_code = 200 + return existing + + count = ( + await session.execute( + select(func.count()).select_from(Bookmark).where(Bookmark.user_id == user.id) + ) + ).scalar_one() + if count >= BOOKMARK_MAX_PER_USER: + raise HTTPException( + status_code=409, + detail=f"You can have at most {BOOKMARK_MAX_PER_USER} bookmarks.", + ) + + bookmark = Bookmark( + user_id=user.id, + cell_id=cell_id, + label=body.label, + lat=body.lat, + lon=body.lon, + ) + session.add(bookmark) + await session.commit() + await session.refresh(bookmark) + audit.log_activity("bookmark.create", {"user_id": str(user.id), "bookmark_id": bookmark.id, + "cell_id": cell_id}) + return bookmark + + +@router.patch("/bookmarks/{bookmark_id}", response_model=BookmarkOut) +async def update_bookmark( + bookmark_id: int, + body: BookmarkPatch, + user=Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + bookmark = await _owned_bookmark(session, bookmark_id, user) + bookmark.label = body.label + await session.commit() + await session.refresh(bookmark) + audit.log_activity("bookmark.rename", {"user_id": str(user.id), "bookmark_id": bookmark.id}) + return bookmark + + +@router.delete("/bookmarks/{bookmark_id}", status_code=204) +async def delete_bookmark( + bookmark_id: int, + user=Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + bookmark = await _owned_bookmark(session, bookmark_id, user) + await session.delete(bookmark) + await session.commit() + audit.log_activity("bookmark.delete", {"user_id": str(user.id), "bookmark_id": bookmark_id}) + + +@router.post("/bookmarks/import", response_model=BookmarkImportOut) +async def import_bookmarks( + body: BookmarkImportIn, + user=Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + """Bulk-import a visitor's localStorage bookmarks at login. Existing rows keep + their label (never overwritten); duplicates within the payload itself are also + skipped. Stops importing once the user's total hits BOOKMARK_MAX_PER_USER — + any remaining items count as skipped rather than erroring.""" + existing_rows = ( + await session.execute(select(Bookmark).where(Bookmark.user_id == user.id)) + ).scalars().all() + existing_cells = {r.cell_id for r in existing_rows} + count = len(existing_rows) + + imported = 0 + skipped = 0 + seen_cells: set[str] = set() + for item in body.items: + cell_id = grid.snap(item.lat, item.lon)["id"] + if cell_id in existing_cells or cell_id in seen_cells: + skipped += 1 + continue + if count >= BOOKMARK_MAX_PER_USER: + skipped += 1 + continue + session.add(Bookmark( + user_id=user.id, + cell_id=cell_id, + label=item.label, + lat=item.lat, + lon=item.lon, + )) + seen_cells.add(cell_id) + count += 1 + imported += 1 + + if imported: + await session.commit() + audit.log_activity("bookmark.import", {"user_id": str(user.id), + "imported": imported, "skipped": skipped}) + + rows = ( + await session.execute( + select(Bookmark) + .where(Bookmark.user_id == user.id) + .order_by(Bookmark.created_at) + ) + ).scalars().all() + return BookmarkImportOut( + imported=imported, + skipped=skipped, + bookmarks=[BookmarkOut.model_validate(r) for r in rows], + ) -- 2.45.2 From 3fafd6760076de398f94da78f10d268573bb49b4 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:27:41 +0000 Subject: [PATCH 14/37] bookmarks: account-synced saved locations API --- backend/accounts/models.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/backend/accounts/models.py b/backend/accounts/models.py index bc3dc54..56ed510 100644 --- a/backend/accounts/models.py +++ b/backend/accounts/models.py @@ -195,3 +195,31 @@ class PendingDigest(Base): # duplicating, which is what makes the form idempotent. UniqueConstraint("email", name="uq_pending_digest_email"), ) + + +class Bookmark(Base): + """A saved location, owned by a user — the "bookmarked locations" feature. + + Keyed like ``Subscription`` on ``grid.snap(lat, lon)["id"]``: one bookmark per + (user, cell), so re-bookmarking the same cell is an upsert of the label rather + than a duplicate row (see ``create_bookmark`` / ``import_bookmarks`` in + api_accounts.py, which enforce this at the application layer too). + """ + __tablename__ = "bookmark" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + user_id: Mapped[uuid.UUID] = mapped_column( + GUID, ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + # grid.snap(lat, lon)["id"] — the stable per-location key used across the app. + cell_id: Mapped[str] = mapped_column(String(40), nullable=False) + label: Mapped[str] = mapped_column(String(80), nullable=False) + lat: Mapped[float] = mapped_column(Float, nullable=False) + lon: Mapped[float] = mapped_column(Float, nullable=False) + created_at: Mapped[float] = mapped_column(Float, nullable=False, default=time.time) + + __table_args__ = ( + # One bookmark per location per user — re-bookmarking upserts the label. + UniqueConstraint("user_id", "cell_id", name="uq_bookmark_user_cell"), + Index("idx_bookmark_user", "user_id"), + ) -- 2.45.2 From 53fd5d92253d7b63f60c42b049dcfbca6bcaf843 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:27:43 +0000 Subject: [PATCH 15/37] bookmarks: account-synced saved locations API --- backend/accounts/schemas.py | 71 +++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/backend/accounts/schemas.py b/backend/accounts/schemas.py index 0715e12..7385240 100644 --- a/backend/accounts/schemas.py +++ b/backend/accounts/schemas.py @@ -16,6 +16,11 @@ from data import grading ALLOWED_METRICS = tuple(grading.CLIMO_METRICS) DEFAULT_METRICS = ["tmax", "feels", "precip"] +# Bookmark limits, shared by the single-create and bulk-import paths. +BOOKMARK_LABEL_MAX_LEN = 80 +BOOKMARK_MAX_PER_USER = 200 +BOOKMARK_IMPORT_MAX_ITEMS = 200 + # --- users (fastapi-users) --------------------------------------------------- class UserRead(schemas.BaseUser[uuid.UUID]): @@ -135,3 +140,69 @@ class PushSubscriptionIn(BaseModel): class PushUnsubscribeIn(BaseModel): endpoint: str + + +# --- bookmarks ---------------------------------------------------------------- +def _check_label(v: str) -> str: + v = v.strip() + if not v: + raise ValueError("label is required") + if len(v) > BOOKMARK_LABEL_MAX_LEN: + raise ValueError(f"label must be at most {BOOKMARK_LABEL_MAX_LEN} characters") + return v + + +class BookmarkIn(BaseModel): + lat: float = Field(ge=-90, le=90) + lon: float = Field(ge=-180, le=180) + label: str + + @field_validator("label") + @classmethod + def _label(cls, v): + return _check_label(v) + + +class BookmarkPatch(BaseModel): + label: str + + @field_validator("label") + @classmethod + def _label(cls, v): + return _check_label(v) + + +class BookmarkOut(BaseModel): + id: int + cell_id: str + label: str + lat: float + lon: float + created_at: float + + model_config = {"from_attributes": True} + + +class BookmarkList(BaseModel): + bookmarks: list[BookmarkOut] + + +class BookmarkImportItem(BaseModel): + lat: float = Field(ge=-90, le=90) + lon: float = Field(ge=-180, le=180) + label: str + + @field_validator("label") + @classmethod + def _label(cls, v): + return _check_label(v) + + +class BookmarkImportIn(BaseModel): + items: list[BookmarkImportItem] = Field(max_length=BOOKMARK_IMPORT_MAX_ITEMS) + + +class BookmarkImportOut(BaseModel): + imported: int + skipped: int + bookmarks: list[BookmarkOut] -- 2.45.2 From 58e3a8d625c7d6464f84fb2585c1fca78bf34e7c Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:27:44 +0000 Subject: [PATCH 16/37] bookmarks: account-synced saved locations API --- backend/alembic/versions/0004_bookmarks.py | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 backend/alembic/versions/0004_bookmarks.py diff --git a/backend/alembic/versions/0004_bookmarks.py b/backend/alembic/versions/0004_bookmarks.py new file mode 100644 index 0000000..436add5 --- /dev/null +++ b/backend/alembic/versions/0004_bookmarks.py @@ -0,0 +1,45 @@ +"""bookmark: saved locations + +Adds the ``bookmark`` table backing the "bookmarked locations" feature — one row +per (user, grid cell), following the same shape as ``subscription``. + +Conditional on purpose, like 0003_user_discord_only: revision 0001 builds the +schema from the *current* models.py via ``Base.metadata.create_all``, so a +database created after ``Bookmark`` was added to the ORM (fresh envs, most tests) +already has the table, and an unconditional ``create_table`` would fail there +with a duplicate-table error. Only a database stamped before this revision is +missing it. + +Revision ID: 0004_bookmarks +Revises: 0003_user_discord_only +Create Date: 2026-07-26 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0004_bookmarks" +down_revision = "0003_user_discord_only" +branch_labels = None +depends_on = None + + +def _has_table() -> bool: + return sa.inspect(op.get_bind()).has_table("bookmark") + + +def upgrade() -> None: + if _has_table(): + return + # Built from the live ORM table (accounts.models.Bookmark) rather than + # hand-duplicated column definitions, so this can never drift from models.py. + from accounts.models import Bookmark + + Bookmark.__table__.create(bind=op.get_bind()) + + +def downgrade() -> None: + if not _has_table(): + return + from accounts.models import Bookmark + + Bookmark.__table__.drop(bind=op.get_bind()) -- 2.45.2 From 41858f3e4b1d73094b554287a417858f74587e21 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:27:45 +0000 Subject: [PATCH 17/37] bookmarks: account-synced saved locations API --- backend/tests/accounts/test_bookmarks.py | 160 +++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 backend/tests/accounts/test_bookmarks.py diff --git a/backend/tests/accounts/test_bookmarks.py b/backend/tests/accounts/test_bookmarks.py new file mode 100644 index 0000000..c155abd --- /dev/null +++ b/backend/tests/accounts/test_bookmarks.py @@ -0,0 +1,160 @@ +"""Route-level tests for the bookmarks feature: CRUD, upsert-on-duplicate, +ownership, auth, bulk import (dedupe/counts), and the per-user cap. Uses a +throwaway accounts DB (conftest points THERMOGRAPH_ACCOUNTS_DB at a temp file), +same as test_api_accounts.py.""" +import pytest +from fastapi.testclient import TestClient + +from web import app as appmod +from accounts import db + +V2 = "/thermograph/api/v2" +PW = "supersecret123" + + +@pytest.fixture(scope="module", autouse=True) +def _tables(): + # TestClient(app) (no context manager) skips the lifespan, so create the + # account tables directly against the temp DB. + db.Base.metadata.create_all(db.sync_engine) + + +def _client(): + return TestClient(appmod.app) + + +def _login(client, email): + r = client.post(f"{V2}/auth/register", json={"email": email, "password": PW}) + assert r.status_code in (201, 400) # 400 only if the email already exists + r = client.post(f"{V2}/auth/login", data={"username": email, "password": PW}) + assert r.status_code == 204 # sets the session cookie on the client + + +def test_bookmarks_require_auth(): + c = _client() + assert c.get(f"{V2}/bookmarks").status_code == 401 + assert c.post(f"{V2}/bookmarks", json={"lat": 1, "lon": 1, "label": "x"}).status_code == 401 + assert c.post(f"{V2}/bookmarks/import", json={"items": []}).status_code == 401 + + +def test_bookmark_crud(): + c = _client() + _login(c, "bm-crud@example.com") + + r = c.post(f"{V2}/bookmarks", json={"lat": 47.6, "lon": -122.3, "label": "Seattle"}) + assert r.status_code == 201 + bm = r.json() + assert bm["cell_id"] and bm["label"] == "Seattle" + assert bm["lat"] == 47.6 and bm["lon"] == -122.3 + + listed = c.get(f"{V2}/bookmarks").json()["bookmarks"] + assert len(listed) == 1 and listed[0]["id"] == bm["id"] + + patched = c.patch(f"{V2}/bookmarks/{bm['id']}", json={"label": "Home"}) + assert patched.status_code == 200 + assert patched.json()["label"] == "Home" + + assert c.delete(f"{V2}/bookmarks/{bm['id']}").status_code == 204 + assert c.get(f"{V2}/bookmarks").json()["bookmarks"] == [] + + +def test_bookmark_upsert_on_duplicate(): + c = _client() + _login(c, "bm-upsert@example.com") + body = {"lat": 33.4, "lon": -112.0, "label": "Phoenix"} + + first = c.post(f"{V2}/bookmarks", json=body) + assert first.status_code == 201 + bm_id = first.json()["id"] + + # Re-bookmarking the same location (same cell) upserts the label — 200, not 409. + second = c.post(f"{V2}/bookmarks", json={**body, "label": "Phoenix Home"}) + assert second.status_code == 200 + assert second.json()["id"] == bm_id + assert second.json()["label"] == "Phoenix Home" + + assert len(c.get(f"{V2}/bookmarks").json()["bookmarks"]) == 1 + + +def test_bookmark_validation(): + c = _client() + _login(c, "bm-valid@example.com") + assert c.post(f"{V2}/bookmarks", json={"lat": 1, "lon": 1, "label": " "}).status_code == 422 + assert c.post(f"{V2}/bookmarks", json={"lat": 1, "lon": 1, "label": "x" * 81}).status_code == 422 + assert c.post(f"{V2}/bookmarks", json={"lat": 91, "lon": 1, "label": "x"}).status_code == 422 + assert c.post(f"{V2}/bookmarks", json={"lat": 1, "lon": 181, "label": "x"}).status_code == 422 + # a label that's only over-length due to surrounding whitespace is fine once trimmed + ok = c.post(f"{V2}/bookmarks", json={"lat": 2, "lon": 2, "label": " Trimmed "}) + assert ok.status_code == 201 and ok.json()["label"] == "Trimmed" + + +def test_ownership_is_404(): + a, b = _client(), _client() + _login(a, "bm-owner-a@example.com") + _login(b, "bm-owner-b@example.com") + bm_id = a.post(f"{V2}/bookmarks", json={"lat": 40.7, "lon": -74.0, "label": "NYC"}).json()["id"] + + # B cannot see, patch, or delete A's bookmark — 404, not 403 (no existence leak) + assert b.patch(f"{V2}/bookmarks/{bm_id}", json={"label": "Nope"}).status_code == 404 + assert b.delete(f"{V2}/bookmarks/{bm_id}").status_code == 404 + assert b.get(f"{V2}/bookmarks").json()["bookmarks"] == [] + + +def test_bookmark_import_dedupe_and_counts(): + c = _client() + _login(c, "bm-import@example.com") + # Pre-existing bookmark at the Seattle cell, with a label import must NOT overwrite. + existing = c.post( + f"{V2}/bookmarks", json={"lat": 47.6, "lon": -122.3, "label": "Original"} + ).json() + assert existing["label"] == "Original" + + r = c.post(f"{V2}/bookmarks/import", json={"items": [ + {"lat": 47.6, "lon": -122.3, "label": "Should Not Overwrite"}, # dup of existing cell + {"lat": 34.0, "lon": -118.2, "label": "LA"}, # new + {"lat": 34.0, "lon": -118.2, "label": "LA Dup"}, # dup within payload + {"lat": 51.5, "lon": -0.1, "label": "London"}, # new + ]}) + assert r.status_code == 200 + body = r.json() + assert body["imported"] == 2 + assert body["skipped"] == 2 + assert len(body["bookmarks"]) == 3 + + by_label = {b["label"] for b in body["bookmarks"]} + assert by_label == {"Original", "LA", "London"} # existing label untouched + + +def test_bookmark_cap(): + c = _client() + _login(c, "bm-cap@example.com") + for i in range(200): + r = c.post(f"{V2}/bookmarks", json={"lat": i * 0.1, "lon": 0.0, "label": f"spot-{i}"}) + assert r.status_code == 201, r.text + + over = c.post(f"{V2}/bookmarks", json={"lat": 89.0, "lon": 179.0, "label": "one too many"}) + assert over.status_code == 409 + + assert len(c.get(f"{V2}/bookmarks").json()["bookmarks"]) == 200 + + +def test_bookmark_import_stops_at_cap(): + c = _client() + _login(c, "bm-import-cap@example.com") + for i in range(198): + r = c.post(f"{V2}/bookmarks", json={"lat": i * 0.1, "lon": 10.0, "label": f"spot-{i}"}) + assert r.status_code == 201, r.text + + r = c.post(f"{V2}/bookmarks/import", json={"items": [ + {"lat": 60.0, "lon": 60.0, "label": "a"}, + {"lat": 61.0, "lon": 61.0, "label": "b"}, + {"lat": 62.0, "lon": 62.0, "label": "c"}, + {"lat": 63.0, "lon": 63.0, "label": "d"}, + {"lat": 64.0, "lon": 64.0, "label": "e"}, + ]}) + assert r.status_code == 200 + body = r.json() + # Only 2 slots remained before the 200 cap; the rest count as skipped. + assert body["imported"] == 2 + assert body["skipped"] == 3 + assert len(body["bookmarks"]) == 200 -- 2.45.2 From 4e1c4a419741be057e5948268c2983b063772f8b Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:36:31 +0000 Subject: [PATCH 18/37] bookmarks: map + account UI for saved locations --- frontend/static/bookmarks.js | 182 +++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 frontend/static/bookmarks.js diff --git a/frontend/static/bookmarks.js b/frontend/static/bookmarks.js new file mode 100644 index 0000000..361adfa --- /dev/null +++ b/frontend/static/bookmarks.js @@ -0,0 +1,182 @@ +// Bookmarked locations: the single source of truth for "places you've saved". +// Logged out, this is pure localStorage. Logged in, the server is authoritative +// and this module mirrors it into localStorage so the list still renders +// instantly (and offline) on the next load. Every consumer (the map page's +// star toggle + saved-locations dropdown, the alerts page's Saved locations +// section) reads through list()/subscribe() and never touches storage or the +// API directly. +import { apiFetch, onAuthChange, uv } from "./account.js"; + +const LS_KEY = "thermograph:bookmarks"; +const DISMISS_KEY = "thermograph:bookmarks:import-dismissed"; +const MAX_LOCAL = 200; + +// A stable local key for a spot: rounded to ~4dp (~11m), so re-picking +// "the same place" a pixel off the first pin still matches. +export function cellId(lat, lon) { + return `${lat.toFixed(4)},${lon.toFixed(4)}`; +} + +function readLocal() { + try { + const arr = JSON.parse(localStorage.getItem(LS_KEY)); + return Array.isArray(arr) + ? arr.filter((b) => b && typeof b.lat === "number" && typeof b.lon === "number") + : []; + } catch (e) { return []; } +} +function writeLocal(rows) { + try { localStorage.setItem(LS_KEY, JSON.stringify(rows.slice(0, MAX_LOCAL))); } catch (e) { /* private mode, quota, etc. — degrade quietly */ } +} + +let cache = readLocal(); // what every consumer renders +let pending = []; // local-only bookmarks a signed-in user hasn't imported yet +let signedIn = false; +const subs = []; + +function notify() { subs.forEach((cb) => { try { cb(cache.slice()); } catch (e) {} }); } + +/** Subscribe to bookmark-list changes. Returns an unsubscribe function. */ +export function subscribe(cb) { + subs.push(cb); + return () => { const i = subs.indexOf(cb); if (i > -1) subs.splice(i, 1); }; +} + +export function list() { return cache.slice(); } +export function find(lat, lon) { + const id = cellId(lat, lon); + return cache.find((b) => cellId(b.lat, b.lon) === id) || null; +} +export function isBookmarked(lat, lon) { return !!find(lat, lon); } +export function pendingImportCount() { return pending.length; } + +export function importDismissed() { + try { return localStorage.getItem(DISMISS_KEY) === "1"; } catch (e) { return false; } +} +export function dismissImportPrompt() { + try { localStorage.setItem(DISMISS_KEY, "1"); } catch (e) {} + pending = []; + notify(); +} + +function fromServerRow(b) { + return { + id: b.id != null ? String(b.id) : (b.cell_id || cellId(b.lat, b.lon)), + cell_id: b.cell_id || null, + lat: b.lat, lon: b.lon, + label: b.label || null, + created_at: b.created_at || Date.now() / 1000, + }; +} + +// Returns the server's list, or null on 401 (signed out) / offline / error — +// callers treat null as "can't say, fall back to local" rather than "empty". +async function fetchServerList() { + try { + const res = await apiFetch(uv("bookmarks")); + if (res.status === 401) { signedIn = false; return null; } + if (!res.ok) return null; + signedIn = true; + const data = await res.json(); + return Array.isArray(data.bookmarks) ? data.bookmarks : []; + } catch (e) { return null; } +} + +// Reconciles cache with the server, diffing against whatever was in +// localStorage right before this call so any local-only spots can be offered +// up for import instead of silently vanishing behind the server's list. +async function syncFromServer(preSyncLocal) { + const rows = await fetchServerList(); + if (rows == null) return; // signed out or offline: leave cache as-is + const serverList = rows.map(fromServerRow); + const serverIds = new Set(serverList.map((b) => cellId(b.lat, b.lon))); + const localOnly = preSyncLocal.filter((b) => !serverIds.has(cellId(b.lat, b.lon))); + cache = serverList; + writeLocal(cache); + pending = importDismissed() ? [] : localOnly; + notify(); +} + +/** Save (or re-save with a new label) the given spot. Upserts either way. */ +export async function add(lat, lon, label) { + const id = cellId(lat, lon); + if (signedIn) { + try { + const res = await apiFetch(uv("bookmarks"), { method: "POST", json: { lat, lon, label: label || undefined } }); + if (res.ok) { + const rows = await fetchServerList(); + if (rows != null) { cache = rows.map(fromServerRow); writeLocal(cache); notify(); return find(lat, lon); } + } + } catch (e) { /* fall through so the star still visibly sticks, even offline */ } + } + let b = cache.find((x) => cellId(x.lat, x.lon) === id); + if (b) { if (label) b.label = label; } + else { + b = { id, lat, lon, label: label || null, created_at: Date.now() / 1000 }; + cache = [b, ...cache].slice(0, MAX_LOCAL); + } + writeLocal(cache); + notify(); + return b; +} + +export async function remove(id) { + if (signedIn) { + try { await apiFetch(uv(`bookmarks/${encodeURIComponent(id)}`), { method: "DELETE" }); } catch (e) {} + } + cache = cache.filter((b) => b.id !== id); + writeLocal(cache); + notify(); +} + +export async function rename(id, label) { + if (signedIn) { + try { await apiFetch(uv(`bookmarks/${encodeURIComponent(id)}`), { method: "PATCH", json: { label } }); } + catch (e) { /* still apply locally so the edit shows up */ } + } + const b = cache.find((x) => x.id === id); + if (b) { b.label = label; writeLocal(cache); notify(); } +} + +async function runImport(items) { + if (!items.length) return { imported: 0, skipped: 0 }; + const res = await apiFetch(uv("bookmarks/import"), { method: "POST", json: { items } }); + if (!res.ok) throw new Error(`Import failed (${res.status}).`); + const data = await res.json(); + if (Array.isArray(data.bookmarks)) { cache = data.bookmarks.map(fromServerRow); writeLocal(cache); } + pending = []; + notify(); + return data; +} + +/** Import the local-only bookmarks the login prompt flagged. */ +export function importPending() { + return runImport(pending.map((b) => ({ lat: b.lat, lon: b.lon, label: b.label }))); +} + +/** Import everything currently in localStorage — the explicit "Import from + * this device" button on the alerts page, reachable even after the login + * prompt was dismissed. */ +export function importAllLocal() { + return runImport(readLocal().map((b) => ({ lat: b.lat, lon: b.lon, label: b.label }))); +} + +// --- boot + auth transitions ------------------------------------------------- +(async function boot() { + const local = readLocal(); + cache = local; + notify(); // render local state immediately, don't wait on the network + await syncFromServer(local); +})(); + +onAuthChange(async (user) => { + if (user) { + await syncFromServer(readLocal()); + } else { + // Logged out: fall back to local bookmarks without wiping them. + signedIn = false; + pending = []; + cache = readLocal(); + notify(); + } +}); -- 2.45.2 From d6ead569714692b0419398a772b4aa013d52ba2f Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:36:34 +0000 Subject: [PATCH 19/37] bookmarks: map + account UI for saved locations --- frontend/static/bookmarks-ui.js | 252 ++++++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 frontend/static/bookmarks-ui.js diff --git a/frontend/static/bookmarks-ui.js b/frontend/static/bookmarks-ui.js new file mode 100644 index 0000000..82877ed --- /dev/null +++ b/frontend/static/bookmarks-ui.js @@ -0,0 +1,252 @@ +// Map-page bookmark UI: a star toggle in the results "share" row, and a +// compact "Saved" pill + dropdown near the Find button for jumping between +// saved spots without retyping. Pure DOM glue over bookmarks.js, which is the +// only place bookmark data actually lives — this module holds no state of its +// own beyond references to the elements it built. +// +// Loaded as an extra module alongside app.js rather than merged into it: app.js +// owns #results' markup (it rewrites results.innerHTML wholesale on every grade +// via its own render()), so a MutationObserver on #results is what lets this +// module re-attach the star after every re-render without needing a hook +// app.js doesn't expose. selectLocation() is module-private to app.js, so +// jumping to a saved spot goes through the same hash the app already treats as +// authoritative on load (see nav.js: "a hash always wins") via a reload — +// the same path opening a shared link takes, so the normal grade flow runs. +import * as bookmarks from "./bookmarks.js"; + +const STAR_OUTLINE = ``; +const STAR_FILLED = ``; +const CHEV_IC = ``; +const CLOSE_X = ``; +const PENCIL_IC = ``; + +function esc(s) { + return String(s == null ? "" : s).replace(/[&<>"']/g, (c) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); +} + +function injectStyle() { + if (document.getElementById("tg-bookmarks-style")) return; + const style = document.createElement("style"); + style.id = "tg-bookmarks-style"; + style.textContent = ` +.bkm-star[aria-pressed="true"] { color: var(--accent); border-color: var(--accent); } +.bkm-pill-wrap { position: relative; display: inline-flex; align-self: center; } +.bkm-pill { + border-radius: 10px; border: 1px solid var(--border); cursor: pointer; + background: var(--surface-2); color: var(--text); font-weight: 600; font-size: 13px; + padding: 9px 12px; min-height: 40px; display: inline-flex; align-items: center; gap: 6px; +} +.bkm-pill:hover { border-color: var(--accent); } +.bkm-pill[hidden] { display: none; } +.bkm-drop { + position: absolute; left: 0; top: calc(100% + 6px); z-index: 1100; min-width: 260px; + max-width: min(340px, 90vw); max-height: 60vh; overflow-y: auto; + display: flex; flex-direction: column; padding: 6px; margin: 0; list-style: none; + background: var(--surface); border: 1px solid var(--border); border-radius: 12px; + box-shadow: 0 16px 40px rgba(0, 0, 0, .45); +} +.bkm-drop[hidden] { display: none; } +.bkm-item { display: flex; align-items: center; gap: 2px; border-radius: 8px; } +.bkm-item:hover { background: var(--surface-2); } +.bkm-item-go { + flex: 1 1 auto; min-width: 0; text-align: left; background: none; border: 0; cursor: pointer; + color: var(--text); font: inherit; font-size: 13.5px; padding: 9px 8px; + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; +} +.bkm-item-btn { + flex-shrink: 0; background: none; border: 0; color: var(--muted); cursor: pointer; + padding: 7px; border-radius: 6px; display: inline-flex; +} +.bkm-item-btn:hover { color: var(--text); background: var(--border); } +.bkm-empty { padding: 10px 8px; font-size: 12.5px; color: var(--muted); } +.bkm-import-banner { + display: flex; align-items: center; justify-content: space-between; gap: 10px; flex-wrap: wrap; + background: var(--surface-2); border: 1px solid var(--border); border-left: 3px solid var(--accent); + border-radius: 12px; padding: 10px 14px; margin: 0 0 16px; font-size: 13.5px; +} +.bkm-import-banner .bkm-import-actions { display: flex; gap: 8px; flex-shrink: 0; } +.bkm-import-banner button { + border-radius: 8px; border: 1px solid var(--border); cursor: pointer; + padding: 7px 12px; font-size: 12.5px; font-weight: 600; background: var(--surface); color: var(--text); +} +.bkm-import-banner button.bkm-import-yes { background: var(--accent); color: #1a1206; border-color: var(--accent); } +@media (max-width: 480px) { .bkm-drop { left: auto; right: 0; } } +`; + document.head.appendChild(style); +} + +function parseHash() { + const p = new URLSearchParams(location.hash.slice(1)); + const lat = parseFloat(p.get("lat")), lon = parseFloat(p.get("lon")); + if (isNaN(lat) || isNaN(lon)) return null; + return { lat, lon }; +} + +function jumpTo(lat, lon) { + const dateInput = document.getElementById("date-input"); + const date = dateInput && dateInput.value ? dateInput.value : ""; + let h = `#lat=${lat.toFixed(5)}&lon=${lon.toFixed(5)}`; + if (date) h += `&date=${date}`; + location.hash = h; + location.reload(); +} + +function currentLabel() { + const h2 = document.querySelector("#results .loc-title h2"); + return (h2 && h2.textContent.trim()) || null; +} + +// ---- star toggle in the .share row ---- +function buildStar() { + const btn = document.createElement("button"); + btn.type = "button"; + btn.id = "btn-bookmark"; + btn.className = "bkm-star"; + btn.addEventListener("click", async () => { + const loc = parseHash(); + if (!loc) return; + const existing = bookmarks.find(loc.lat, loc.lon); + btn.disabled = true; + try { + if (existing) await bookmarks.remove(existing.id); + else await bookmarks.add(loc.lat, loc.lon, currentLabel()); + } finally { + btn.disabled = false; + } + }); + return btn; +} + +function paintStar() { + const share = document.querySelector("#results .share"); + if (!share) return; + let btn = share.querySelector("#btn-bookmark"); + if (!btn) { + btn = buildStar(); + share.insertBefore(btn, share.firstChild); + } + const loc = parseHash(); + const saved = loc ? !!bookmarks.find(loc.lat, loc.lon) : false; + btn.setAttribute("aria-pressed", saved ? "true" : "false"); + btn.title = saved ? "Remove this saved location" : "Save this location"; + btn.setAttribute("aria-label", btn.title); + btn.innerHTML = (saved ? STAR_FILLED : STAR_OUTLINE) + `${saved ? "Saved" : "Save"}`; +} + +const resultsEl = document.getElementById("results"); +if (resultsEl) { + new MutationObserver(() => paintStar()).observe(resultsEl, { childList: true }); +} + +// ---- "Saved" dropdown near the Find button ---- +let dropEl = null, pillEl = null; + +function buildSavedControl() { + const findBar = document.querySelector(".find-bar"); + if (!findBar) return; + const wrap = document.createElement("div"); + wrap.className = "bkm-pill-wrap"; + wrap.innerHTML = ` + + `; + findBar.appendChild(wrap); + pillEl = wrap.querySelector("#bkm-pill"); + dropEl = wrap.querySelector("#bkm-drop"); + + pillEl.addEventListener("click", () => { + const open = dropEl.hidden; + dropEl.hidden = !open; + pillEl.setAttribute("aria-expanded", open ? "true" : "false"); + }); + document.addEventListener("click", (e) => { + if (!wrap.contains(e.target) && !dropEl.hidden) { + dropEl.hidden = true; + pillEl.setAttribute("aria-expanded", "false"); + } + }); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && !dropEl.hidden) { + dropEl.hidden = true; + pillEl.setAttribute("aria-expanded", "false"); + pillEl.focus(); + } + }); +} + +function renderDropdown(list) { + if (!pillEl || !dropEl) return; + pillEl.hidden = list.length === 0; // new visitors see nothing extra at all + if (!list.length) { dropEl.hidden = true; return; } + dropEl.innerHTML = list.map((b) => { + const label = b.label || `${b.lat.toFixed(3)}, ${b.lon.toFixed(3)}`; + return ` +
  • + + + +
    • `; + }).join(""); + dropEl.querySelectorAll(".bkm-item-go").forEach((el) => { + el.addEventListener("click", () => { + const b = list.find((x) => x.id === el.dataset.id); + if (b) { dropEl.hidden = true; jumpTo(b.lat, b.lon); } + }); + }); + dropEl.querySelectorAll(".bkm-item-rename").forEach((el) => { + el.addEventListener("click", (e) => { + e.stopPropagation(); + const b = list.find((x) => x.id === el.dataset.id); + if (!b) return; + const next = window.prompt("Rename this saved location:", b.label || ""); + if (next && next.trim()) bookmarks.rename(b.id, next.trim()); + }); + }); + dropEl.querySelectorAll(".bkm-item-remove").forEach((el) => { + el.addEventListener("click", (e) => { + e.stopPropagation(); + const b = list.find((x) => x.id === el.dataset.id); + if (b) bookmarks.remove(b.id); + }); + }); +} + +// ---- import prompt banner (non-blocking, shown once until dismissed) ---- +let bannerEl = null; +function renderBanner() { + const n = bookmarks.pendingImportCount(); + if (!n) { if (bannerEl) bannerEl.hidden = true; return; } + const controls = document.querySelector(".controls"); + if (!bannerEl && controls) { + bannerEl = document.createElement("div"); + bannerEl.className = "bkm-import-banner"; + controls.insertAdjacentElement("afterend", bannerEl); + } + if (!bannerEl) return; + bannerEl.hidden = false; + bannerEl.innerHTML = ` + Import ${n} saved location${n === 1 ? "" : "s"} into your account? + + + + `; + bannerEl.querySelector(".bkm-import-yes").onclick = async () => { + const btn = bannerEl.querySelector(".bkm-import-yes"); + btn.disabled = true; + try { await bookmarks.importPending(); } catch (e) { btn.disabled = false; } + }; + bannerEl.querySelector(".bkm-import-no").onclick = () => bookmarks.dismissImportPrompt(); +} + +// ---- boot ---- +injectStyle(); +buildSavedControl(); +bookmarks.subscribe((list) => { + renderDropdown(list); + paintStar(); + renderBanner(); +}); +renderDropdown(bookmarks.list()); +renderBanner(); -- 2.45.2 From 17cc14b48bd37807d84873cfde5b7cc4fbda8bd4 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:36:39 +0000 Subject: [PATCH 20/37] bookmarks: map + account UI for saved locations --- frontend/static/subscriptions.js | 99 +++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/frontend/static/subscriptions.js b/frontend/static/subscriptions.js index 22dc124..24701cc 100644 --- a/frontend/static/subscriptions.js +++ b/frontend/static/subscriptions.js @@ -2,10 +2,16 @@ // them add a city (via the shared map picker), tune the percentile threshold and // watched metrics, toggle active, and remove. Auth-gated: signed-out visitors get // a sign-in prompt. All calls go through account.js's cookie-aware apiFetch. +// +// Also the closest thing to an account page, so it doubles as the Saved +// locations surface: bookmarks.js is the single source of truth for those (see +// its header comment), this just lists/renders/edits them the same way it does +// alert subscriptions, reusing .sub-list/.sub-card. import "./nav.js"; import { apiFetch, openAuth, onAuthChange, uv } from "./account.js"; import { open as openPicker } from "./mappicker.js"; import * as pushClient from "./push-client.js"; +import * as bookmarks from "./bookmarks.js"; // Metric keys a user can watch, with friendly labels. (fmax/fmin exist server-side // but are omitted from the picker to keep it focused; labelled here if present.) @@ -24,6 +30,7 @@ METRIC_LABEL.fmin = "Feels-like low"; const DEFAULT_METRICS = ["tmax", "feels", "precip"]; const TRASH_IC = ``; +const PENCIL_IC = ``; const body = document.getElementById("alerts-body"); let subs = []; @@ -33,6 +40,25 @@ function esc(s) { ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); } +// Small stylesheet for the bits of the Saved locations section that don't +// already have a home in .sub-list/.sub-card — injected once, not appended to +// the shared style.css, so this stays self-contained. +(function injectBkmStyle() { + if (document.getElementById("tg-bkm-alerts-style")) return; + const s = document.createElement("style"); + s.id = "tg-bkm-alerts-style"; + s.textContent = ` +.bkm-heading { font-size: 15px; margin: 26px 0 4px; } +.bkm-import-row { display: flex; align-items: center; justify-content: space-between; gap: 10px; flex-wrap: wrap; margin: 0 0 12px; } +.bkm-import-status { margin: -4px 0 10px; } +.bkm-card-actions { display: flex; align-items: center; gap: 2px; flex-shrink: 0; } +.bkm-rename { background: none; border: 0; color: var(--muted); cursor: pointer; padding: 7px; border-radius: 6px; display: inline-flex; } +.bkm-rename:hover { color: var(--text); background: var(--border); } +.bkm-card-loc { color: var(--muted); font-size: 12.5px; } +`; + document.head.appendChild(s); +})(); + async function readErr(res) { try { const d = await res.json(); @@ -80,17 +106,84 @@ function render() {

    Each alert sends at most one notification per week.

    -
      `; +
        +
        `; body.querySelector("#add-alert").onclick = startAdd; renderPushBar(); const list = body.querySelector("#sub-list"); if (!subs.length) { list.innerHTML = `
      • No alerts yet. Add a city to get started.
        • `; - return; + } else { + subs.forEach((s) => list.appendChild(subCard(s))); } - subs.forEach((s) => list.appendChild(subCard(s))); + renderBookmarksSection(); } +// --- Saved locations (bookmarks) --------------------------------------------- +function bkmCard(b) { + const li = document.createElement("li"); + li.className = "sub-card"; + const label = b.label || `${b.lat.toFixed(3)}, ${b.lon.toFixed(3)}`; + li.innerHTML = ` +
        +
        + ${esc(label)} + ${b.label ? `${esc(b.lat.toFixed(3))}, ${esc(b.lon.toFixed(3))}` : ""} +
        + + + + +
        `; + li.querySelector(".bkm-rename").onclick = () => { + const next = window.prompt("Rename this saved location:", b.label || ""); + if (next && next.trim()) bookmarks.rename(b.id, next.trim()); + }; + li.querySelector(".sub-remove").onclick = () => bookmarks.remove(b.id); + return li; +} + +function renderBookmarksSection() { + const el = document.getElementById("bkm-alerts-section"); + if (!el) return; + const list = bookmarks.list(); + el.innerHTML = ` +

        Saved locations

        +
        +

        Places you've starred on the map, kept in sync with your account.

        + +
        + +
          `; + const ul = el.querySelector("#bkm-list"); + if (!list.length) { + ul.innerHTML = `
        • No saved locations yet. Star a spot on the map to see it here.
          • `; + } else { + list.forEach((b) => ul.appendChild(bkmCard(b))); + } + el.querySelector("#bkm-import-btn").onclick = async () => { + const btn = el.querySelector("#bkm-import-btn"); + const status = el.querySelector("#bkm-import-status"); + btn.disabled = true; + try { + const res = await bookmarks.importAllLocal(); + status.hidden = false; + status.textContent = res.imported + ? `Imported ${res.imported} location${res.imported === 1 ? "" : "s"}${res.skipped ? ` (${res.skipped} already saved)` : ""}.` + : "Nothing new to import from this device."; + } catch (e) { + status.hidden = false; + status.textContent = e.message || "Couldn't import right now."; + } finally { + btn.disabled = false; + } + }; +} + +// Bookmarks can change from outside this render cycle (rename/remove above, +// or an import banner) — keep the section live rather than only painting once. +bookmarks.subscribe(() => renderBookmarksSection()); + // --- push notifications toggle (this device) --------------------------------- // Delivery over OS push is per-device: a subscription lives in each browser, so // this control reflects/toggles *this* device, separate from the account-wide -- 2.45.2 From 1c6ecc2a6f7f105a1a2e176bd7b619f792023e49 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:36:43 +0000 Subject: [PATCH 21/37] bookmarks: map + account UI for saved locations --- frontend/templates/home.html.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/templates/home.html.j2 b/frontend/templates/home.html.j2 index 0551ca5..387c76d 100644 --- a/frontend/templates/home.html.j2 +++ b/frontend/templates/home.html.j2 @@ -198,6 +198,10 @@ {% block body_scripts %} + {# Bookmarked locations: a star toggle in the results share row + a "Saved" + dropdown by the Find button. Loaded as its own module alongside app.js + (not merged into it) — see bookmarks-ui.js's header comment for why. #} + {# The records strip renders real temperatures server-side as .temp spans, so it needs the same converter the SEO pages use or they'd stay in °F while the toggle says °C. Both import units.js, which the module loader dedupes. #} -- 2.45.2 From b57e280ad6989585fb538f5c289c1b35745264af Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:49:42 +0000 Subject: [PATCH 22/37] frontend: load bookmarks-ui.js from the Go SSR home template too (#120) --- frontend/server/internal/render/templates/home.html.tmpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/server/internal/render/templates/home.html.tmpl b/frontend/server/internal/render/templates/home.html.tmpl index 2ce3918..7a03313 100644 --- a/frontend/server/internal/render/templates/home.html.tmpl +++ b/frontend/server/internal/render/templates/home.html.tmpl @@ -175,7 +175,10 @@

          Browse all cities →

          {{template "base_body_end" .}} - {{/* The records strip renders real temperatures server-side as .temp spans, so + {{/* Bookmarked locations: a star toggle in the results share row + a "Saved" + dropdown by the Find button. Loaded as its own module alongside app.js + (not merged into it) — see bookmarks-ui.js's header comment for why. */}} + {{/* The records strip renders real temperatures server-side as .temp spans, so it needs the same converter the SEO pages use or they'd stay in °F while the toggle says °C. Both import units.js, which the module loader dedupes. */}} -- 2.45.2 From 3ec72ca75b3de9652473c81de6b96e11554bcd5e Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:51:02 +0000 Subject: [PATCH 23/37] bookmarks: don't fabricate a local save when the server rejects it (#121) --- frontend/static/bookmarks-ui.js | 4 ++++ frontend/static/bookmarks.js | 23 ++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/frontend/static/bookmarks-ui.js b/frontend/static/bookmarks-ui.js index 82877ed..2f3b6a7 100644 --- a/frontend/static/bookmarks-ui.js +++ b/frontend/static/bookmarks-ui.js @@ -111,6 +111,10 @@ function buildStar() { try { if (existing) await bookmarks.remove(existing.id); else await bookmarks.add(loc.lat, loc.lon, currentLabel()); + } catch (e) { + // A real server rejection (e.g. the per-user cap) — bookmarks.add() no + // longer fakes a local save for this, so say why nothing changed. + alert(e.message || "Couldn't save this location."); } finally { btn.disabled = false; } diff --git a/frontend/static/bookmarks.js b/frontend/static/bookmarks.js index 361adfa..fd64a25 100644 --- a/frontend/static/bookmarks.js +++ b/frontend/static/bookmarks.js @@ -97,17 +97,34 @@ async function syncFromServer(preSyncLocal) { notify(); } -/** Save (or re-save with a new label) the given spot. Upserts either way. */ +/** Save (or re-save with a new label) the given spot. Upserts either way. + * + * Throws if signed in and the server explicitly rejects the request (e.g. the + * 409 per-user cap, a 422 on bad input) — that is a real failure, not the + * "offline" case below, and must not be papered over with a local-only + * bookmark that doesn't actually exist server-side and would just vanish, + * unexplained, on the next sync. A genuine network failure (no response at + * all) still falls through to the local-only save so the star visibly + * sticks even offline. */ export async function add(lat, lon, label) { const id = cellId(lat, lon); if (signedIn) { + let res = null; try { - const res = await apiFetch(uv("bookmarks"), { method: "POST", json: { lat, lon, label: label || undefined } }); + res = await apiFetch(uv("bookmarks"), { method: "POST", json: { lat, lon, label: label || undefined } }); + } catch (e) { + // offline / network failure: fall through to the local-only path below. + } + if (res) { if (res.ok) { const rows = await fetchServerList(); if (rows != null) { cache = rows.map(fromServerRow); writeLocal(cache); notify(); return find(lat, lon); } + } else { + let msg = `Couldn't save this location (${res.status}).`; + try { const d = await res.json(); if (typeof d.detail === "string") msg = d.detail; } catch (e) { /* no body */ } + throw new Error(msg); } - } catch (e) { /* fall through so the star still visibly sticks, even offline */ } + } } let b = cache.find((x) => cellId(x.lat, x.lon) === id); if (b) { if (label) b.label = label; } -- 2.45.2 From 2a4f51679817b7a86b87f2624c868856300662f4 Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:49:42 +0000 Subject: [PATCH 24/37] frontend: load bookmarks-ui.js from the Go SSR home template too (#120) --- frontend/server/internal/render/templates/home.html.tmpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/server/internal/render/templates/home.html.tmpl b/frontend/server/internal/render/templates/home.html.tmpl index 2ce3918..7a03313 100644 --- a/frontend/server/internal/render/templates/home.html.tmpl +++ b/frontend/server/internal/render/templates/home.html.tmpl @@ -175,7 +175,10 @@

          Browse all cities →

          {{template "base_body_end" .}} - {{/* The records strip renders real temperatures server-side as .temp spans, so + {{/* Bookmarked locations: a star toggle in the results share row + a "Saved" + dropdown by the Find button. Loaded as its own module alongside app.js + (not merged into it) — see bookmarks-ui.js's header comment for why. */}} + {{/* The records strip renders real temperatures server-side as .temp spans, so it needs the same converter the SEO pages use or they'd stay in °F while the toggle says °C. Both import units.js, which the module loader dedupes. */}} -- 2.45.2 From beb5fa5259b08d61078940d7f0d2b573781b7c2a Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 18:51:02 +0000 Subject: [PATCH 25/37] bookmarks: don't fabricate a local save when the server rejects it (#121) --- frontend/static/bookmarks-ui.js | 4 ++++ frontend/static/bookmarks.js | 23 ++++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/frontend/static/bookmarks-ui.js b/frontend/static/bookmarks-ui.js index 82877ed..2f3b6a7 100644 --- a/frontend/static/bookmarks-ui.js +++ b/frontend/static/bookmarks-ui.js @@ -111,6 +111,10 @@ function buildStar() { try { if (existing) await bookmarks.remove(existing.id); else await bookmarks.add(loc.lat, loc.lon, currentLabel()); + } catch (e) { + // A real server rejection (e.g. the per-user cap) — bookmarks.add() no + // longer fakes a local save for this, so say why nothing changed. + alert(e.message || "Couldn't save this location."); } finally { btn.disabled = false; } diff --git a/frontend/static/bookmarks.js b/frontend/static/bookmarks.js index 361adfa..fd64a25 100644 --- a/frontend/static/bookmarks.js +++ b/frontend/static/bookmarks.js @@ -97,17 +97,34 @@ async function syncFromServer(preSyncLocal) { notify(); } -/** Save (or re-save with a new label) the given spot. Upserts either way. */ +/** Save (or re-save with a new label) the given spot. Upserts either way. + * + * Throws if signed in and the server explicitly rejects the request (e.g. the + * 409 per-user cap, a 422 on bad input) — that is a real failure, not the + * "offline" case below, and must not be papered over with a local-only + * bookmark that doesn't actually exist server-side and would just vanish, + * unexplained, on the next sync. A genuine network failure (no response at + * all) still falls through to the local-only save so the star visibly + * sticks even offline. */ export async function add(lat, lon, label) { const id = cellId(lat, lon); if (signedIn) { + let res = null; try { - const res = await apiFetch(uv("bookmarks"), { method: "POST", json: { lat, lon, label: label || undefined } }); + res = await apiFetch(uv("bookmarks"), { method: "POST", json: { lat, lon, label: label || undefined } }); + } catch (e) { + // offline / network failure: fall through to the local-only path below. + } + if (res) { if (res.ok) { const rows = await fetchServerList(); if (rows != null) { cache = rows.map(fromServerRow); writeLocal(cache); notify(); return find(lat, lon); } + } else { + let msg = `Couldn't save this location (${res.status}).`; + try { const d = await res.json(); if (typeof d.detail === "string") msg = d.detail; } catch (e) { /* no body */ } + throw new Error(msg); } - } catch (e) { /* fall through so the star still visibly sticks, even offline */ } + } } let b = cache.find((x) => cellId(x.lat, x.lon) === id); if (b) { if (label) b.label = label; } -- 2.45.2 From 0617a3b067601d37e2922b1073bbb586612200ec Mon Sep 17 00:00:00 2001 From: emi Date: Sun, 26 Jul 2026 19:48:50 +0000 Subject: [PATCH 26/37] db: raise max_connections to 200, cap work_mem at 64MB, trim async pool overflow (#124) --- backend/accounts/db.py | 23 +++++++--- backend/tests/accounts/test_db_engines.py | 54 +++++++++++++++++++++-- infra/deploy/db/init/20-tuning.sh | 35 ++++++++++++--- 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/backend/accounts/db.py b/backend/accounts/db.py index 8b97ff8..b22d9e5 100644 --- a/backend/accounts/db.py +++ b/backend/accounts/db.py @@ -62,11 +62,21 @@ def _apply_sqlite_pragmas(dbapi_conn, _rec): # # pool_size=1/max_overflow=1 (the original setting on both) meant a 3rd # concurrent request per worker had to wait out pool_timeout (~30s default) for a -# slot — easy to hit with 4 uvicorn workers x real traffic. 5+5 gives each async -# engine headroom for a burst of concurrent account requests without coming close -# to Postgres's own connection ceiling (two async engines here plus the sync -# engine below, times N workers, all share that budget). -_ASYNC_POOL_KWARGS = dict(pool_size=5, max_overflow=5, pool_pre_ping=True, +# slot — easy to hit with 4 uvicorn workers x real traffic. pool_size=5 keeps that +# headroom for the steady state. +# +# max_overflow is deliberately *tighter* than pool_size (2, not 5), because the +# overflow is the part nobody budgets for and it is what saturated Postgres on +# 2026-07-26. The connection ceiling is shared far more widely than this process +# knows: two async engines here plus the sync engine below, times N uvicorn +# workers, times the replica count (web autoscales), and prod and beta are two +# databases on ONE Postgres instance. At 5+5 the configured worst case across all +# of that exceeded the server's max_connections; at 5+2 prod web's ceiling is +# 4x14 rather than 4x20. Steady-state behaviour is unchanged — only the burst +# ceiling narrows, and a burst that would have opened a 6th..10th connection now +# waits on pool_timeout instead of being refused outright by the server (which is +# an error, not a wait). See infra/deploy/db/init/20-tuning.sh for the server side. +_ASYNC_POOL_KWARGS = dict(pool_size=5, max_overflow=2, pool_pre_ping=True, pool_recycle=1800, future=True) # Smaller than the async engines: only the notifier leader (one process # cluster-wide, see core/singleton.py) uses the sync engine, driving its periodic @@ -74,7 +84,8 @@ _ASYNC_POOL_KWARGS = dict(pool_size=5, max_overflow=5, pool_pre_ping=True, # statement_timeout (via psycopg's `options`) bound how long a wedged Postgres # can hang the notifier — it runs for the lifetime of that process's leader # flock, so an indefinite hang here would silently stop every subscription -# notification until the process is restarted. +# notification until the process is restarted. Left at 2+3: one process holds it, +# so it is not multiplied by the worker or replica count. _SYNC_POOL_KWARGS = dict(pool_size=2, max_overflow=3, pool_pre_ping=True, pool_recycle=1800, future=True, connect_args={"connect_timeout": 5, diff --git a/backend/tests/accounts/test_db_engines.py b/backend/tests/accounts/test_db_engines.py index 29454d5..cbe6e24 100644 --- a/backend/tests/accounts/test_db_engines.py +++ b/backend/tests/accounts/test_db_engines.py @@ -9,6 +9,28 @@ from sqlalchemy import select from accounts import db +# The server-side ceiling these pools spend from, and the deployment shape that +# spends it. Kept here so a pool change that stops fitting is a failing test +# rather than a 2am "sorry, too many clients already". Mirrors max_connections in +# infra/deploy/db/init/20-tuning.sh. ONE Postgres instance serves prod and beta. +MAX_CONNECTIONS = 200 +PROD_WEB_WORKERS = 4 # uvicorn workers per prod web replica +BETA_WEB_WORKERS = 2 +ASYNC_ENGINES_PER_PROCESS = 2 # RW + RO, both built from _ASYNC_POOL_KWARGS + + +def _async_ceiling_per_process() -> int: + """Connections one backend process can hold from its async pools at once.""" + k = db._ASYNC_POOL_KWARGS + return ASYNC_ENGINES_PER_PROCESS * (k["pool_size"] + k["max_overflow"]) + + +def _sync_ceiling() -> int: + """The notifier's sync pool. One process holds it cluster-wide (leader flock), + so it is NOT multiplied by workers or replicas.""" + k = db._SYNC_POOL_KWARGS + return k["pool_size"] + k["max_overflow"] + def test_defaults_to_sqlite_when_no_database_url(): assert db.IS_POSTGRES is False @@ -32,13 +54,18 @@ def test_postgres_pool_sizing_and_sync_timeouts(): """The Postgres engine-construction kwargs, defined at module scope so they're testable without a real Postgres connection (the engines built from them are only actually constructed when IS_POSTGRES, exercised by the docker-compose - stack -- see the module docstring). pool_size=1/max_overflow=1 (the old + stack -- see the module docstring). pool_size=1/max_overflow=1 (the original setting) let a 3rd concurrent request per worker wait out pool_timeout for a - slot; both async engines need real headroom, and the sync engine (the + slot, so the steady-state pool needs real headroom; the sync engine (the notifier's, driven off-event-loop) must bound connect + statement time so a wedged Postgres can't hang it forever under its leader flock.""" assert db._ASYNC_POOL_KWARGS["pool_size"] >= 5 - assert db._ASYNC_POOL_KWARGS["max_overflow"] >= 5 + + # The overflow is the part nobody budgets for -- it is multiplied by two async + # engines x uvicorn workers x replicas, against a ceiling shared with beta. + # Prod saturated max_connections on 2026-07-26 with this at 5. Keep it well + # under pool_size; the budget test below is the real constraint. + assert db._ASYNC_POOL_KWARGS["max_overflow"] <= 2 assert db._SYNC_POOL_KWARGS["pool_size"] < db._ASYNC_POOL_KWARGS["pool_size"] connect_args = db._SYNC_POOL_KWARGS["connect_args"] @@ -53,6 +80,27 @@ def test_postgres_pool_sizing_and_sync_timeouts(): engine.dispose() +def test_pool_budget_fits_under_max_connections(): + """At today's replica counts the whole estate's configured worst case must fit + inside the server's max_connections, with margin -- prod web + prod worker + + beta web + beta worker all draw on ONE Postgres instance. This is the test + that would have failed before 2026-07-26, when the configured worst case was + 170 against a ceiling of 100. + + Note this counts *configured maxima*: every pool at full overflow at the same + instant. Prod web autoscaling to its max of 3 replicas is deliberately NOT in + this sum -- that peak still does not fit, and closing it needs a connection + pooler or a lower autoscale ceiling rather than a bigger number here.""" + per_process = _async_ceiling_per_process() + prod = per_process * PROD_WEB_WORKERS + (per_process + _sync_ceiling()) + beta = per_process * BETA_WEB_WORKERS + (per_process + _sync_ceiling()) + + assert prod + beta <= MAX_CONNECTIONS * 0.8, ( + f"configured worst case {prod + beta} leaves too little room under " + f"max_connections={MAX_CONNECTIONS}" + ) + + def test_read_session_yields_a_working_session(): # On SQLite the read session is fully usable (no read-only enforcement); this # just confirms the dependency wiring resolves to a live AsyncSession. diff --git a/infra/deploy/db/init/20-tuning.sh b/infra/deploy/db/init/20-tuning.sh index 4791d10..78f8161 100755 --- a/infra/deploy/db/init/20-tuning.sh +++ b/infra/deploy/db/init/20-tuning.sh @@ -8,10 +8,10 @@ # 10-timescaledb.sql enables timescaledb. Settings are written via ALTER SYSTEM # (persisted to postgresql.auto.conf, which the timescaledb image's own # timescaledb-tune postgresql.conf defers to); the container's post-init restart brings -# restart-only settings (shared_buffers, …) into effect. NB: never ALTER SYSTEM SET -# shared_preload_libraries here — that would land in auto.conf and override the image's -# `timescaledb` preload. Init scripts do NOT re-run on an existing volume — to -# re-tune later, set DB_MEMORY and run this by hand, then restart: +# restart-only settings (shared_buffers, max_connections, …) into effect. NB: never +# ALTER SYSTEM SET shared_preload_libraries here — that would land in auto.conf and +# override the image's `timescaledb` preload. Init scripts do NOT re-run on an existing +# volume — to re-tune later, set DB_MEMORY and run this by hand, then restart: # docker compose exec -e DB_MEMORY=16g db bash /docker-entrypoint-initdb.d/20-tuning.sh # docker compose restart db set -euo pipefail @@ -33,13 +33,30 @@ if [ "$mb" -lt 1024 ]; then mb=8192; fi # maintenance_work_mem 512 MB) and scale linearly on a bigger box. shared_buffers=$((mb / 4)) # 25% — the shared page cache effective_cache=$((mb * 3 / 4)) # 75% — planner's view of total cache (PG + OS) -work_mem=$((mb / 128)) # ~64 MB at 8 GB (per-operation; kept modest) +maint_mem=$((mb / 16)) # 512 MB at 8 GB — index builds / VACUUM + +# work_mem is per *sort operation*, not per connection, so its real cost is +# work_mem x concurrent sorts x max_connections (200, below) — and the db container +# has a hard memory limit. Left uncapped the ratio gives 128 MB at prod's 16g +# budget, which is more than a 200-connection instance should carry: a burst of +# heavy sorts can walk into the container limit, and an OOM-killed Postgres is a +# far worse day than a refused connection. Cap at 64 MB (dev's 8g is unaffected). +work_mem=$((mb / 128)) +if [ "$work_mem" -gt 64 ]; then work_mem=64; fi if [ "$work_mem" -lt 16 ]; then work_mem=16; fi -maint_mem=$((mb / 16)) # 512 MB at 8 GB — index builds / VACUUM + +# Connection ceiling. NOT derived from DB_MEMORY: it is bounded by how many pools +# the app opens, not by RAM. One instance serves prod and beta, and every backend +# process opens three pools (two async engines + the notifier's sync engine, see +# backend/accounts/db.py), so the image default of 100 was below the configured +# worst case — prod saturated it on 2026-07-26 and started refusing clients with +# "sorry, too many clients already". 200 covers today's replica counts including +# web's autoscale maximum. ~+1 GB of per-connection overhead at 200. +max_conn=200 echo "[tuning] DB_MEMORY=${budget} -> ${mb}MB: shared_buffers=${shared_buffers}MB" \ "effective_cache_size=${effective_cache}MB work_mem=${work_mem}MB" \ - "maintenance_work_mem=${maint_mem}MB" + "maintenance_work_mem=${maint_mem}MB max_connections=${max_conn}" psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < Date: Sun, 26 Jul 2026 12:53:33 -0700 Subject: [PATCH 27/37] docs: the orchestrator runbook and the module map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .claude/BRANCHING.md — how the trunk-based flow is actually run: the serialised merge queue (Forgejo has no native one, so whoever orchestrates is it), the two promotions and who decides each, and the hotfix down-merge that has to happen in the same session or it becomes a regression scheduled for the next promotion. Two things it states because getting them wrong is expensive here. CI does not gate PRs into `release` — pr-build.yml fires only for dev and main — so a required check on that branch would make every production promotion unmergeable with a 405 naming no cause; closing that gap means changing the workflow first and turning the requirement on second. And the chain cannot fast-forward: every promotion leaves a merge commit on the target the source never receives, so promotions are merge commits until someone deliberately reconciles the branches. .claude/ownership.md — which tasks may run concurrently, seamed along the four domains, plus the shared spine that has to be serialised: the workflows (CI lives only in the root .forgejo/), env-topology.sh, the deploy entry points and the CLAUDE.md files. Names the recurring collisions too — a schema change spans alembic and whatever reads the column, and if the payload shape moves it is PAYLOAD_VER and the frontend as well, which is one task and not two. --- .claude/BRANCHING.md | 190 +++++++++++++++++++++++++++++++++++++++++++ .claude/ownership.md | 60 ++++++++++++++ CLAUDE.md | 5 ++ 3 files changed, 255 insertions(+) create mode 100644 .claude/BRANCHING.md create mode 100644 .claude/ownership.md diff --git a/.claude/BRANCHING.md b/.claude/BRANCHING.md new file mode 100644 index 0000000..f4b7366 --- /dev/null +++ b/.claude/BRANCHING.md @@ -0,0 +1,190 @@ +# Orchestrator instructions — trunk-based flow + +`feat/*` → `dev` → `main` → `release`. One PR per hop, merges serialised by the +orchestrator. Forgejo has no native merge queue, so you are the merge queue. + +Use the Centralis `forge_*` tools for every remote operation — PRs, protection, +merges, tags. Use local git only for rebases you have to resolve by hand. +Never call the Forgejo API with curl and a pasted token; the server holds it. + +The same file exists in `centralis` with the same rules. Where the two repos +differ it is noted inline. + +--- + +## The branches + +Each branch has its own environment here, and `infra/deploy/env-topology.sh` is +the single source of truth for where each one lives: + +| Branch | Deploys to | Receives | +|---|---|---| +| `feat/*`, `fix/*` | nothing | your work | +| `dev` | dev — vps1, mesh-only, own Postgres | feature PRs | +| `main` | beta — beta.thermograph.org, vps2 | promotions from `dev` | +| `release` | prod — thermograph.org, vps2 | promotions from `main`, and hotfixes | + +`deploy.yml` handles all three: the branch selects the environment, a matrix +covers backend and frontend, and each leg checks whether the push actually +touched its domain before rolling. So **FE and BE ship out of lockstep** — a +backend change lands without a frontend deploy. That is the point of the split, +and it is why `/api/version` and `PAYLOAD_VER` discipline are load-bearing: +whatever you promote, the other half may already be running something older. + +`dev` is the default branch, so PRs target it without anyone remembering to +retarget. All three are protected: **everything is a PR**, for humans and agents +alike. + +## Who may merge what + +This is not the same question as how a change travels, and it is deliberately +asymmetric. `MERGE_AUTHORITY` in centralis' +`src/lib/promotion.ts` is the whole policy; `forge_pr_merge` enforces it. + +- **Into `dev` — free.** Merge feature PRs yourself, and do not leave them open. + A PR opened against `dev` and abandoned is an unfinished task wearing the + costume of a delivered one. +- **`dev` → `main` — yours, batched.** Beta is the test environment, so + withholding this merge withholds the testing. The judgement being asked of you + is *when a batch is coherent enough to be worth testing*, not whether you are + allowed to test it. +- **`main` → `release` — the owner's.** That merge deploys to prod. Call + `promote(from="main", to="release")`, which prepares the PR and returns what + would ship, the CI evidence and a recommendation. Then put it to the owner — + including "I would wait", if that is what you think. No flag overrides this, + and routing around it is not an option. + +`confirm_protected_base` is unrelated to the above: it is the hotfix path for a +*non-promotion* PR aimed straight at `main` or `release`, and it is audited +loudly. + +--- + +## Phase 1 — repo setup + +Idempotent. Re-run it whenever you are unsure; every step reports "already +correct" rather than erroring. + +1. **The three branches exist.** + `forge_branches(create={branch:"main", from:"dev"})` and the same for + `release`. If a branch exists but points elsewhere, the tool says so — that + is a divergence to reconcile deliberately, not to overwrite. +2. **`dev` is the default branch.** + `forge_repo_settings(apply_flow_defaults=true)` — sets the default branch and + permits every merge style the chain uses, including `fast-forward-only`, plus + rebase-updates (without which the merge queue's core primitive 403s). +3. **Protection.** `forge_branch_protection(branch="dev", apply=true)`, then + `main`, then `release`. + + `dev` and `main` get the required check; **`release` does not**, and that is + deliberate. `pr-build.yml` fires only for PRs into `dev` and `main`, so a PR + into `release` reports no status at all — requiring a context nothing + produces would make every production promotion permanently unmergeable, with + a 405 that names no cause. This estate has already lost an afternoon to + exactly that, from a single mistyped check name. If you want it closed + properly, do it in this order: add `release` to `pr-build.yml`'s + `on.pull_request.branches`, confirm on a real PR that the context appears, + *then* pass `require_checks_on_release: true`. The other order blocks prod. +4. **Stale branches.** `forge_branches()` lists every branch with its age, how + much unmerged work it carries, and a verdict: + - `merged` — nothing on it the trunk lacks. Delete freely. + - `stale` — unmerged work, untouched 14+ days. **File a `stale-branch: ` + issue summarising its diff first**, then delete. Never discard work + silently. + - `ageing` — 2+ days with work still on it. Report it; do not delete. +5. **Module map.** `.claude/ownership.md` — read it before partitioning work. + +## Phase 2 — the ongoing loop + +### Dispatching subagents + +Partition by module using `.claude/ownership.md`. Never give two concurrent +agents tasks touching the same files; if that is unavoidable, sequence them. + +Inject this into every subagent task: + +``` +Branch off latest `dev` as feat/. Touch only these files/modules: . +One logical change, small diff. Rebase onto dev before opening your PR. +PR targets `dev`. The description must list: what changed, how it was tested, +files touched. Do not merge your own PR — the orchestrator handles all merges. +``` + +Keep a branch alive **less than a day**. A task bigger than that is several +stacked tasks. + +### The merge queue + +You merge **one PR into `dev` at a time**. For each: + +1. `forge_prs(base="dev")` — the queue, with each PR's head sha and triage flag. +2. `forge_pr_update(pr=N)` — replay it onto the current tip. If the head sha + does not move, it was already current and no CI will re-fire; do not sit + waiting for a run that was never scheduled. +3. `forge_pr_await(pr=N, until="checks_complete")` — checks on the **rebased** + head. The run you saw before the update was for a merge base that no longer + exists. +4. `forge_pr_merge(pr=N, method="rebase", delete_branch=true)`. +5. Only then move to the next PR. + +`block_on_outdated_branch` is set on `dev`, so Forgejo enforces step 2 rather +than trusting you to remember it: after one merge, the next PR is refused until +it has been replayed. Treat that 405 as the queue working, not as a fault. + +If a rebase conflicts: resolve it yourself if it is trivial and mechanical +(under ~20 lines), otherwise hand it back to the owning agent with the conflict +context. Do not resolve a large conflict on someone else's behalf. + +### Promotion: `dev` → `main` + +Promote when `dev` is green and a coherent batch is complete — not on a timer, +not mid-feature. `promote(from="dev", to="main")` opens the PR with the +divergence, the conflict prediction and the changelog written into its body. +Merge it with `method="merge"`. + +Unready work belongs behind a feature flag, or not on `dev` yet. **Never promote +around it with cherry-picks.** + +`promote` refuses a hop whose two tips have identical trees — the commit counts +look like real work while the content is already on the target under different +SHAs, and merging it would fire the target's deploys for nothing. + +### Promotion: `main` → `release` + +Prepare with `promote`, then ask. On the owner's yes, merge with `method="merge"` +and tag it: `forge_releases(create={tag:"v", target:"release", ...})`. + +**On fast-forward.** The written policy is that `release` moves only by +fast-forward. It cannot today: every promotion so far has been a merge commit +*into* the target, which the source never receives, so the branches are mutually +divergent by construction and Forgejo will refuse a `fast-forward-only` merge — +correctly. The style is implemented and accepted by `forge_pr_merge`; adopting +it needs a one-time reconciliation of the protected branches, which is the +owner's decision. Until that happens, promotions are merge commits. Do not +attempt to "fix" this with a rebase or a force-push. + +### Hotfix + +1. `forge_branches(create={branch:"hotfix/", from:"release"})`. +2. Fix, PR into `release`, checks green, merge with + `confirm_protected_base: true`, tag `v`. +3. **Immediately down-merge** `release` → `main` and `main` → `dev`, as merges, + not cherry-picks, in the same session. A hotfix that exists only on `release` + is a regression scheduled for the next promotion. + +Note that CI does not gate PRs into `release` — `pr-build.yml` fires only for +`dev` and `main`. A hotfix's green evidence therefore has to come from somewhere +else; say plainly where it came from rather than implying a check passed. + +A hotfix that touches only one domain deploys only that domain. Check the other +half's `/api/version` before assuming the fix is live end to end. + +## Guardrails + +- Never force-push `dev`, `main` or `release`. +- Never merge a red PR, hotfix included. Fix CI, or fix the tests in the same PR. +- Never merge two green PRs back to back without re-checking the second against + the new tip. +- If two down-merges or promotions conflict beyond the trivial, stop and report + rather than resolving autonomously. +- Weekly: `forge_branches()` and report anything `ageing` or `stale`. diff --git a/.claude/ownership.md b/.claude/ownership.md new file mode 100644 index 0000000..dcf8239 --- /dev/null +++ b/.claude/ownership.md @@ -0,0 +1,60 @@ +# Module map — what can be worked on concurrently + +Used for partitioning work across concurrent subagents. The rule is one owner +per file at a time: **never** give two concurrent agents tasks that touch the +same module. If two tasks genuinely need the same file, sequence them instead of +racing them — a rebase conflict costs more than the parallelism saved. + +Read this before dispatching. It answers one question: *can these two tasks run +at the same time?* + +The monorepo's four domains are the natural seam, and they are a real one: FE +and BE build and deploy independently, and `deploy.yml` checks per-leg whether a +push actually touched that domain. Two agents in two different domains will not +collide in CI either. + +## Safe to assign concurrently + +| Module | Files | Notes | +|---|---|---| +| Backend — API | `backend/api/**`, `backend/app.py` | Route handlers and serialisation. | +| Backend — core | `backend/core/**` | Grading, percentiles, climatology. The domain logic. | +| Backend — accounts | `backend/accounts/**` | Auth, sessions, OAuth links. | +| Backend — daemon | `backend/daemon/**` | Scheduled work. Anything named "prefetch" must not spend the Open-Meteo quota. | +| Backend — data & lake | `backend/data/**`, `backend/gen_era5_lake.py`, `backend/gen_cities.py` | | +| Backend — migrations | `backend/alembic/**`, `backend/alembic.ini` | One owner, always. Two agents generating revisions produce two heads. | +| Frontend — server | `frontend/server/**`, `frontend/app.py`, `frontend/api_client.py` | Go SSR and the Python app. | +| Frontend — static | `frontend/static/**` | | +| Frontend — templates | `frontend/templates/**`, `frontend/content/**`, `frontend/content.py` | | +| Infra — deploy | `infra/deploy/**` (excluding `secrets/`), `infra/docker-compose*.yml` | | +| Infra — secrets | `infra/deploy/secrets/**` | SOPS vault. One owner, and never alongside `infra/deploy`. | +| Infra — ops | `infra/ops/**`, `infra/lake-iceberg/**` | | +| Infra — terraform | `infra/terraform/**` | | +| Observability | `observability/**` | Grafana dashboards are provisioned from this JSON; a UI edit is overwritten. | +| Docs | `docs/**` | | +| Agent tooling | `.claude/hooks/**` | | + +## Serialise — shared spine + +Touched by most changes. A change here is its own task with nothing else running +against it. + +| File | Why | +|---|---| +| `.forgejo/workflows/**` | CI lives **only** here, path-filtered per domain. Every domain's changes route through the same files, so two agents editing workflows conflict even when their domains do not. | +| `CLAUDE.md` and every domain `CLAUDE.md` | Read before every change; a stale one is a correctness bug. Small edits, landed fast. | +| `CUTOVER-NOTES.md` | The source of truth for what is and is not live. | +| `infra/deploy/env-topology.sh` | The single source of truth for where each environment's checkout, branch, stack and ports live. | +| `infra/deploy/deploy.sh`, `infra/deploy/stack/deploy-stack.sh` | The deploy contract's entry points. | +| `backend/CLAUDE.md` + the `/api/version` contract | FE and BE ship out of lockstep, so `PAYLOAD_VER` discipline is load-bearing. A change to the payload shape is one task spanning both domains — not two concurrent ones. | + +## The recurring collisions + +- **A schema change is not a backend-only task.** It is `backend/alembic` plus + whatever reads the column, and if the payload shape moves it is `PAYLOAD_VER` + and the frontend too. Scope it as one task across both domains rather than + two agents discovering each other mid-flight. +- **Adding CI for a domain edits the shared workflows.** Sequence it against any + other workflow work. +- **`infra/deploy/secrets/` and `infra/deploy/` are one owner between them.** + The rendered artifact and the thing that renders it move together. diff --git a/CLAUDE.md b/CLAUDE.md index 3b640aa..ddf58a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,11 @@ every change, so a stale one is a correctness bug, not a documentation bug. `dev`, `main` and `release` are protected: **everything is a PR**, for humans and agents alike. Promotion is one PR per hop, `dev` → `main` → `release`. +`.claude/BRANCHING.md` is the orchestrator's runbook for that flow — the +serialised merge queue, promotions, hotfix down-merges — and +`.claude/ownership.md` is the module map saying which tasks may run +concurrently. Read both before dispatching parallel work. + **`dev` is a first-class hosted environment, not just an integration branch.** It runs on vps1 — the same box as Forgejo and the monitoring stack — with its own Postgres container, reachable only on the WireGuard mesh -- 2.45.2 From 1d4defefb9701535ed8c9249084ee67bf2b89ed5 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sun, 26 Jul 2026 15:05:26 -0700 Subject: [PATCH 28/37] BRANCHING: the escape hatch, now that apply_to_admins binds the owner too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protection an admin walks through is documentation rather than a control, so apply_to_admins goes on — but that removes the direct-push route for everyone, including in an emergency, and the way back in should be written down before it is needed rather than improvised. There is no force flag; the supported route is lift the rule, act, restore it, with both halves audited. Names the failure mode that actually justifies it: a renamed status context makes the required check unsatisfiable, and with apply_to_admins on that blocks every merge into dev and main with no exemption for anyone. --- .claude/BRANCHING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.claude/BRANCHING.md b/.claude/BRANCHING.md index f4b7366..e602fbc 100644 --- a/.claude/BRANCHING.md +++ b/.claude/BRANCHING.md @@ -179,6 +179,35 @@ else; say plainly where it came from rather than implying a check passed. A hotfix that touches only one domain deploys only that domain. Check the other half's `/api/version` before assuming the fix is live end to end. +### When you genuinely need to push directly + +`apply_to_admins` is on, which means these rules bind the owner too. That is +deliberate — protection an admin walks straight through is documentation, not a +control, and every account here is an admin — but it does remove the escape +hatch, so the way back in is worth writing down before it is needed at 3am. + +There is no force flag. The supported route is to lift the rule, do the thing, +and put it back: + +``` +forge_branch_protection(branch="dev", delete=true) # rule gone; direct push allowed +git push origin dev # the emergency change +forge_branch_protection(branch="dev", apply=true) # restored to policy +``` + +Both the removal and the restore are audited, which is the point: the hatch is +open on the record rather than by a quiet exception. Restore it in the same +session — a protection rule "temporarily" removed is how a repo ends up +unprotected for a month. + +The failure mode that actually justifies this: if the required status context +is ever renamed and no longer matches, **nothing can merge into `dev` or `main` +at all**, including by you, because `apply_to_admins` no longer exempts anyone. +`forge_pr_await` and `forge_pr_merge` both name that case explicitly when they +see it. The fix is to correct `pr-build.yml`'s workflow/job names and +`CENTRALIS_REQUIRED_CHECK` together — but if you need to land the fix itself, +lift the rule as above. + ## Guardrails - Never force-push `dev`, `main` or `release`. -- 2.45.2 From deb039ee249f9c9595a1836c42492082aaec3327 Mon Sep 17 00:00:00 2001 From: emi Date: Mon, 27 Jul 2026 00:56:43 +0000 Subject: [PATCH 29/37] accounts: sign in with Google, on a shared provider-agnostic OAuth engine (#122) --- backend/accounts/models.py | 62 +- backend/accounts/oauth.py | 528 ++++++++++++++++++ backend/accounts/schemas.py | 22 +- .../alembic/versions/0005_oauth_accounts.py | 104 ++++ backend/notifications/discord_link.py | 342 +----------- backend/tests/accounts/test_oauth.py | 351 ++++++++++++ .../tests/notifications/test_discord_dm.py | 13 +- .../tests/notifications/test_discord_link.py | 190 +++---- .../tests/notifications/test_discord_login.py | 251 --------- backend/web/app.py | 9 +- frontend/static/account.js | 184 ++++-- frontend/static/style.css | 22 +- infra/deploy/thermograph.env.example | 12 + 13 files changed, 1333 insertions(+), 757 deletions(-) create mode 100644 backend/accounts/oauth.py create mode 100644 backend/alembic/versions/0005_oauth_accounts.py create mode 100644 backend/tests/accounts/test_oauth.py delete mode 100644 backend/tests/notifications/test_discord_login.py diff --git a/backend/accounts/models.py b/backend/accounts/models.py index 56ed510..debea78 100644 --- a/backend/accounts/models.py +++ b/backend/accounts/models.py @@ -24,7 +24,7 @@ from sqlalchemy import ( UniqueConstraint, false, ) -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.orm import Mapped, mapped_column, relationship from accounts.db import Base @@ -41,12 +41,60 @@ class User(SQLAlchemyBaseUserTableUUID, Base): # opt-in); the user can mute it while staying linked. Same migration caveat. discord_dm: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false()) - # True when the account was created by "Sign in with Discord" and so has no - # password its owner has ever seen (hashed_password is NOT NULL, so the row - # carries a generated one nobody knows). Load-bearing: unlinking Discord from - # such an account would remove its only way in, so discord_link.py refuses. - discord_only: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, - server_default=false()) + # True when the account was created by signing in with an OAuth provider and so + # has no password its owner has ever seen (hashed_password is NOT NULL, so the + # row carries a generated one nobody knows). Load-bearing: unlinking the *last* + # linked provider from such an account would remove its only way in, so + # accounts/oauth.py refuses that. Was `discord_only` before Google was added. + oauth_only: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, + server_default=false()) + + # selectin, not the lazy default: these are read while serialising /users/me on + # the async engine, where a lazy load raises MissingGreenlet rather than + # quietly issuing a query. One extra SELECT per user load is the price. + oauth_accounts: Mapped[list["OAuthAccount"]] = relationship( + "OAuthAccount", lazy="selectin", cascade="all, delete-orphan", + ) + + @property + def oauth_providers(self) -> list[str]: + """Linked provider names — read straight onto UserRead by from_attributes.""" + return sorted(a.provider for a in self.oauth_accounts) + + +class OAuthAccount(Base): + """One external identity — a provider plus that provider's own user id — bound + to a Thermograph account. This is what "sign in with X" resolves against. + + Keyed on ``subject``, never email: the provider's id for an account is stable, + while an email address can be changed or reassigned. Email is stored only for + support and debugging, and is deliberately not used for lookup. + + Note ``User.discord_id`` is *not* redundant with a discord row here. That column + is a delivery address — notify.py DMs it — and survives on its own terms; this + table is purely about authentication. + """ + __tablename__ = "oauth_account" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + user_id: Mapped[uuid.UUID] = mapped_column( + GUID, ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + provider: Mapped[str] = mapped_column(String(32), nullable=False) + # "sub" for Google, "id" for Discord. + subject: Mapped[str] = mapped_column(String(64), nullable=False) + email: Mapped[str | None] = mapped_column(String(320), nullable=True) + created_at: Mapped[float] = mapped_column(Float, nullable=False, default=time.time) + + __table_args__ = ( + # A provider account signs into exactly one Thermograph account — this is + # the constraint that stops one Google login resolving two ways. + UniqueConstraint("provider", "subject", name="uq_oauth_provider_subject"), + # And an account holds at most one identity per provider, so "connect + # Google" is idempotent rather than accumulating rows. + UniqueConstraint("user_id", "provider", name="uq_oauth_user_provider"), + Index("idx_oauth_user", "user_id"), + ) class AccessToken(SQLAlchemyBaseAccessTokenTableUUID, Base): diff --git a/backend/accounts/oauth.py b/backend/accounts/oauth.py new file mode 100644 index 0000000..75e82ad --- /dev/null +++ b/backend/accounts/oauth.py @@ -0,0 +1,528 @@ +"""Sign in with an external provider, and connect one to an existing account. + +One engine, two providers (Discord, Google). Adding a third means adding a +``Provider`` entry and nothing else — the flow, the signed state, the account +resolution and the session issuance are all provider-agnostic on purpose. That is +not tidiness for its own sake: account resolution is the security-critical part, +and a second hand-rolled copy is where a subtle divergence becomes a takeover. + +Two flows over one authorization-code grant, told apart by the signed `state`: + + link — /oauth/{provider}/link/start a signed-in user attaches an identity + login — /oauth/{provider}/login/start an anonymous visitor authenticates + +`state` is signed with the app's auth secret (stdlib hmac, no new dependency) and +carries the provider, the purpose, and — for a link — the Thermograph user id. All +three are inside the HMAC, so a state cannot be replayed, bound to a different +account, replayed as the other flow, or replayed against a different provider. + +How a login resolves to an account, in order: + + 1. An ``oauth_account`` row for (provider, subject) — the durable key, no email + needed and immune to the user changing their address at the provider. + 2. Otherwise the provider's email, but **only if it reports it verified**. That + flag is the sole evidence the person owns the address; matching an existing + Thermograph account on an unverified one would hand that account to whoever + typed the address into Discord or Google. Unverified is refused outright. + 3. No account for that email: create one, flagged ``oauth_only`` because it has + no password its owner has ever seen. + +**Legacy routes.** Discord shipped first, under /discord/*, and its callback URL is +registered in Discord's developer portal — so /discord/link/callback keeps working +forever, and the /discord/* aliases stay because the frontend deploys independently +of this service and an older one must keep working against a newer backend. +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import os +import time +import urllib.parse +from dataclasses import dataclass, field +from typing import Callable + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from sqlalchemy import delete, func, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from accounts.db import get_async_session +from accounts.models import OAuthAccount, User +from accounts.users import ( + SECRET, + attach_session, + current_active_user, + current_user_optional, + get_access_token_db, + get_user_manager, +) +from core import audit + +router = APIRouter(tags=["oauth"]) + +BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").rstrip("/") + +# How long a start->callback round trip may take before the signed state expires. +_STATE_TTL = 600 +_TIMEOUT = httpx.Timeout(10.0) + + +# --- providers ---------------------------------------------------------------- +def _parse_discord(p: dict) -> tuple[str, str, bool]: + return (str(p.get("id") or ""), + str(p.get("email") or "").strip().lower(), + p.get("verified") is True) + + +def _parse_google(p: dict) -> tuple[str, str, bool]: + # OpenID Connect userinfo. `email_verified` arrives as a real bool from the + # JSON endpoint, but Google has historically also serialised it as the string + # "true" in some responses, so accept both rather than silently reading a + # verified address as unverified and refusing every Google login. + verified = p.get("email_verified") + return (str(p.get("sub") or ""), + str(p.get("email") or "").strip().lower(), + verified is True or str(verified).lower() == "true") + + +@dataclass(frozen=True) +class Provider: + name: str + label: str + authorize_url: str + token_url: str + userinfo_url: str + # Linking needs only an identity; signing in also needs the address to resolve + # or create an account by, which is a separate scope the user is prompted for. + scope_link: str + scope_login: str + # Where this provider's callback lives. Discord's is grandfathered to the path + # already registered in its portal; anything new gets the canonical one. + callback_path: str + parse: Callable[[dict], tuple[str, str, bool]] + # Extra authorize-URL params. Google needs prompt=consent to re-issue consent, + # and Discord takes the same key, so this stays per-provider rather than shared. + extra_authorize: dict = field(default_factory=dict) + + +PROVIDERS: dict[str, Provider] = { + "discord": Provider( + name="discord", + label="Discord", + authorize_url="https://discord.com/api/oauth2/authorize", + token_url="https://discord.com/api/oauth2/token", + userinfo_url="https://discord.com/api/users/@me", + scope_link="identify", + scope_login="identify email", + callback_path="/api/v2/discord/link/callback", + parse=_parse_discord, + extra_authorize={"prompt": "consent"}, + ), + "google": Provider( + name="google", + label="Google", + authorize_url="https://accounts.google.com/o/oauth2/v2/auth", + token_url="https://oauth2.googleapis.com/token", + userinfo_url="https://openidconnect.googleapis.com/v1/userinfo", + # Google has no identity-only scope worth the name: openid alone yields a + # subject but no address, and we want the address recorded on the link too. + scope_link="openid email", + scope_login="openid email", + callback_path="/api/v2/oauth/google/callback", + parse=_parse_google, + # access_type=online: there is no offline work to do on the user's behalf, + # so asking for a refresh token would be collecting a credential we would + # never use. include_granted_scopes keeps incremental consent sane. + extra_authorize={"prompt": "consent", "access_type": "online", + "include_granted_scopes": "true"}, + ), +} + +# Credentials, read once at import. A dict rather than per-provider constants so a +# test can enable or disable one provider without touching the others. +CREDENTIALS: dict[str, tuple[str, str]] = { + "discord": (os.environ.get("THERMOGRAPH_DISCORD_APP_ID", "").strip(), + os.environ.get("THERMOGRAPH_DISCORD_CLIENT_SECRET", "").strip()), + "google": (os.environ.get("THERMOGRAPH_GOOGLE_CLIENT_ID", "").strip(), + os.environ.get("THERMOGRAPH_GOOGLE_CLIENT_SECRET", "").strip()), +} + + +def enabled(provider: str) -> bool: + """Whether this provider is configured. An unconfigured provider surfaces no UI + and every one of its routes bounces, so a half-set environment is inert rather + than broken.""" + cid, secret = CREDENTIALS.get(provider, ("", "")) + return bool(cid and secret) + + +def _provider_or_404(name: str) -> Provider: + p = PROVIDERS.get(name) + if p is None: + raise HTTPException(status_code=404, detail="Unknown provider.") + return p + + +# --- signed state ------------------------------------------------------------ +def _b64(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def _unb64(s: str) -> bytes: + return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4)) + + +def _sign_state(uid: str | None, purpose: str, provider: str) -> str: + payload = _b64(json.dumps( + {"u": uid or "", "t": int(time.time()), "p": purpose, "pr": provider} + ).encode()) + mac = hmac.new(SECRET.encode(), payload.encode(), hashlib.sha256).hexdigest()[:32] + return f"{payload}.{mac}" + + +def _verify_state(state: str, purpose: str, provider: str) -> str | None: + """The Thermograph user id this state was minted for, or None if it's missing, + tampered, expired, or was minted for a different purpose or provider. + + A login state has no user id yet, so it verifies as ``""`` — falsy, but distinct + from the None that means reject. Callers must test against None, not + truthiness, or a valid login state reads as a failure. + """ + try: + payload, mac = state.split(".", 1) + except (ValueError, AttributeError): + return None + expect = hmac.new(SECRET.encode(), payload.encode(), hashlib.sha256).hexdigest()[:32] + if not hmac.compare_digest(mac, expect): + return None + try: + data = json.loads(_unb64(payload)) + except (ValueError, json.JSONDecodeError): + return None + if time.time() - float(data.get("t", 0)) > _STATE_TTL: + return None + if data.get("p") != purpose: + return None + # States minted before the provider field existed are Discord's; they age out + # in _STATE_TTL, so this fallback only spans a deploy. + if data.get("pr", "discord") != provider: + return None + return data.get("u") or "" + + +# --- redirects --------------------------------------------------------------- +def _redirect_uri(request: Request, provider: Provider) -> str: + """The callback URL, matched to how the app is actually reached (scheme/host + + base path). Must equal the redirect registered in the provider's console.""" + proto = request.headers.get("x-forwarded-proto") or request.url.scheme + host = request.headers.get("host") or request.url.netloc + return f"{proto}://{host}{BASE}{provider.callback_path}" + + +def _authorize_redirect(request: Request, provider: Provider, scope: str, + state: str) -> RedirectResponse: + cid, _ = CREDENTIALS[provider.name] + params = { + "client_id": cid, + "redirect_uri": _redirect_uri(request, provider), + "response_type": "code", + "scope": scope, + "state": state, + **provider.extra_authorize, + } + return RedirectResponse( + url=f"{provider.authorize_url}?{urllib.parse.urlencode(params)}", status_code=303) + + +def _account_redirect(status: str, provider: str) -> RedirectResponse: + """Back to the alerts page, which surfaces the outcome as a toast; 303 so the + browser issues a GET after the callback. + + Emits `discord=` as well for Discord, because the frontend deploys + independently of this service and the one in production reads that parameter. + Drop it once no deployed frontend predates `oauth=`.""" + params = {"oauth": status, "provider": provider} + if provider == "discord": + params["discord"] = status + return RedirectResponse(url=f"{BASE}/alerts?{urllib.parse.urlencode(params)}", + status_code=303) + + +# --- provider API ------------------------------------------------------------- +async def _fetch_profile(request: Request, provider: Provider, code: str) -> dict | None: + """Trade the authorization code for the provider's user object, or None if any + step fails. Callers treat None as a graceful error, never a crash.""" + cid, secret = CREDENTIALS[provider.name] + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + tok = await client.post(provider.token_url, data={ + "client_id": cid, + "client_secret": secret, + "grant_type": "authorization_code", + "code": code, + "redirect_uri": _redirect_uri(request, provider), + }, headers={"Content-Type": "application/x-www-form-urlencoded"}) + if tok.status_code >= 300: + return None + access = tok.json().get("access_token") + if not access: + return None + me = await client.get(provider.userinfo_url, + headers={"Authorization": f"Bearer {access}"}) + if me.status_code >= 300: + return None + profile = me.json() + except Exception: # noqa: BLE001 - any network/parse failure is a graceful error + return None + return profile if isinstance(profile, dict) else None + + +# --- persistence -------------------------------------------------------------- +async def _account_for(session: AsyncSession, provider: str, + subject: str) -> OAuthAccount | None: + return (await session.execute( + select(OAuthAccount).where(OAuthAccount.provider == provider, + OAuthAccount.subject == subject) + )).scalar_one_or_none() + + +async def _link_row(session: AsyncSession, user_id, provider: str, subject: str, + email: str | None) -> None: + """Bind (provider, subject) to a user, plus the Discord-only side effect. + + discord_id is a *delivery* address, not an identity — notify.py DMs it — so the + Discord link has to write it as well as the oauth_account row. Turning DMs on + here mirrors the original behaviour: linking is an active opt-in, and the user + can mute it (POST /discord/dm) while staying linked. + """ + session.add(OAuthAccount(user_id=user_id, provider=provider, subject=subject, + email=email or None)) + if provider == "discord": + await session.execute( + update(User).where(User.id == user_id) + .values(discord_id=subject, discord_dm=True)) + await session.commit() + + +async def _create_user(user_manager, email: str) -> User: + """Create an account for an identity that matched none of ours. + + Deliberately goes around ``UserManager.create()``: there is no password to + validate, and its ``on_after_register`` mails a confirmation link for an address + the provider has already verified. The generated password exists only because + ``hashed_password`` is NOT NULL — nobody ever sees it, which is what + ``oauth_only`` records. + """ + password = user_manager.password_helper.generate() + return await user_manager.user_db.create({ + "email": email, + "hashed_password": user_manager.password_helper.hash(password), + "is_active": True, + "is_superuser": False, + "is_verified": True, + "oauth_only": True, + }) + + +# --- routes ------------------------------------------------------------------- +@router.get("/oauth/config") +async def oauth_config(): + """Which providers this server can actually complete a flow with. The sign-in + modal and account menu render from this, so an unconfigured provider shows no + dead-end UI and configuring one later makes it appear on its own.""" + return {"providers": [ + {"name": p.name, "label": p.label, "enabled": enabled(p.name)} + for p in PROVIDERS.values() + ]} + + +@router.get("/oauth/{provider}/link/start") +async def link_start(provider: str, request: Request, user=Depends(current_active_user)): + p = _provider_or_404(provider) + if not enabled(provider): + return _account_redirect("unavailable", provider) + return _authorize_redirect(request, p, p.scope_link, + _sign_state(str(user.id), "link", provider)) + + +@router.get("/oauth/{provider}/login/start") +async def login_start(provider: str, request: Request, + user=Depends(current_user_optional)): + """Sign in with a provider. Unauthenticated by design — this is how you get a + session, not something you do with one.""" + p = _provider_or_404(provider) + if not enabled(provider): + return _account_redirect("unavailable", provider) + # Someone already signed in who lands here wants to connect, not to be switched + # into whichever account the external identity resolves to. + if user is not None: + return _authorize_redirect(request, p, p.scope_link, + _sign_state(str(user.id), "link", provider)) + return _authorize_redirect(request, p, p.scope_login, + _sign_state(None, "login", provider)) + + +async def _callback(provider: str, request: Request, user, session, user_manager, + access_token_db) -> RedirectResponse: + p = _provider_or_404(provider) + if not enabled(provider): + return _account_redirect("unavailable", provider) + # User declined on the provider's screen, or the state doesn't check out. + code = request.query_params.get("code") + state = request.query_params.get("state", "") + if not code: + return _account_redirect("cancelled", provider) + + # Which flow minted this state? Purpose and provider are both inside the HMAC, + # so a state only verifies under the flow and provider it was signed for. + uid = _verify_state(state, "link", provider) + if uid is not None: + return await _finish_link(p, request, code, uid, user, session) + if _verify_state(state, "login", provider) is not None: + return await _finish_login(p, request, code, session, user_manager, + access_token_db) + return _account_redirect("error", provider) + + +@router.get("/oauth/{provider}/callback") +async def oauth_callback( + provider: str, + request: Request, + user=Depends(current_user_optional), + session: AsyncSession = Depends(get_async_session), + user_manager=Depends(get_user_manager), + access_token_db=Depends(get_access_token_db), +): + return await _callback(provider, request, user, session, user_manager, + access_token_db) + + +async def _finish_link(p: Provider, request: Request, code: str, uid: str, user, + session: AsyncSession) -> RedirectResponse: + # Linking always acts on the person actually logged in, and only on the one the + # state was minted for. + if user is None or str(user.id) != uid: + return _account_redirect("error", p.name) + profile = await _fetch_profile(request, p, code) + if profile is None: + return _account_redirect("error", p.name) + subject, email, _ = p.parse(profile) + if not subject: + return _account_redirect("error", p.name) + # (provider, subject) is UNIQUE, so without this the insert would raise an + # IntegrityError and 500 instead of explaining itself. + owner = await _account_for(session, p.name, subject) + if owner is not None: + return _account_redirect("linked" if owner.user_id == user.id else "taken", p.name) + # And (user_id, provider) is UNIQUE: this account already has a different + # identity from this provider attached. + existing = (await session.execute( + select(OAuthAccount).where(OAuthAccount.user_id == user.id, + OAuthAccount.provider == p.name) + )).scalar_one_or_none() + if existing is not None: + return _account_redirect("already", p.name) + await _link_row(session, user.id, p.name, subject, email) + return _account_redirect("linked", p.name) + + +async def _finish_login(p: Provider, request: Request, code: str, + session: AsyncSession, user_manager, + access_token_db) -> RedirectResponse: + """Resolve the external identity to an account and sign in as it. + + Deliberately ignores any session already present. A login state carries no user + id — it can't, there is no user yet — so it is replayable against whoever + happens to be signed in. Treating it as a link would therefore be a + forced-linking takeover: an attacker who gets a victim to open this callback + with a code from the *attacker's* provider account would attach their identity + to the victim's account, then sign in as them. Only link/start, whose state is + bound to a specific user id, may link. The worst a replayed login state can do + is sign someone into the attacker's own account. + """ + profile = await _fetch_profile(request, p, code) + if profile is None: + return _account_redirect("error", p.name) + subject, email, verified = p.parse(profile) + if not subject: + return _account_redirect("error", p.name) + + created = False + acct = await _account_for(session, p.name, subject) + if acct is not None: + user = await user_manager.user_db.get(acct.user_id) + if user is None: + return _account_redirect("error", p.name) + else: + # Nothing carries this identity, so fall back to the email — the only other + # identity the provider gives us, and only when it vouches for it. + if not email: + return _account_redirect("noemail", p.name) + if not verified: + return _account_redirect("unverified", p.name) + user = await user_manager.user_db.get_by_email(email) + if user is None: + user = await _create_user(user_manager, email) + await _link_row(session, user.id, p.name, subject, email) + audit.log_activity("auth.register", {"user_id": str(user.id), "via": p.name}) + created = True + elif not user.is_active: + # Checked before the link, not just before the session: a deactivated + # account must not quietly acquire an identity on the way out. + return _account_redirect("inactive", p.name) + else: + # That address belongs to an account already tied to a different + # identity from this provider; connecting would silently move it. + existing = (await session.execute( + select(OAuthAccount).where(OAuthAccount.user_id == user.id, + OAuthAccount.provider == p.name) + )).scalar_one_or_none() + if existing is not None: + return _account_redirect("mismatch", p.name) + await _link_row(session, user.id, p.name, subject, email) + if not user.is_active: + return _account_redirect("inactive", p.name) + response = _account_redirect("created" if created else "signedin", p.name) + return await attach_session(response, user, user_manager, access_token_db, request) + + +async def _unlink(provider: str, user, session: AsyncSession) -> None: + p = _provider_or_404(provider) + linked = (await session.execute( + select(func.count()).select_from(OAuthAccount) + .where(OAuthAccount.user_id == user.id) + )).scalar_one() + mine = await session.execute( + select(OAuthAccount).where(OAuthAccount.user_id == user.id, + OAuthAccount.provider == p.name)) + if mine.scalar_one_or_none() is None: + return # already unlinked; idempotent + if user.oauth_only and linked <= 1: + # No password anyone has ever seen, and this is the last way in. No + # reset-password route is mounted to recover from it either. + raise HTTPException( + status_code=409, + detail=f"This account was created with {p.label} and has no password, " + "so unlinking would leave no way to sign in.", + ) + await session.execute( + delete(OAuthAccount).where(OAuthAccount.user_id == user.id, + OAuthAccount.provider == p.name)) + if p.name == "discord": + await session.execute( + update(User).where(User.id == user.id) + .values(discord_id=None, discord_dm=False)) + await session.commit() + + +@router.post("/oauth/{provider}/unlink", status_code=204) +async def unlink( + provider: str, + user=Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + await _unlink(provider, user, session) diff --git a/backend/accounts/schemas.py b/backend/accounts/schemas.py index 7385240..43f77c0 100644 --- a/backend/accounts/schemas.py +++ b/backend/accounts/schemas.py @@ -8,7 +8,7 @@ import uuid from typing import Literal from fastapi_users import schemas -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, computed_field, field_validator from data import grading @@ -26,12 +26,24 @@ BOOKMARK_IMPORT_MAX_ITEMS = 200 class UserRead(schemas.BaseUser[uuid.UUID]): display_name: str | None = None # Present so the frontend can show linked/unlinked state; set via the OAuth - # flow (discord_link.py), not editable through the profile. + # flow (accounts/oauth.py), not editable through the profile. discord_id: str | None = None discord_dm: bool = False - # True when Discord is the account's only credential, so the UI can explain - # why "Unlink Discord" is refused rather than just failing. - discord_only: bool = False + # True when an OAuth provider is the account's only credential, so the UI can + # explain why unlinking the last one is refused rather than just failing. + oauth_only: bool = False + # Provider names currently linked, e.g. ["discord", "google"] — what the + # account menu renders its connected-accounts section from. + oauth_providers: list[str] = [] + + # Superseded by oauth_only. Kept because frontend and backend deploy + # independently: the frontend in production reads this field, and dropping it + # would break its account menu the moment this backend shipped ahead. Remove + # once no deployed frontend predates oauth_only. + @computed_field + @property + def discord_only(self) -> bool: + return self.oauth_only class UserCreate(schemas.BaseUserCreate): diff --git a/backend/alembic/versions/0005_oauth_accounts.py b/backend/alembic/versions/0005_oauth_accounts.py new file mode 100644 index 0000000..1b20b73 --- /dev/null +++ b/backend/alembic/versions/0005_oauth_accounts.py @@ -0,0 +1,104 @@ +"""oauth_account table; user.discord_only -> user.oauth_only + +Generalises "sign in with Discord" to any provider (see accounts/oauth.py). Three +steps, each conditional so the revision is safe on a database created before or +after the ORM gained these definitions — revision 0001 builds the schema with +``Base.metadata.create_all`` off *live* model metadata, so a freshly created +database already has both the table and the renamed column, and an unconditional +DDL would fail there with duplicate-object errors. + +The backfill is the load-bearing step. Discord identities used to live in +``user.discord_id``, and account resolution now keys on ``oauth_account``; without +copying the existing rows across, every already-linked user would silently stop +being recognised at login and would get a *second* account created on their next +sign-in. ``user.discord_id`` is deliberately left in place — it is the DM delivery +address notify.py reads, not merely an identity. + +Revision ID: 0005_oauth_accounts +Revises: 0004_bookmarks +Create Date: 2026-07-26 +""" +import time + +import sqlalchemy as sa +from alembic import op +# The same UUID type the ORM uses for user.id — a plain sa.Uuid renders differently +# on SQLite and would not match the column create_all produces. +from fastapi_users_db_sqlalchemy.generics import GUID + +revision = "0005_oauth_accounts" +down_revision = "0004_bookmarks" +branch_labels = None +depends_on = None + + +def _columns(table: str) -> set[str]: + return {c["name"] for c in sa.inspect(op.get_bind()).get_columns(table)} + + +def _tables() -> set[str]: + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def upgrade() -> None: + cols = _columns("user") + + # 1. Rename the flag. It no longer means "Discord created this account", it + # means "no password its owner has ever seen, whichever provider made it". + if "oauth_only" in cols: + # A database built by 0001 already has oauth_only (create_all reads *current* + # metadata), and 0003 then re-added an empty discord_only beside it, because + # 0003 tests for the name it knew. Drop that vestige, or a fresh database + # carries a column an upgraded one does not — schema drift that would only + # surface much later. Safe: on this path the column was created empty + # moments ago and nothing has ever read it. + if "discord_only" in cols: + op.drop_column("user", "discord_only") + elif "discord_only" in cols: + # The real upgrade path: a deployed database carrying live values. Renamed, + # never dropped, so nothing is lost. + op.alter_column("user", "discord_only", new_column_name="oauth_only") + else: + # Database predates even discord_only (never deployed with it). + op.add_column("user", sa.Column("oauth_only", sa.Boolean(), nullable=False, + server_default=sa.false())) + + # 2. The identity table. + if "oauth_account" not in _tables(): + op.create_table( + "oauth_account", + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("user_id", GUID(), sa.ForeignKey("user.id", ondelete="CASCADE"), + nullable=False), + sa.Column("provider", sa.String(length=32), nullable=False), + sa.Column("subject", sa.String(length=64), nullable=False), + sa.Column("email", sa.String(length=320), nullable=True), + sa.Column("created_at", sa.Float(), nullable=False), + sa.UniqueConstraint("provider", "subject", name="uq_oauth_provider_subject"), + sa.UniqueConstraint("user_id", "provider", name="uq_oauth_user_provider"), + ) + op.create_index("idx_oauth_user", "oauth_account", ["user_id"]) + + # 3. Backfill every existing Discord link. Guarded by NOT EXISTS so re-running + # (or running after create_all already produced the table) cannot violate + # either unique constraint. + op.execute(sa.text(""" + INSERT INTO oauth_account (user_id, provider, subject, email, created_at) + SELECT u.id, 'discord', u.discord_id, u.email, :now + FROM "user" u + WHERE u.discord_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM oauth_account o + WHERE o.provider = 'discord' + AND (o.subject = u.discord_id OR o.user_id = u.id) + ) + """).bindparams(now=time.time())) + + +def downgrade() -> None: + if "oauth_account" in _tables(): + op.drop_index("idx_oauth_user", table_name="oauth_account") + op.drop_table("oauth_account") + cols = _columns("user") + if "oauth_only" in cols and "discord_only" not in cols: + op.alter_column("user", "oauth_only", new_column_name="discord_only") diff --git a/backend/notifications/discord_link.py b/backend/notifications/discord_link.py index 272d81c..dcb710e 100644 --- a/backend/notifications/discord_link.py +++ b/backend/notifications/discord_link.py @@ -1,243 +1,54 @@ -"""Discord as an identity: sign in with it, and connect it to an account. +"""Discord's legacy /discord/* auth routes, plus the DM opt-in toggle. -Two flows over one authorization-code grant, told apart by the signed `state`: +The OAuth flow itself moved to ``accounts/oauth.py`` when Google was added — this +module holds no auth logic, only the paths that have to keep answering: - link — /discord/link/start an already-signed-in user attaches a Discord - account to the session they're in - login — /discord/login/start an anonymous visitor authenticates, creating or - resolving a Thermograph account from the Discord - identity +* **/discord/link/callback is registered in Discord's developer portal.** Changing + it would need an operator to edit the portal before any Discord login worked, so + it stays exactly where it is and forwards to the shared handler. +* The other /discord/* routes are what the currently-deployed frontend calls. + Frontend and backend deploy independently (see the root CLAUDE.md), so a newer + backend must keep serving an older frontend. They can go once no deployed + frontend predates /oauth/*. -**Both come back to /discord/link/callback.** Discord only honours redirect URIs -pre-registered in the developer portal, so a second callback path would mean an -operator has to add one there before login could work anywhere. Sharing the -registered one keeps this deployable with no portal change; the flows separate on -the state's purpose, which is inside the HMAC and so can't be flipped by hand. - -`state` is signed with the app's auth secret (stdlib hmac, no new dependency) and -carries the purpose plus, for a link, the Thermograph user id — so a callback can't -be replayed, bound to a different account, or replayed *as the other flow*. - -How a login resolves to an account, in order: - - 1. A user already carrying this `discord_id` — the durable key, no email needed. - 2. Otherwise the Discord email, but **only if Discord reports it verified**. - That flag is the sole evidence the person owns the address; matching an - existing Thermograph account on an unverified one would hand that account to - whoever typed the address into Discord. Unverified is refused outright. - 3. No account for that email: create one. It has no password its owner has ever - seen, so it is flagged `discord_only` and unlink is refused (see `unlink`) — - otherwise unlinking would delete the only way back in. +``POST /discord/dm`` is not legacy and is not moving: it toggles alert delivery, +which is a notifications concern, not an authentication one. """ from __future__ import annotations -import base64 -import hashlib -import hmac -import json -import os -import time -import urllib.parse - -import httpx -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import RedirectResponse +from fastapi import APIRouter, Depends, Request from pydantic import BaseModel -from sqlalchemy import select, update +from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession +from accounts import oauth from accounts.db import get_async_session from accounts.models import User from accounts.users import ( - SECRET, - attach_session, current_active_user, current_user_optional, get_access_token_db, get_user_manager, ) -from core import audit router = APIRouter(tags=["discord"]) -CLIENT_ID = os.environ.get("THERMOGRAPH_DISCORD_APP_ID", "").strip() -CLIENT_SECRET = os.environ.get("THERMOGRAPH_DISCORD_CLIENT_SECRET", "").strip() -BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").rstrip("/") -_AUTHORIZE = "https://discord.com/api/oauth2/authorize" -_TOKEN = "https://discord.com/api/oauth2/token" -_ME = "https://discord.com/api/users/@me" - -# Linking only needs the account id. Signing in also needs the address to match or -# create a Thermograph account by, which is a separate scope Discord prompts for. -_SCOPE_LINK = "identify" -_SCOPE_LOGIN = "identify email" - -# How long a start->callback round trip may take before the signed state expires. -_STATE_TTL = 600 -_TIMEOUT = httpx.Timeout(10.0) - - -def enabled() -> bool: - return bool(CLIENT_ID and CLIENT_SECRET) - - -# --- signed state ------------------------------------------------------------ -def _b64(raw: bytes) -> str: - return base64.urlsafe_b64encode(raw).decode().rstrip("=") - - -def _unb64(s: str) -> bytes: - return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4)) - - -def _sign_state(uid: str | None, purpose: str = "link") -> str: - payload = _b64(json.dumps( - {"u": uid or "", "t": int(time.time()), "p": purpose} - ).encode()) - mac = hmac.new(SECRET.encode(), payload.encode(), hashlib.sha256).hexdigest()[:32] - return f"{payload}.{mac}" - - -def _verify_state(state: str, purpose: str = "link") -> str | None: - """The Thermograph user id this state was minted for, or None if it's missing, - tampered, expired, or was minted for a *different* purpose. - - A login state has no user id yet, so it verifies as ``""`` — falsy, but - distinct from the None that means "reject". Callers must test against None, - not truthiness, or a valid login state reads as a failure. - """ - try: - payload, mac = state.split(".", 1) - except (ValueError, AttributeError): - return None - expect = hmac.new(SECRET.encode(), payload.encode(), hashlib.sha256).hexdigest()[:32] - if not hmac.compare_digest(mac, expect): - return None - try: - data = json.loads(_unb64(payload)) - except (ValueError, json.JSONDecodeError): - return None - if time.time() - float(data.get("t", 0)) > _STATE_TTL: - return None - # States minted before the purpose field existed are links; they age out in - # _STATE_TTL, so this fallback only spans a deploy. - if data.get("p", "link") != purpose: - return None - return data.get("u") or "" - - -def _redirect_uri(request: Request) -> str: - """The callback URL, matched to how the app is actually reached (scheme/host + - base path). Must equal the redirect registered in the Discord portal.""" - proto = request.headers.get("x-forwarded-proto") or request.url.scheme - host = request.headers.get("host") or request.url.netloc - return f"{proto}://{host}{BASE}/api/v2/discord/link/callback" - - -def _authorize_redirect(request: Request, scope: str, state: str) -> RedirectResponse: - params = { - "client_id": CLIENT_ID, - "redirect_uri": _redirect_uri(request), - "response_type": "code", - "scope": scope, - "state": state, - "prompt": "consent", - } - return RedirectResponse(url=f"{_AUTHORIZE}?{urllib.parse.urlencode(params)}", - status_code=303) - - -def _account_redirect(status: str) -> RedirectResponse: - # Back to the alerts page, which surfaces the outcome as a toast; 303 so the - # browser issues a GET after the callback. The frontend serves this page at - # /alerts — the route is named for the feature, not for subscriptions.html. - return RedirectResponse(url=f"{BASE}/alerts?discord={status}", status_code=303) - - -# --- Discord API -------------------------------------------------------------- -async def _fetch_profile(request: Request, code: str) -> dict | None: - """Trade the authorization code for the Discord user object, or None if any - step fails. Callers treat None as a graceful error, never a crash.""" - try: - async with httpx.AsyncClient(timeout=_TIMEOUT) as client: - tok = await client.post(_TOKEN, data={ - "client_id": CLIENT_ID, - "client_secret": CLIENT_SECRET, - "grant_type": "authorization_code", - "code": code, - "redirect_uri": _redirect_uri(request), - }, headers={"Content-Type": "application/x-www-form-urlencoded"}) - if tok.status_code >= 300: - return None - access = tok.json().get("access_token") - me = await client.get(_ME, headers={"Authorization": f"Bearer {access}"}) - if me.status_code >= 300: - return None - profile = me.json() - except Exception: # noqa: BLE001 - any network/parse failure is a graceful error - return None - return profile if isinstance(profile, dict) else None - - -async def _user_by_discord_id(session: AsyncSession, discord_id: str) -> User | None: - return ( - await session.execute(select(User).where(User.discord_id == discord_id)) - ).scalar_one_or_none() - - -async def _create_from_discord(user_manager, email: str, discord_id: str) -> User: - """Create an account for a Discord identity that matched none of ours. - - Deliberately goes around ``UserManager.create()``: there is no password to - validate, and its ``on_after_register`` mails a confirmation link for an - address Discord has already verified. The generated password exists only - because ``hashed_password`` is NOT NULL — nobody ever sees it, which is what - ``discord_only`` records. - """ - password = user_manager.password_helper.generate() - user = await user_manager.user_db.create({ - "email": email, - "hashed_password": user_manager.password_helper.hash(password), - "is_active": True, - "is_superuser": False, - "is_verified": True, - "discord_id": discord_id, - "discord_dm": True, - "discord_only": True, - }) - audit.log_activity("auth.register", {"user_id": str(user.id), "via": "discord"}) - return user - - -# --- routes ------------------------------------------------------------------ @router.get("/discord/config") async def discord_config(): - """Whether Discord is configured on this server. The account menu and the sign-in - modal use it to offer Discord only when it will actually work — so an - unconfigured server surfaces no dead-end Discord UI, and enabling it later - (setting the OAuth env vars) makes both entries appear on their own.""" - return {"enabled": enabled()} + """Superseded by /oauth/config, which reports every provider. Kept because the + deployed frontend asks this one.""" + return {"enabled": oauth.enabled("discord")} @router.get("/discord/link/start") async def link_start(request: Request, user=Depends(current_active_user)): - if not enabled(): - return _account_redirect("unavailable") - return _authorize_redirect(request, _SCOPE_LINK, _sign_state(str(user.id), "link")) + return await oauth.link_start("discord", request, user) @router.get("/discord/login/start") async def login_start(request: Request, user=Depends(current_user_optional)): - """Sign in with Discord. Unauthenticated by design — this is how you get a - session, not something you do with one.""" - if not enabled(): - return _account_redirect("unavailable") - # Someone already signed in who lands here wants to connect, not to be - # switched into whichever account the Discord identity resolves to. - if user is not None: - return _authorize_redirect(request, _SCOPE_LINK, _sign_state(str(user.id), "link")) - return _authorize_redirect(request, _SCOPE_LOGIN, _sign_state(None, "login")) + return await oauth.login_start("discord", request, user) @router.get("/discord/link/callback") @@ -248,103 +59,9 @@ async def link_callback( user_manager=Depends(get_user_manager), access_token_db=Depends(get_access_token_db), ): - """The single redirect target both flows return to (see the module docstring).""" - if not enabled(): - return _account_redirect("unavailable") - # User declined on Discord's screen, or the state doesn't check out. - code = request.query_params.get("code") - state = request.query_params.get("state", "") - if not code: - return _account_redirect("cancelled") - - # Which flow minted this state? The purpose is inside the HMAC, so a state - # can only verify under the one it was signed for. - uid = _verify_state(state, "link") - if uid is not None: - return await _finish_link(request, code, uid, user, session) - if _verify_state(state, "login") is not None: - return await _finish_login(request, code, session, user_manager, access_token_db) - return _account_redirect("error") - - -async def _finish_link(request: Request, code: str, uid: str, user, - session: AsyncSession) -> RedirectResponse: - # Linking always acts on the person actually logged in, and only on the one - # the state was minted for. - if user is None or str(user.id) != uid: - return _account_redirect("error") - profile = await _fetch_profile(request, code) - if profile is None: - return _account_redirect("error") - discord_id = str(profile.get("id") or "") - if not discord_id: - return _account_redirect("error") - # discord_id is UNIQUE, so without this check the UPDATE would raise an - # IntegrityError and 500 instead of explaining itself. - owner = await _user_by_discord_id(session, discord_id) - if owner is not None and owner.id != user.id: - return _account_redirect("taken") - # Linking is an active opt-in, so turn DM alerts on; the user can mute them - # (POST /discord/dm) while staying linked. - await session.execute( - update(User).where(User.id == user.id).values(discord_id=discord_id, discord_dm=True) - ) - await session.commit() - return _account_redirect("linked") - - -async def _finish_login(request: Request, code: str, session: AsyncSession, - user_manager, access_token_db) -> RedirectResponse: - """Resolve the Discord identity to an account and sign in as it. - - Deliberately ignores any session already present. A login state carries no - user id — it can't, there is no user yet — so it is replayable against whoever - happens to be signed in. Treating it as a link would therefore be a forced- - linking takeover: an attacker who gets a victim to open this callback with a - code from the *attacker's* Discord would attach their identity to the victim's - account, then sign in as them. Only /discord/link/start, whose state is bound - to a specific user id, may link. The worst a replayed login state can do is - sign someone into the attacker's own account. - """ - profile = await _fetch_profile(request, code) - if profile is None: - return _account_redirect("error") - discord_id = str(profile.get("id") or "") - if not discord_id: - return _account_redirect("error") - - created = False - user = await _user_by_discord_id(session, discord_id) - if user is None: - # No account carries this Discord id, so fall back to the email — the only - # other identity Discord gives us, and only when Discord vouches for it. - email = str(profile.get("email") or "").strip().lower() - if not email: - return _account_redirect("noemail") - if profile.get("verified") is not True: - return _account_redirect("unverified") - user = await user_manager.user_db.get_by_email(email) - if user is None: - user = await _create_from_discord(user_manager, email, discord_id) - created = True - elif user.discord_id: - # That address belongs to an account already tied to a different - # Discord identity; connecting would silently move it. - return _account_redirect("mismatch") - elif not user.is_active: - # Checked before the link, not just before the session: a deactivated - # account must not quietly acquire a Discord identity on the way out. - return _account_redirect("inactive") - else: - await session.execute( - update(User).where(User.id == user.id) - .values(discord_id=discord_id, discord_dm=True) - ) - await session.commit() - if not user.is_active: - return _account_redirect("inactive") - response = _account_redirect("created" if created else "signedin") - return await attach_session(response, user, user_manager, access_token_db, request) + """The URL registered in Discord's portal. Both Discord flows land here.""" + return await oauth._callback("discord", request, user, session, user_manager, + access_token_db) @router.post("/discord/unlink", status_code=204) @@ -352,18 +69,7 @@ async def unlink( user=Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): - if user.discord_only: - # The account has no password anyone has ever seen, and no reset-password - # route is mounted to set one — unlinking here is unrecoverable. - raise HTTPException( - status_code=409, - detail="This account was created with Discord and has no password, " - "so unlinking would leave no way to sign in.", - ) - await session.execute( - update(User).where(User.id == user.id).values(discord_id=None, discord_dm=False) - ) - await session.commit() + await oauth._unlink("discord", user, session) class DmToggle(BaseModel): diff --git a/backend/tests/accounts/test_oauth.py b/backend/tests/accounts/test_oauth.py new file mode 100644 index 0000000..0b736d4 --- /dev/null +++ b/backend/tests/accounts/test_oauth.py @@ -0,0 +1,351 @@ +"""Sign in with an external provider: account resolution, session issuance, and the +guards that keep the login flow from becoming an account-takeover path. + +Parametrised over every configured provider, so Discord and Google are held to the +same rules rather than one being tested and the other assumed. Provider HTTP is +mocked; reuses the throwaway accounts DB from conftest. +""" +import pytest +from fastapi.testclient import TestClient + +from web import app as appmod +from accounts import db, oauth + +V2 = "/thermograph/api/v2" +PW = "supersecret123" + +# (provider, callback path, profile builder). Discord's callback is grandfathered +# to the URL registered in its developer portal; anything newer uses the canonical +# /oauth/{provider}/callback. +PROVIDERS = [ + ("discord", f"{V2}/discord/link/callback", + lambda sub, email, ver: {"id": sub, "email": email, "verified": ver}), + ("google", f"{V2}/oauth/google/callback", + lambda sub, email, ver: {"sub": sub, "email": email, "email_verified": ver}), +] +IDS = [p[0] for p in PROVIDERS] + + +@pytest.fixture(scope="module", autouse=True) +def _tables(): + db.Base.metadata.create_all(db.sync_engine) + + +@pytest.fixture(autouse=True) +def _configured(monkeypatch): + for name in oauth.PROVIDERS: + monkeypatch.setitem(oauth.CREDENTIALS, name, (f"{name}-id", f"{name}-secret")) + + +@pytest.fixture(params=PROVIDERS, ids=IDS) +def prov(request): + """A provider under test: .name, .callback, .profile(sub, email, verified).""" + name, callback, builder = request.param + + class P: + pass + p = P() + p.name, p.callback, p.profile = name, callback, builder + return p + + +class _Resp: + def __init__(self, code, data): self.status_code = code; self._data = data + def json(self): return self._data + + +def _mock(monkeypatch, profile): + """Stand in for httpx.AsyncClient: token exchange, then userinfo -> profile.""" + class _Client: + def __init__(self, *a, **k): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, url, **k): return _Resp(200, {"access_token": "tok"}) + async def get(self, url, **k): return _Resp(200, profile) + monkeypatch.setattr(oauth.httpx, "AsyncClient", _Client) + + +def _register(client, email): + assert client.post(f"{V2}/auth/register", + json={"email": email, "password": PW}).status_code in (201, 400) + + +def _login(client, email): + _register(client, email) + assert client.post(f"{V2}/auth/login", + data={"username": email, "password": PW}).status_code == 204 + + +def _cb(client, prov, state): + return client.get(f"{prov.callback}?code=abc&state={state}", follow_redirects=False) + + +def _me(client): + return client.get(f"{V2}/users/me").json() + + +# --- config ------------------------------------------------------------------- + +def test_config_lists_every_provider_and_its_enabled_state(monkeypatch): + c = TestClient(appmod.app) + monkeypatch.setitem(oauth.CREDENTIALS, "google", ("", "")) + by_name = {p["name"]: p for p in c.get(f"{V2}/oauth/config").json()["providers"]} + assert by_name["discord"]["enabled"] is True + assert by_name["google"]["enabled"] is False # half-configured is inert + assert by_name["google"]["label"] == "Google" + + +def test_unknown_provider_is_404(): + c = TestClient(appmod.app) + assert c.get(f"{V2}/oauth/gitlab/login/start", + follow_redirects=False).status_code == 404 + + +# --- start -------------------------------------------------------------------- + +def test_login_start_needs_no_session_and_asks_for_an_email_scope(prov): + c = TestClient(appmod.app) + r = c.get(f"{V2}/oauth/{prov.name}/login/start", follow_redirects=False) + assert r.status_code == 303 + loc = r.headers["location"] + assert loc.startswith(oauth.PROVIDERS[prov.name].authorize_url) + # Whatever the provider calls it, the login scope must be able to yield an + # address — without one there is no account to resolve or create. + assert "email" in loc and "state=" in loc + + +def test_login_start_while_signed_in_links_instead_of_switching_account(prov): + c = TestClient(appmod.app) + _login(c, f"already-in-{prov.name}@example.com") + r = c.get(f"{V2}/oauth/{prov.name}/login/start", follow_redirects=False) + state = r.headers["location"].split("state=")[1].split("&")[0] + assert oauth._verify_state(state, "link", prov.name) == _me(c)["id"] + + +def test_start_when_unconfigured_bounces_back(prov, monkeypatch): + monkeypatch.setitem(oauth.CREDENTIALS, prov.name, ("", "")) + c = TestClient(appmod.app) + r = c.get(f"{V2}/oauth/{prov.name}/login/start", follow_redirects=False) + assert r.status_code == 303 and "oauth=unavailable" in r.headers["location"] + + +# --- signed state ------------------------------------------------------------- + +def test_state_purposes_and_providers_do_not_cross_over(): + login = oauth._sign_state(None, "login", "google") + link = oauth._sign_state("user-123", "link", "google") + # A login state carries no user id but verifies as "" — distinct from reject. + assert oauth._verify_state(login, "login", "google") == "" + assert oauth._verify_state(login, "link", "google") is None + assert oauth._verify_state(link, "link", "google") == "user-123" + assert oauth._verify_state(link, "login", "google") is None + # And a state minted for one provider is worthless at another's callback. + assert oauth._verify_state(link, "link", "discord") is None + assert oauth._verify_state(login, "login", "discord") is None + + +def test_state_rejects_tampering_and_expiry(monkeypatch): + s = oauth._sign_state("u", "link", "google") + payload, mac = s.split(".", 1) + assert oauth._verify_state(f"{payload}.{'0' * len(mac)}", "link", "google") is None + assert oauth._verify_state(payload, "link", "google") is None + assert oauth._verify_state("garbage", "link", "google") is None + monkeypatch.setattr(oauth, "_STATE_TTL", -1) + assert oauth._verify_state(oauth._sign_state("u", "link", "google"), + "link", "google") is None + + +def test_state_is_secret_dependent(monkeypatch): + s = oauth._sign_state("u", "link", "google") + monkeypatch.setattr(oauth, "SECRET", oauth.SECRET + "-different") + assert oauth._verify_state(s, "link", "google") is None + + +def test_a_login_state_cannot_link_the_signed_in_account(prov, monkeypatch): + """Forced-linking guard. A login state carries no user id, so it is replayable + against whoever is signed in; if the callback treated it as a link, an attacker + could attach their own identity to a victim's account and then sign in as them. + It must complete as a plain login instead.""" + attacker = f"attacker-{prov.name}@example.com" + _mock(monkeypatch, prov.profile(f"{prov.name}-cross", attacker, True)) + c = TestClient(appmod.app) + _login(c, f"crossover-{prov.name}@example.com") + r = _cb(c, prov, oauth._sign_state(_me(c)["id"], "login", prov.name)) + assert "oauth=created" in r.headers["location"] + assert _me(c)["email"] == attacker + # The account that had been signed in is untouched. + victim = TestClient(appmod.app) + _login(victim, f"crossover-{prov.name}@example.com") + assert _me(victim)["oauth_providers"] == [] + + +# --- login resolves an account ------------------------------------------------ + +def test_login_creates_an_account_and_signs_in(prov, monkeypatch): + # Mixed case on purpose: the address must be normalised before it is matched or + # stored, or the same person gets two accounts depending on how they typed it. + _mock(monkeypatch, prov.profile(f"{prov.name}-new", + f"New.User.{prov.name}@Example.com", True)) + c = TestClient(appmod.app) + r = _cb(c, prov, oauth._sign_state(None, "login", prov.name)) + assert r.status_code == 303 and "oauth=created" in r.headers["location"] + me = _me(c) + assert me["email"] == f"new.user.{prov.name}@example.com" # normalised + assert me["is_verified"] is True # the provider vouched + assert me["oauth_only"] is True + assert me["oauth_providers"] == [prov.name] + + +def test_login_recognises_a_previously_linked_account(prov, monkeypatch): + """The subject is the durable key: it resolves the account with no email, which + is what keeps a login working after someone changes their address upstream.""" + email = f"link-me-{prov.name}@example.com" + _mock(monkeypatch, prov.profile(f"{prov.name}-ret", email, True)) + linker = TestClient(appmod.app) + _login(linker, email) + uid = _me(linker)["id"] + assert "oauth=linked" in _cb( + linker, prov, oauth._sign_state(uid, "link", prov.name)).headers["location"] + # A fresh client, no session, and the provider now returns no email at all. + _mock(monkeypatch, prov.profile(f"{prov.name}-ret", "", False)) + c = TestClient(appmod.app) + r = _cb(c, prov, oauth._sign_state(None, "login", prov.name)) + assert r.status_code == 303 and "oauth=signedin" in r.headers["location"] + assert _me(c)["id"] == uid + + +def test_login_connects_a_verified_email_to_an_existing_account(prov, monkeypatch): + """The 'connect the account' path: an existing password account picks up the + identity and is signed in.""" + email = f"existing-{prov.name}@example.com" + c = TestClient(appmod.app) + _register(c, email) + _mock(monkeypatch, prov.profile(f"{prov.name}-exist", email, True)) + fresh = TestClient(appmod.app) + r = _cb(fresh, prov, oauth._sign_state(None, "login", prov.name)) + assert r.status_code == 303 and "oauth=signedin" in r.headers["location"] + me = _me(fresh) + assert me["email"] == email and me["oauth_providers"] == [prov.name] + # It kept its password, so it is not OAuth-only and may unlink. + assert me["oauth_only"] is False + + +def test_login_refuses_an_unverified_email(prov, monkeypatch): + """The provider's verified flag is the only evidence the person owns the + address. Without it, this would hand over any account whose email was typed in.""" + email = f"victim-{prov.name}@example.com" + c = TestClient(appmod.app) + _register(c, email) + _mock(monkeypatch, prov.profile(f"{prov.name}-attacker", email, False)) + fresh = TestClient(appmod.app) + r = _cb(fresh, prov, oauth._sign_state(None, "login", prov.name)) + assert r.status_code == 303 and "oauth=unverified" in r.headers["location"] + assert fresh.get(f"{V2}/users/me").status_code == 401 # no session issued + _login(c, email) + assert _me(c)["oauth_providers"] == [] # target untouched + + +def test_login_without_an_email_cannot_create_an_account(prov, monkeypatch): + _mock(monkeypatch, prov.profile(f"{prov.name}-noemail", "", True)) + c = TestClient(appmod.app) + r = _cb(c, prov, oauth._sign_state(None, "login", prov.name)) + assert r.status_code == 303 and "oauth=noemail" in r.headers["location"] + assert c.get(f"{V2}/users/me").status_code == 401 + + +def test_login_will_not_move_an_account_between_identities(prov, monkeypatch): + email = f"owned-{prov.name}@example.com" + owner = TestClient(appmod.app) + _login(owner, email) + uid = _me(owner)["id"] + _mock(monkeypatch, prov.profile(f"{prov.name}-owner", email, True)) + assert "oauth=linked" in _cb( + owner, prov, oauth._sign_state(uid, "link", prov.name)).headers["location"] + # A second provider account claiming the same address. + _mock(monkeypatch, prov.profile(f"{prov.name}-interloper", email, True)) + fresh = TestClient(appmod.app) + r = _cb(fresh, prov, oauth._sign_state(None, "login", prov.name)) + assert r.status_code == 303 and "oauth=mismatch" in r.headers["location"] + assert fresh.get(f"{V2}/users/me").status_code == 401 + + +def test_linking_an_identity_someone_else_owns_is_refused(prov, monkeypatch): + first = TestClient(appmod.app) + _login(first, f"first-owner-{prov.name}@example.com") + _mock(monkeypatch, prov.profile(f"{prov.name}-contested", + f"first-owner-{prov.name}@example.com", True)) + assert "oauth=linked" in _cb( + first, prov, + oauth._sign_state(_me(first)["id"], "link", prov.name)).headers["location"] + second = TestClient(appmod.app) + _login(second, f"second-owner-{prov.name}@example.com") + r = _cb(second, prov, oauth._sign_state(_me(second)["id"], "link", prov.name)) + assert r.status_code == 303 and "oauth=taken" in r.headers["location"] + assert _me(second)["oauth_providers"] == [] + + +def test_relinking_the_same_identity_is_idempotent(prov, monkeypatch): + email = f"relink-{prov.name}@example.com" + c = TestClient(appmod.app) + _login(c, email) + _mock(monkeypatch, prov.profile(f"{prov.name}-relink", email, True)) + uid = _me(c)["id"] + for _ in range(2): + r = _cb(c, prov, oauth._sign_state(uid, "link", prov.name)) + assert "oauth=linked" in r.headers["location"] + assert _me(c)["oauth_providers"] == [prov.name] # not duplicated + + +# --- unlink guards ------------------------------------------------------------ + +def test_last_provider_cannot_be_unlinked_into_a_lockout(prov, monkeypatch): + _mock(monkeypatch, prov.profile(f"{prov.name}-onlyway", + f"onlyway-{prov.name}@example.com", True)) + c = TestClient(appmod.app) + assert "oauth=created" in _cb( + c, prov, oauth._sign_state(None, "login", prov.name)).headers["location"] + r = c.post(f"{V2}/oauth/{prov.name}/unlink") + assert r.status_code == 409 and "no password" in r.json()["detail"] + assert _me(c)["oauth_providers"] == [prov.name] + + +def test_an_oauth_only_account_may_unlink_once_a_second_provider_is_linked(monkeypatch): + """The lockout rule is about the *last* credential, not about Discord. With two + providers linked, dropping one still leaves a way in.""" + _mock(monkeypatch, {"sub": "g-two", "email": "two@example.com", + "email_verified": True}) + c = TestClient(appmod.app) + assert "oauth=created" in _cb( + c, PROV_GOOGLE, oauth._sign_state(None, "login", "google")).headers["location"] + uid = _me(c)["id"] + # Now connect Discord to the same, still password-less, account. + _mock(monkeypatch, {"id": "d-two", "email": "two@example.com", "verified": True}) + assert "oauth=linked" in _cb( + c, PROV_DISCORD, oauth._sign_state(uid, "link", "discord")).headers["location"] + assert _me(c)["oauth_providers"] == ["discord", "google"] + # Either one may now go. + assert c.post(f"{V2}/oauth/google/unlink").status_code == 204 + assert _me(c)["oauth_providers"] == ["discord"] + # But the survivor is once again the only way in. + assert c.post(f"{V2}/oauth/discord/unlink").status_code == 409 + + +def test_unlink_requires_auth(prov): + c = TestClient(appmod.app) + assert c.post(f"{V2}/oauth/{prov.name}/unlink").status_code == 401 + + +def test_link_start_requires_auth(prov): + c = TestClient(appmod.app) + assert c.get(f"{V2}/oauth/{prov.name}/link/start", + follow_redirects=False).status_code == 401 + + +# Bare provider handles for the cross-provider test above, which needs both at once. +class _P: + def __init__(self, name, callback): + self.name, self.callback = name, callback + + +PROV_GOOGLE = _P("google", f"{V2}/oauth/google/callback") +PROV_DISCORD = _P("discord", f"{V2}/discord/link/callback") diff --git a/backend/tests/notifications/test_discord_dm.py b/backend/tests/notifications/test_discord_dm.py index aa9cc56..e0fc6d0 100644 --- a/backend/tests/notifications/test_discord_dm.py +++ b/backend/tests/notifications/test_discord_dm.py @@ -7,8 +7,8 @@ from fastapi.testclient import TestClient from web import app as appmod from accounts import db +from accounts import oauth from notifications import discord -from notifications import discord_link as dl from notifications import notify V2 = "/thermograph/api/v2" @@ -161,14 +161,15 @@ class _MockClient: def test_linking_opts_in_and_toggle_flips(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "app") - monkeypatch.setattr(dl, "CLIENT_SECRET", "sec") - monkeypatch.setattr(dl.httpx, "AsyncClient", _MockClient) + monkeypatch.setitem(oauth.CREDENTIALS, "discord", ("app", "sec")) + monkeypatch.setattr(oauth.httpx, "AsyncClient", _MockClient) c = TestClient(appmod.app) _login(c, "dm-toggle@example.com") uid = c.get(f"{V2}/users/me").json()["id"] - # Link -> opted in by default. - r = c.get(f"{V2}/discord/link/callback?code=x&state={dl._sign_state(uid)}", follow_redirects=False) + # Link -> opted in by default. discord_id is a delivery address, so the link + # has to write it as well as the oauth_account row. + state = oauth._sign_state(uid, "link", "discord") + r = c.get(f"{V2}/discord/link/callback?code=x&state={state}", follow_redirects=False) assert r.status_code == 303 me = c.get(f"{V2}/users/me").json() assert me["discord_id"] == "discord-777" and me["discord_dm"] is True diff --git a/backend/tests/notifications/test_discord_link.py b/backend/tests/notifications/test_discord_link.py index ffaade0..fdf56c8 100644 --- a/backend/tests/notifications/test_discord_link.py +++ b/backend/tests/notifications/test_discord_link.py @@ -1,11 +1,20 @@ -"""Discord account linking: signed-state integrity, OAuth callback (Discord HTTP -mocked), unlink, and auth gating. Reuses the throwaway accounts DB from conftest.""" +"""The legacy /discord/* routes. + +These are not duplicate coverage of tests/accounts/test_oauth.py — they pin the two +compatibility promises that outlive the refactor: + +* **/discord/link/callback is registered in Discord's developer portal.** If it + stops answering, every Discord login breaks until an operator edits the portal. +* The other /discord/* paths are what the deployed frontend calls, and frontend and + backend deploy independently, so a newer backend must keep serving an older one. + +The flow itself lives in accounts/oauth.py and is tested there. +""" import pytest from fastapi.testclient import TestClient from web import app as appmod -from accounts import db -from notifications import discord_link as dl +from accounts import db, oauth V2 = "/thermograph/api/v2" PW = "supersecret123" @@ -16,72 +25,10 @@ def _tables(): db.Base.metadata.create_all(db.sync_engine) -def _login(client, email): - r = client.post(f"{V2}/auth/register", json={"email": email, "password": PW}) - assert r.status_code in (201, 400) - assert client.post(f"{V2}/auth/login", data={"username": email, "password": PW}).status_code == 204 - - -# --- signed state ------------------------------------------------------------ - -def test_state_round_trips_and_rejects_tampering(): - s = dl._sign_state("user-123") - assert dl._verify_state(s) == "user-123" - payload, mac = s.split(".", 1) - assert dl._verify_state(f"{payload}.{'0' * len(mac)}") is None # bad mac - assert dl._verify_state(payload) is None # no mac - assert dl._verify_state("garbage") is None - - -def test_state_expires(monkeypatch): - monkeypatch.setattr(dl, "_STATE_TTL", -1) # already expired - assert dl._verify_state(dl._sign_state("u")) is None - - -def test_state_is_secret_dependent(monkeypatch): - s = dl._sign_state("u") - monkeypatch.setattr(dl, "SECRET", dl.SECRET + "-different") - assert dl._verify_state(s) is None # signed under the old secret - - -# --- routes ------------------------------------------------------------------ - -def test_config_reports_enabled_state(monkeypatch): - c = TestClient(appmod.app) - monkeypatch.setattr(dl, "CLIENT_ID", "") - monkeypatch.setattr(dl, "CLIENT_SECRET", "") - r = c.get(f"{V2}/discord/config") # no auth required - assert r.status_code == 200 and r.json() == {"enabled": False} - monkeypatch.setattr(dl, "CLIENT_ID", "app123") - monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") - assert c.get(f"{V2}/discord/config").json() == {"enabled": True} - - -def test_link_requires_auth(): - c = TestClient(appmod.app) - assert c.get(f"{V2}/discord/link/start", follow_redirects=False).status_code == 401 - assert c.post(f"{V2}/discord/unlink").status_code == 401 - - -def test_start_redirects_to_discord(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "app123") - monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") - c = TestClient(appmod.app) - _login(c, "start@example.com") - r = c.get(f"{V2}/discord/link/start", follow_redirects=False) - assert r.status_code == 303 - loc = r.headers["location"] - assert loc.startswith("https://discord.com/api/oauth2/authorize") - assert "client_id=app123" in loc and "scope=identify" in loc and "state=" in loc - - -def test_start_when_unconfigured_bounces_back(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "") - monkeypatch.setattr(dl, "CLIENT_SECRET", "") - c = TestClient(appmod.app) - _login(c, "noconf@example.com") - r = c.get(f"{V2}/discord/link/start", follow_redirects=False) - assert r.status_code == 303 and "discord=unavailable" in r.headers["location"] +@pytest.fixture(autouse=True) +def _configured(monkeypatch): + monkeypatch.setitem(oauth.CREDENTIALS, "discord", ("app123", "secret456")) + monkeypatch.setattr(oauth.httpx, "AsyncClient", _MockClient) class _Resp: @@ -90,47 +37,88 @@ class _Resp: class _MockClient: - """Stands in for httpx.AsyncClient: token exchange then /users/@me.""" def __init__(self, *a, **k): pass async def __aenter__(self): return self async def __aexit__(self, *a): return False async def post(self, url, **k): return _Resp(200, {"access_token": "tok"}) - async def get(self, url, **k): return _Resp(200, {"id": "discord-999", "username": "someone"}) + async def get(self, url, **k): + return _Resp(200, {"id": "discord-999", "email": "legacy@example.com", + "verified": True}) -def test_callback_links_the_account(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "app123") - monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") - monkeypatch.setattr(dl.httpx, "AsyncClient", _MockClient) +def _login(client, email): + assert client.post(f"{V2}/auth/register", + json={"email": email, "password": PW}).status_code in (201, 400) + assert client.post(f"{V2}/auth/login", + data={"username": email, "password": PW}).status_code == 204 + + +def test_legacy_config_still_reports_enabled_state(monkeypatch): c = TestClient(appmod.app) - _login(c, "callback@example.com") + monkeypatch.setitem(oauth.CREDENTIALS, "discord", ("", "")) + assert c.get(f"{V2}/discord/config").json() == {"enabled": False} + monkeypatch.setitem(oauth.CREDENTIALS, "discord", ("app123", "secret456")) + assert c.get(f"{V2}/discord/config").json() == {"enabled": True} + + +def test_legacy_link_start_still_requires_auth(): + c = TestClient(appmod.app) + assert c.get(f"{V2}/discord/link/start", follow_redirects=False).status_code == 401 + assert c.post(f"{V2}/discord/unlink").status_code == 401 + + +def test_legacy_start_redirects_to_discord(): + c = TestClient(appmod.app) + _login(c, "legacy-start@example.com") + r = c.get(f"{V2}/discord/link/start", follow_redirects=False) + assert r.status_code == 303 + loc = r.headers["location"] + assert loc.startswith("https://discord.com/api/oauth2/authorize") + assert "client_id=app123" in loc and "scope=identify" in loc and "state=" in loc + # The redirect_uri it asks Discord to call back on must be the registered one. + assert "discord%2Flink%2Fcallback" in loc + + +def test_the_registered_callback_url_still_completes_a_link(): + c = TestClient(appmod.app) + _login(c, "legacy-cb@example.com") uid = c.get(f"{V2}/users/me").json()["id"] - state = dl._sign_state(uid) - r = c.get(f"{V2}/discord/link/callback?code=abc&state={state}", follow_redirects=False) - assert r.status_code == 303 and "discord=linked" in r.headers["location"] - # /users/me now reports the linked id, and unlink clears it. - assert c.get(f"{V2}/users/me").json()["discord_id"] == "discord-999" - assert c.post(f"{V2}/discord/unlink").status_code == 204 - assert c.get(f"{V2}/users/me").json()["discord_id"] is None - - -def test_callback_rejects_a_forged_state(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "app123") - monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") - monkeypatch.setattr(dl.httpx, "AsyncClient", _MockClient) - c = TestClient(appmod.app) - _login(c, "forged@example.com") - # State signed for a different user id must not link this session. - r = c.get(f"{V2}/discord/link/callback?code=abc&state={dl._sign_state('someone-else')}", + state = oauth._sign_state(uid, "link", "discord") + r = c.get(f"{V2}/discord/link/callback?code=abc&state={state}", follow_redirects=False) - assert r.status_code == 303 and "discord=error" in r.headers["location"] - assert c.get(f"{V2}/users/me").json()["discord_id"] is None + assert r.status_code == 303 and "discord=linked" in r.headers["location"] + me = c.get(f"{V2}/users/me").json() + assert me["discord_id"] == "discord-999" # the DM delivery address + assert me["oauth_providers"] == ["discord"] # and the identity row + # Legacy unlink clears both. + assert c.post(f"{V2}/discord/unlink").status_code == 204 + me = c.get(f"{V2}/users/me").json() + assert me["discord_id"] is None and me["oauth_providers"] == [] -def test_callback_without_code_is_cancelled(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "app123") - monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") +def test_the_callback_still_emits_the_discord_query_param(): + """The deployed frontend reads ?discord=, not ?oauth=. Dropping it would + silently stop it showing any outcome at all.""" c = TestClient(appmod.app) - _login(c, "cancel@example.com") r = c.get(f"{V2}/discord/link/callback?error=access_denied", follow_redirects=False) - assert r.status_code == 303 and "discord=cancelled" in r.headers["location"] + loc = r.headers["location"] + assert "discord=cancelled" in loc and "oauth=cancelled" in loc + + +def test_users_me_still_carries_the_deprecated_discord_only_field(monkeypatch): + """Renamed to oauth_only, but the deployed frontend still reads the old name.""" + monkeypatch.setattr(oauth.httpx, "AsyncClient", _MockClient) + c = TestClient(appmod.app) + r = c.get(f"{V2}/discord/link/callback?code=abc" + f"&state={oauth._sign_state(None, 'login', 'discord')}", + follow_redirects=False) + assert "discord=created" in r.headers["location"] + me = c.get(f"{V2}/users/me").json() + assert me["oauth_only"] is True and me["discord_only"] is True + + +def test_legacy_login_start_is_still_mounted(): + c = TestClient(appmod.app) + r = c.get(f"{V2}/discord/login/start", follow_redirects=False) + assert r.status_code == 303 + assert r.headers["location"].startswith("https://discord.com/api/oauth2/authorize") diff --git a/backend/tests/notifications/test_discord_login.py b/backend/tests/notifications/test_discord_login.py deleted file mode 100644 index cc3d4f9..0000000 --- a/backend/tests/notifications/test_discord_login.py +++ /dev/null @@ -1,251 +0,0 @@ -"""Sign in with Discord: account resolution, session issuance, and the guards that -keep the login flow from becoming an account-takeover path. Discord HTTP is mocked; -reuses the throwaway accounts DB from conftest.""" -import pytest -from fastapi.testclient import TestClient - -from web import app as appmod -from accounts import db -from notifications import discord_link as dl - -V2 = "/thermograph/api/v2" -PW = "supersecret123" - - -@pytest.fixture(scope="module", autouse=True) -def _tables(): - db.Base.metadata.create_all(db.sync_engine) - - -@pytest.fixture(autouse=True) -def _configured(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "app123") - monkeypatch.setattr(dl, "CLIENT_SECRET", "secret456") - - -class _Resp: - def __init__(self, code, data): self.status_code = code; self._data = data - def json(self): return self._data - - -def _mock_discord(monkeypatch, profile): - """Stand in for httpx.AsyncClient: token exchange, then /users/@me -> profile.""" - class _Client: - def __init__(self, *a, **k): pass - async def __aenter__(self): return self - async def __aexit__(self, *a): return False - async def post(self, url, **k): return _Resp(200, {"access_token": "tok"}) - async def get(self, url, **k): return _Resp(200, profile) - monkeypatch.setattr(dl.httpx, "AsyncClient", _Client) - - -def _register(client, email): - r = client.post(f"{V2}/auth/register", json={"email": email, "password": PW}) - assert r.status_code in (201, 400) - - -def _login(client, email): - _register(client, email) - assert client.post(f"{V2}/auth/login", - data={"username": email, "password": PW}).status_code == 204 - - -def _callback(client, state): - return client.get(f"{V2}/discord/link/callback?code=abc&state={state}", - follow_redirects=False) - - -# --- start ------------------------------------------------------------------- - -def test_login_start_needs_no_session_and_asks_for_email(): - c = TestClient(appmod.app) - r = c.get(f"{V2}/discord/login/start", follow_redirects=False) - assert r.status_code == 303 - loc = r.headers["location"] - assert loc.startswith("https://discord.com/api/oauth2/authorize") - # The email scope is what makes account resolution possible at all. - assert "scope=identify+email" in loc and "state=" in loc - - -def test_login_start_while_signed_in_links_instead_of_switching_account(): - """Someone already signed in wants to connect Discord, not be switched into - whichever account the Discord identity happens to resolve to.""" - c = TestClient(appmod.app) - _login(c, "already-in@example.com") - r = c.get(f"{V2}/discord/login/start", follow_redirects=False) - assert r.status_code == 303 - loc = r.headers["location"] - assert "scope=identify&" in loc or loc.endswith("scope=identify") - state = loc.split("state=")[1].split("&")[0] - uid = c.get(f"{V2}/users/me").json()["id"] - assert dl._verify_state(state, "link") == uid - - -def test_login_start_when_unconfigured_bounces_back(monkeypatch): - monkeypatch.setattr(dl, "CLIENT_ID", "") - monkeypatch.setattr(dl, "CLIENT_SECRET", "") - c = TestClient(appmod.app) - r = c.get(f"{V2}/discord/login/start", follow_redirects=False) - assert r.status_code == 303 and "discord=unavailable" in r.headers["location"] - - -# --- state purposes are not interchangeable ---------------------------------- - -def test_state_purposes_do_not_cross_over(): - login_state = dl._sign_state(None, "login") - link_state = dl._sign_state("user-123", "link") - # A login state carries no user id, but verifies as "" — distinct from reject. - assert dl._verify_state(login_state, "login") == "" - assert dl._verify_state(login_state, "link") is None - assert dl._verify_state(link_state, "link") == "user-123" - assert dl._verify_state(link_state, "login") is None - - -def test_a_login_state_cannot_link_the_signed_in_account(monkeypatch): - """Forced-linking guard. A login state carries no user id, so it is replayable - against whoever is signed in; if the callback treated it as a link, an attacker - could attach their own Discord identity to a victim's account and then sign in - as them. It must complete as a plain login instead.""" - _mock_discord(monkeypatch, {"id": "d-crossover", "email": "attacker@example.com", - "verified": True}) - c = TestClient(appmod.app) - _login(c, "crossover@example.com") - # A state whose purpose says login, but naming the signed-in user — the shape - # a forced-link attempt would take. - r = _callback(c, dl._sign_state(c.get(f"{V2}/users/me").json()["id"], "login")) - # Resolved as a login into the Discord identity's own account, not as a link. - assert "discord=created" in r.headers["location"] - assert c.get(f"{V2}/users/me").json()["email"] == "attacker@example.com" - # The account that had been signed in is untouched. - victim = TestClient(appmod.app) - _login(victim, "crossover@example.com") - assert victim.get(f"{V2}/users/me").json()["discord_id"] is None - - -# --- login resolves an account ----------------------------------------------- - -def test_login_creates_an_account_and_signs_in(monkeypatch): - _mock_discord(monkeypatch, {"id": "d-new", "email": "New.User@Example.com", - "verified": True}) - c = TestClient(appmod.app) - r = _callback(c, dl._sign_state(None, "login")) - assert r.status_code == 303 and "discord=created" in r.headers["location"] - me = c.get(f"{V2}/users/me") - assert me.status_code == 200 - body = me.json() - assert body["email"] == "new.user@example.com" # normalised - assert body["discord_id"] == "d-new" - assert body["is_verified"] is True # Discord vouched for it - assert body["discord_only"] is True - assert body["discord_dm"] is True - - -def test_login_recognises_a_previously_linked_account(monkeypatch): - """The discord_id is the durable key: it resolves the account with no email.""" - _mock_discord(monkeypatch, {"id": "d-returning", "email": "link-me@example.com", - "verified": True}) - linker = TestClient(appmod.app) - _login(linker, "link-me@example.com") - uid = linker.get(f"{V2}/users/me").json()["id"] - assert "discord=linked" in _callback(linker, dl._sign_state(uid, "link") - ).headers["location"] - # A fresh client, no session, and Discord now returns no email at all. - _mock_discord(monkeypatch, {"id": "d-returning"}) - c = TestClient(appmod.app) - r = _callback(c, dl._sign_state(None, "login")) - assert r.status_code == 303 and "discord=signedin" in r.headers["location"] - assert c.get(f"{V2}/users/me").json()["id"] == uid - - -def test_login_connects_a_verified_email_to_an_existing_account(monkeypatch): - """This is the 'connect the account with that' path: an existing password - account picks up the Discord id and is signed in.""" - c = TestClient(appmod.app) - _register(c, "existing@example.com") - _mock_discord(monkeypatch, {"id": "d-existing", "email": "existing@example.com", - "verified": True}) - fresh = TestClient(appmod.app) - r = _callback(fresh, dl._sign_state(None, "login")) - assert r.status_code == 303 and "discord=signedin" in r.headers["location"] - body = fresh.get(f"{V2}/users/me").json() - assert body["email"] == "existing@example.com" - assert body["discord_id"] == "d-existing" - # It kept its password, so it is not Discord-only and may unlink. - assert body["discord_only"] is False - - -def test_login_refuses_an_unverified_discord_email(monkeypatch): - """Discord's verified flag is the only evidence the person owns the address. - Without it, this would hand over any account whose email someone typed in.""" - c = TestClient(appmod.app) - _register(c, "victim@example.com") - _mock_discord(monkeypatch, {"id": "d-attacker", "email": "victim@example.com", - "verified": False}) - fresh = TestClient(appmod.app) - r = _callback(fresh, dl._sign_state(None, "login")) - assert r.status_code == 303 and "discord=unverified" in r.headers["location"] - assert fresh.get(f"{V2}/users/me").status_code == 401 # no session issued - # And the targeted account is untouched. - _login(c, "victim@example.com") - assert c.get(f"{V2}/users/me").json()["discord_id"] is None - - -def test_login_without_an_email_cannot_create_an_account(monkeypatch): - _mock_discord(monkeypatch, {"id": "d-noemail"}) - c = TestClient(appmod.app) - r = _callback(c, dl._sign_state(None, "login")) - assert r.status_code == 303 and "discord=noemail" in r.headers["location"] - assert c.get(f"{V2}/users/me").status_code == 401 - - -def test_login_will_not_move_an_account_between_discord_identities(monkeypatch): - """The email matches an account that is already linked to a different Discord - id — signing in would silently re-point it.""" - owner = TestClient(appmod.app) - _login(owner, "owned@example.com") - uid = owner.get(f"{V2}/users/me").json()["id"] - _mock_discord(monkeypatch, {"id": "d-owner", "email": "owned@example.com", - "verified": True}) - assert "discord=linked" in _callback(owner, dl._sign_state(uid, "link") - ).headers["location"] - # A second Discord account claiming the same address. - _mock_discord(monkeypatch, {"id": "d-interloper", "email": "owned@example.com", - "verified": True}) - fresh = TestClient(appmod.app) - r = _callback(fresh, dl._sign_state(None, "login")) - assert r.status_code == 303 and "discord=mismatch" in r.headers["location"] - assert fresh.get(f"{V2}/users/me").status_code == 401 - assert owner.get(f"{V2}/users/me").json()["discord_id"] == "d-owner" - - -# --- guards on the connected account ----------------------------------------- - -def test_linking_a_discord_account_someone_else_owns_is_refused(monkeypatch): - first = TestClient(appmod.app) - _login(first, "first-owner@example.com") - _mock_discord(monkeypatch, {"id": "d-contested", "email": "first-owner@example.com", - "verified": True}) - uid = first.get(f"{V2}/users/me").json()["id"] - assert "discord=linked" in _callback(first, dl._sign_state(uid, "link") - ).headers["location"] - # A different Thermograph account tries to claim the same Discord identity. - second = TestClient(appmod.app) - _login(second, "second-owner@example.com") - uid2 = second.get(f"{V2}/users/me").json()["id"] - r = _callback(second, dl._sign_state(uid2, "link")) - assert r.status_code == 303 and "discord=taken" in r.headers["location"] - assert second.get(f"{V2}/users/me").json()["discord_id"] is None - - -def test_a_discord_only_account_cannot_unlink_itself_into_a_lockout(monkeypatch): - _mock_discord(monkeypatch, {"id": "d-onlyway", "email": "onlyway@example.com", - "verified": True}) - c = TestClient(appmod.app) - assert "discord=created" in _callback(c, dl._sign_state(None, "login") - ).headers["location"] - r = c.post(f"{V2}/discord/unlink") - assert r.status_code == 409 - assert "no password" in r.json()["detail"] - assert c.get(f"{V2}/users/me").json()["discord_id"] == "d-onlyway" - # Muting DMs is still fine — that is not a lockout. - assert c.post(f"{V2}/discord/dm", json={"enabled": False}).status_code == 204 diff --git a/backend/web/app.py b/backend/web/app.py index cbec4ae..dd29000 100644 --- a/backend/web/app.py +++ b/backend/web/app.py @@ -25,7 +25,7 @@ from data import climate from notifications import digest from notifications import discord_interactions as discord_interactions_mod from notifications import discord_link -from accounts import db +from accounts import db, oauth from data import grid from core import metrics from notifications import notify @@ -988,7 +988,12 @@ app.include_router( ) # Subscriptions + notifications (authed, user-scoped). app.include_router(api_accounts.router, prefix=f"{BASE}/api/v2") -# Discord account linking (OAuth2, authed). +# Sign in with / connect an external provider (Discord, Google). +app.include_router(oauth.router, prefix=f"{BASE}/api/v2") +# Discord's legacy /discord/* paths: the callback URL registered in Discord's +# portal, plus what the currently-deployed frontend calls. Mounted AFTER the +# generic router so neither shadows the other — the paths are disjoint, and +# /oauth/{provider}/... would otherwise be a candidate match for /discord/.... app.include_router(discord_link.router, prefix=f"{BASE}/api/v2") diff --git a/frontend/static/account.js b/frontend/static/account.js index f3eb0c6..8b23790 100644 --- a/frontend/static/account.js +++ b/frontend/static/account.js @@ -10,8 +10,8 @@ // units.js is (imported by each page's entry module). let currentUser = null; // {id, email, display_name} or null -let discordEnabled = false; // is Discord configured on the server? -let discordChecked = false; // have we asked yet? (once per page load) +let providers = []; // [{name, label, enabled}] from /oauth/config +let providersChecked = false; // have we asked yet? (once per page load) const authCbs = []; // notified on login/logout so pages can re-gate // backend's own origin+base (e.g. "https://thermograph.org/thermograph", or @@ -92,16 +92,24 @@ async function refreshUser() { // Learn once whether Discord is configured, so we only offer "Link Discord" and // "Continue with Discord" when they'll actually work. Asked regardless of sign-in // state: the sign-in modal needs the answer precisely when nobody is signed in. - if (!discordChecked) { - discordChecked = true; + if (!providersChecked) { + providersChecked = true; try { - const r = await apiFetch(uv("discord/config")); - if (r.ok) discordEnabled = (await r.json()).enabled === true; - } catch (e) { /* leave it hidden */ } + const r = await apiFetch(uv("oauth/config")); + if (r.ok) providers = ((await r.json()).providers || []).filter((p) => p.enabled); + } catch (e) { /* leave them hidden */ } } return currentUser; } +// Providers the server can complete a flow with. Empty until refreshUser() has +// run, which is why the sign-in modal re-renders its provider row on every open +// rather than only when it is first built. +function enabledProviders() { return providers; } +function isLinked(name) { + return !!(currentUser && (currentUser.oauth_providers || []).includes(name)); +} + // --- auth calls -------------------------------------------------------------- async function login(email, password) { // fastapi-users login is an OAuth2 form: username=email, password. @@ -130,6 +138,23 @@ let modal = null, mode = "login"; // extra request and inherits currentColor. const DISCORD_IC = ``; +// Google's mark is four fixed brand colours, so unlike the others it does not take +// currentColor — hence the white button beneath it, which is also what Google's +// branding terms require. +const GOOGLE_IC = ``; + +const PROVIDER_IC = { discord: DISCORD_IC, google: GOOGLE_IC }; + +// The provider buttons under the sign-in form. Rebuilt on every open because +// `providers` is populated asynchronously and may still have been empty when the +// modal was first constructed. +function providerButtonsHtml() { + return enabledProviders().map((p) => ` + `).join(""); +} + function buildModal() { modal = document.createElement("div"); modal.className = "mp-overlay acct-overlay"; @@ -151,9 +176,7 @@ function buildModal() {

          @@ -183,9 +206,11 @@ function setMode(m) { modal.querySelector(".acct-switch").innerHTML = isLogin ? 'Need an account? ' : 'Already have an account? '; - // One label for both modes: Discord signs you in or creates the account, + // One label for both modes: the provider signs you in or creates the account, // whichever applies, so making the user pick first would be a false choice. - modal.querySelector(".acct-alt").hidden = !discordEnabled; + const enabled = enabledProviders(); + modal.querySelector(".acct-oauth-row").innerHTML = providerButtonsHtml(); + modal.querySelector(".acct-alt").hidden = enabled.length === 0; showError(""); } @@ -341,6 +366,30 @@ const USER_IC = ` { + if (!isLinked(p.name)) { + return `` + + `Link ${escapeHtml(p.label)}`; + } + const dm = p.name === "discord" + ? `` + : ""; + // With no password and nothing else linked, this is the only way back in and + // the server refuses to unlink it — so don't offer a button that can't work. + const stuck = currentUser.oauth_only && linkedCount <= 1; + return dm + (stuck + ? `Signed in with ${escapeHtml(p.label)}` + : ``); + }).join(""); +} + function ensureAcctEl() { const brand = document.querySelector(".brand"); if (!brand) return null; @@ -382,14 +431,7 @@ function renderHeader() { `; @@ -417,22 +459,25 @@ function renderHeader() { } }); el.querySelector(".acct-signout").addEventListener("click", logout); - const unlinkBtn = el.querySelector(".acct-discord-unlink"); - if (unlinkBtn) unlinkBtn.addEventListener("click", async () => { - unlinkBtn.disabled = true; - try { - const res = await apiFetch(uv("discord/unlink"), { method: "POST" }); - // 409: the account was created with Discord and has no password, so - // unlinking would lock it out. Say why instead of appearing to do nothing. - if (res.status === 409) { - const body = await res.json().catch(() => null); - showToast((body && body.detail) || "Discord can't be unlinked from this account.", true); - unlinkBtn.disabled = false; - return; - } - } catch (e) {} - await refreshUser(); - emitAuth(); // repaint the popover in its unlinked state + el.querySelectorAll(".acct-oauth-unlink").forEach((unlinkBtn) => { + unlinkBtn.addEventListener("click", async () => { + const provider = unlinkBtn.dataset.provider; + unlinkBtn.disabled = true; + try { + const res = await apiFetch(uv(`oauth/${provider}/unlink`), { method: "POST" }); + // 409: the account has no password and this was its last provider, so + // unlinking would lock it out. Say why instead of appearing to do nothing. + if (res.status === 409) { + const body = await res.json().catch(() => null); + showToast((body && body.detail) + || "That account can't be unlinked — it's the only way to sign in.", true); + unlinkBtn.disabled = false; + return; + } + } catch (e) {} + await refreshUser(); + emitAuth(); // repaint the popover in its unlinked state + }); }); const dmBtn = el.querySelector(".acct-discord-dm"); if (dmBtn) dmBtn.addEventListener("click", async () => { @@ -479,28 +524,41 @@ function showToast(msg, isError = false) { toastTimer = setTimeout(() => { el.hidden = true; }, 6000); } -// --- Discord OAuth outcomes -------------------------------------------------- +// --- OAuth outcomes ---------------------------------------------------------- // The link/login callback can only talk back to us through a redirect, so it -// lands on ?discord=. Each status maps to [message, isError]; anything -// unrecognised is treated as a generic failure rather than shown raw. -const DISCORD_STATUS = { - linked: ["Discord connected.", false], - signedin: ["Signed in with Discord.", false], - created: ["Account created with Discord — welcome to Thermograph.", false], - cancelled: ["Discord sign-in cancelled.", false], - taken: ["That Discord account is already connected to another Thermograph account.", true], - mismatch: ["That email belongs to an account connected to a different Discord account. " - + "Sign in with your password to change it.", true], - noemail: ["Discord didn't share an email address, so there's no account to sign in to.", true], - unverified: ["Verify your email address with Discord first, then try again.", true], - inactive: ["That account is deactivated.", true], - unavailable: ["Discord sign-in isn't available on this server.", true], +// lands on ?oauth=&provider=. Each status maps to a message builder +// taking the provider's display name; anything unrecognised is treated as a +// generic failure rather than shown raw. +const OAUTH_STATUS = { + linked: [(p) => `${p} connected.`, false], + signedin: [(p) => `Signed in with ${p}.`, false], + created: [(p) => `Account created with ${p} — welcome to Thermograph.`, false], + cancelled: [(p) => `${p} sign-in cancelled.`, false], + already: [(p) => `A different ${p} account is already connected here.`, true], + taken: [(p) => `That ${p} account is already connected to another Thermograph account.`, true], + mismatch: [(p) => `That email belongs to an account connected to a different ${p} ` + + "account. Sign in with your password to change it.", true], + noemail: [(p) => `${p} didn't share an email address, so there's no account to sign in to.`, true], + unverified: [(p) => `Verify your email address with ${p} first, then try again.`, true], + inactive: [() => "That account is deactivated.", true], + unavailable: [(p) => `${p} sign-in isn't available on this server.`, true], }; -function showDiscordStatus(status) { - const [msg, isError] = DISCORD_STATUS[status] - || ["Something went wrong with Discord. Please try again.", true]; - showToast(msg, isError); +function providerLabel(name) { + const known = providers.find((p) => p.name === name); + if (known) return known.label; + // The config probe may have failed, or the provider may have been switched off + // between starting the flow and coming back. Title-case the raw name rather + // than showing "undefined" in the toast. + return name ? name.charAt(0).toUpperCase() + name.slice(1) : "That provider"; +} + +function showOAuthStatus(status, provider) { + const label = providerLabel(provider); + const entry = OAUTH_STATUS[status]; + if (!entry) return showToast(`Something went wrong with ${label}. Please try again.`, true); + const [build, isError] = entry; + showToast(build(label), isError); } // --- boot -------------------------------------------------------------------- @@ -513,18 +571,20 @@ function showDiscordStatus(status) { const params = new URLSearchParams(location.search); const token = params.get("verify_token"); - // Both of these are one-shot handoffs from a redirect; strip them together so a - // reload can't replay either, and so the two never fight over the URL. - const discordStatus = params.get("discord"); - if (token || discordStatus) { - params.delete("verify_token"); - params.delete("discord"); + // `discord` is the pre-Google spelling: a backend older than this frontend sends + // only that one, so fall back to it and assume the provider it implies. + const oauthStatus = params.get("oauth") || params.get("discord"); + const oauthProvider = params.get("provider") || (params.get("discord") ? "discord" : ""); + // These are one-shot handoffs from a redirect; strip them together so a reload + // can't replay any of them, and so they never fight over the URL. + if (token || oauthStatus) { + ["verify_token", "oauth", "provider", "discord"].forEach((k) => params.delete(k)); const clean = location.pathname + (params.toString() ? `?${params}` : "") + location.hash; history.replaceState(null, "", clean); } // refreshUser() above already ran, and the callback's redirect carried the new // session cookie, so the header is painted signed-in before this toast lands. - if (discordStatus) showDiscordStatus(discordStatus); + if (oauthStatus) showOAuthStatus(oauthStatus, oauthProvider); if (token) { try { await verifyEmail(token); diff --git a/frontend/static/style.css b/frontend/static/style.css index f50112b..116c908 100644 --- a/frontend/static/style.css +++ b/frontend/static/style.css @@ -436,14 +436,26 @@ main { max-width: 1200px; margin: 0 auto; padding: 20px 24px 60px; } .acct-or::before, .acct-or::after { content: ""; flex: 1; height: 1px; background: var(--border); } -.acct-discord-login { +.acct-oauth-row { display: flex; flex-direction: column; gap: 8px; } +.acct-oauth-login { display: flex; align-items: center; justify-content: center; gap: 9px; padding: 12px 16px; min-height: 44px; border-radius: 10px; - background: var(--discord); color: #fff; font-weight: 700; font-size: 15px; - text-decoration: none; border: 1px solid transparent; + font-weight: 700; font-size: 15px; text-decoration: none; + border: 1px solid transparent; } -.acct-discord-login:hover { filter: brightness(1.08); } -.acct-discord-login svg { width: 20px; height: 15px; flex: none; } +.acct-oauth-login:hover { filter: brightness(1.08); } +.acct-oauth-login svg { width: 20px; height: 20px; flex: none; } +.acct-oauth-login.is-discord { + background: var(--discord); color: #fff; +} +.acct-oauth-login.is-discord svg { height: 15px; } +/* Google's branding terms require their mark on white (or their own grey), with a + visible border — so this one button is deliberately light in both schemes and + does not follow the surface tokens. */ +.acct-oauth-login.is-google { + background: #fff; color: #1f1f1f; border-color: #747775; +} +.acct-oauth-login.is-google:hover { filter: none; background: #f2f2f2; } .acct-pop-note { display: block; padding: 9px 12px; font-size: 13px; } .acct-switch { margin: 0; font-size: 13px; color: var(--muted); text-align: center; } diff --git a/infra/deploy/thermograph.env.example b/infra/deploy/thermograph.env.example index ef158f8..cdf6403 100644 --- a/infra/deploy/thermograph.env.example +++ b/infra/deploy/thermograph.env.example @@ -224,6 +224,18 @@ THERMOGRAPH_BASE_URL=https://thermograph.org # for per request, not registered, so the email scope needs no portal change # either. An environment without these two vars simply shows no Discord UI. #THERMOGRAPH_DISCORD_CLIENT_SECRET= +# Sign in with Google / connect a Google account — the same engine as Discord +# above (accounts/oauth.py), configured independently, so an environment may offer +# either, both, or neither. From Google Cloud Console -> APIs & Services -> +# Credentials -> OAuth 2.0 Client ID, type "Web application". The CLIENT_SECRET is +# a credential. Register this exact redirect URI on the client: +# https://thermograph.org/api/v2/oauth/google/callback +# (Google matches redirect URIs exactly — scheme, host and path — so beta and dev +# each need their own entry on the same client, or their own client.) The consent +# screen needs only the `openid` and `email` scopes, both non-sensitive, so it +# requires no Google verification review. +#THERMOGRAPH_GOOGLE_CLIENT_ID= +#THERMOGRAPH_GOOGLE_CLIENT_SECRET= # Gateway bot (opt-in): holds a live websocket so the bot replies to messages that # @mention it (or DM it) with a city grade — e.g. "@Thermograph Phoenix". Reuses # THERMOGRAPH_DISCORD_BOT_TOKEN above. Runs on the single notifier leader only, so -- 2.45.2 From 6bd07d33cb053b9be187fa6e7544dd83b7cb15cd Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Wed, 29 Jul 2026 21:35:20 -0700 Subject: [PATCH 30/37] openbao: add a dormant OpenBao backend alongside SOPS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of moving the secret store to OpenBao. Nothing is cut over: TG_SECRETS_BACKEND defaults to sops for all three environments, so SOPS remains authoritative until an environment is flipped in a reviewed PR. The reason for the migration is one specific failure class. infra/CLAUDE.md requires that vps1 never hold prod credentials, but that rule is currently enforced by a caller-side shell flag (THERMOGRAPH_SECRETS_SKIP_COMMON) at three call sites. A fourth call site, or one by-hand render, writes both S3 keypairs, the VAPID private key, REGISTRY_TOKEN and the IndexNow/metrics tokens onto the Forgejo CI-runner box and exits 0. policies/tg-host-dev.hcl makes that a 403 instead: dev's identity cannot read the common path at all. Shape: - vps2 only, native systemd (not a container: a Swarm service is circular, and a plain docker run is one `prune -a` from gone), raft storage (2.6.0 deprecated the file backend), static auto-unseal so a reboot cannot block deploys, mesh-only self-signed TLS (not ACME — that would put DNS in the boot chain of the secret store). - AppRole per environment, CIDR-bound, 5-minute tokens. Each environment's secret-id is owned by its own deploy user, which is a boundary that does not exist today since prod and beta both reach the same root-owned age key. - Render still produces the same raw unquoted dotenv: Centralis compares /etc/thermograph.env textually and the Swarm entrypoint parses it by line. The backend dispatch in render-secrets.sh returns early rather than refactoring the shared write path, so the SOPS path stays byte-identical. Verified by rendering dev (12 keys) and prod (32 keys) through it, matching the live hosts. The duplicated write path is noted for consolidation once prod has been on OpenBao for a release cycle. The OpenBao renderer rejects values containing a newline or a shell metacharacter. /etc/thermograph.env is sourced by bash in six places, so such a value is a command-execution path on the deploy host; the SOPS path has always had this hazard and avoids it only because every current value happens to be alphanumeric. Also records that the age keypair is NOT retired by this migration: it is the backup encryption key for every off-box dump (ops-cron.yml:142,197, 30-day retention), so deleting it with the vault files would silently destroy a month of database recoverability. verify-parity.sh is the cutover gate — it renders both backends and diffs them, reporting key names and counts only, never values. --- infra/deploy/env-topology.sh | 9 + infra/deploy/render-secrets-openbao.sh | 239 +++++++++++++++++++++ infra/deploy/render-secrets.sh | 32 +++ infra/openbao/README.md | 254 +++++++++++++++++++++++ infra/openbao/bootstrap-policies.sh | 115 ++++++++++ infra/openbao/bootstrap.sh | 179 ++++++++++++++++ infra/openbao/config/openbao.hcl | 118 +++++++++++ infra/openbao/openbao.service | 50 +++++ infra/openbao/policies/tg-centralis.hcl | 67 ++++++ infra/openbao/policies/tg-host-beta.hcl | 37 ++++ infra/openbao/policies/tg-host-dev.hcl | 72 +++++++ infra/openbao/policies/tg-host-prod.hcl | 44 ++++ infra/openbao/policies/tg-ops-backup.hcl | 50 +++++ infra/openbao/seed-from-sops.sh | 196 +++++++++++++++++ infra/openbao/verify-parity.sh | 158 ++++++++++++++ 15 files changed, 1620 insertions(+) create mode 100644 infra/deploy/render-secrets-openbao.sh create mode 100644 infra/openbao/README.md create mode 100755 infra/openbao/bootstrap-policies.sh create mode 100755 infra/openbao/bootstrap.sh create mode 100644 infra/openbao/config/openbao.hcl create mode 100644 infra/openbao/openbao.service create mode 100644 infra/openbao/policies/tg-centralis.hcl create mode 100644 infra/openbao/policies/tg-host-beta.hcl create mode 100644 infra/openbao/policies/tg-host-dev.hcl create mode 100644 infra/openbao/policies/tg-host-prod.hcl create mode 100644 infra/openbao/policies/tg-ops-backup.hcl create mode 100755 infra/openbao/seed-from-sops.sh create mode 100755 infra/openbao/verify-parity.sh diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh index 7e4ae38..9e2b7c9 100644 --- a/infra/deploy/env-topology.sh +++ b/infra/deploy/env-topology.sh @@ -68,6 +68,14 @@ thermograph_topology() { TG_TAGS_FILE=""; TG_LOCK_FILE=""; TG_BIND_ADDR=""; TG_SKIP_COMMON=0 TG_SVC_PREFIX=""; TG_DATA_NETWORK=""; TG_DB_SERVICE=""; TG_POST_DEPLOY=0 TG_SSH_HOST=""; TG_SSH_TARGET="" + # Which store render-secrets.sh reads: sops | openbao. Defaults to sops for every + # environment and is overridden per environment below, so a cutover is a one-line + # reviewed change that follows the estate's own dev -> main -> release promotion. + # + # This lives here, and NOT in a host marker file like /etc/thermograph/deploy-mode, + # for the reason that marker was demoted to a fallback in the first place: vps2 runs + # prod and beta side by side, and one host-wide file cannot name two backends. + TG_SECRETS_BACKEND=sops case "$env_name" in prod) @@ -220,6 +228,7 @@ thermograph_topology() { export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY + export TG_SECRETS_BACKEND export TG_SSH_HOST TG_SSH_TARGET } diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh new file mode 100644 index 0000000..82c449c --- /dev/null +++ b/infra/deploy/render-secrets-openbao.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# OpenBao source backend for render-secrets.sh. Sourced, never run directly. +# +# Two functions: +# thermograph_openbao_source -> merged dotenv on STDOUT +# thermograph_render_openbao -> source, then write +# +# render_thermograph_secrets in render-secrets.sh dispatches to the second when +# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path +# keeps byte-identical behaviour and all new risk is confined to this file. See the +# note above thermograph_render_openbao on why the write path is duplicated here +# rather than shared, and when to consolidate. +# +# WHY OUTPUT SHAPE IS FROZEN: /etc/thermograph.env must stay raw, UNQUOTED KEY=value. +# Centralis' secrets_inventory / secrets_render compare that file textually +# (deploy/secrets/README.md:206-210), compose reads it via `env_file:`, and the Swarm +# stack's env-entrypoint.sh parses it line by line. Quoting it "properly" would be an +# improvement in isolation and a silent break in context. + +# thermograph_openbao_source +# +# Reads the common set then the per-environment set and merges them with the +# environment winning — the same last-wins semantic as the SOPS concatenation, except +# performed explicitly here because OpenBao returns a map per path rather than a +# stream that can simply be appended. +thermograph_openbao_source() { + local env_name="${1:?env name required}" + local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" + local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" + local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}" + local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" + + command -v bao >/dev/null 2>&1 || { + echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2 + return 1 + } + command -v python3 >/dev/null 2>&1 || { + echo "!! python3 required to render dotenv from OpenBao JSON" >&2 + return 1 + } + + # AppRole credentials: role_id and secret_id, one per line, 0400 root — the same + # protection level as /etc/thermograph/age.key today. Read directly if we can, else + # via sudo, mirroring how render-secrets.sh lifts the age key. + local creds + if [ -r "$approle" ]; then + creds=$(cat "$approle") + else + creds=$(sudo cat "$approle" 2>/dev/null || true) + fi + [ -n "$creds" ] || { + echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2 + return 1 + } + + local role_id secret_id + role_id=$(printf '%s\n' "$creds" | sed -n '1p') + secret_id=$(printf '%s\n' "$creds" | sed -n '2p') + [ -n "$role_id" ] && [ -n "$secret_id" ] || { + echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2 + return 1 + } + + export BAO_ADDR="$addr" + [ -f "$ca" ] && export BAO_CACERT="$ca" + + # Exchange the AppRole for a short-lived token. The token TTL is set on the role + # (see bootstrap.sh) and is deliberately measured in minutes: it only has to + # outlive one render. + local token + token=$(bao write -field=token auth/approle/login \ + role_id="$role_id" secret_id="$secret_id" 2>/dev/null) || { + echo "!! OpenBao AppRole login failed for env='${env_name}'" >&2 + echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2 + return 1 + } + export BAO_TOKEN="$token" + + # Fetch. `common` is skipped for dev by the same flag the SOPS path uses — but note + # that on the OpenBao path the flag is belt, not braces: dev's AppRole is bound to + # the tg-host-dev policy, which DENIES thermograph/data/common outright. If this + # flag is ever forgotten at a call site, dev gets a 403 rather than a file full of + # production credentials. That inversion — from "the caller must remember" to "the + # store refuses" — is the main reason for this migration. + local common_json="" env_json + if [ "${THERMOGRAPH_SECRETS_SKIP_COMMON:-0}" != 1 ]; then + common_json=$(bao kv get -format=json -mount="$mount" common 2>/dev/null) || { + echo "!! cannot read ${mount}/common for env='${env_name}'" >&2 + echo "!! if this is dev, THERMOGRAPH_SECRETS_SKIP_COMMON=1 was not set and the" >&2 + echo "!! tg-host-dev policy correctly refused — that is the guard working." >&2 + return 1 + } + fi + + env_json=$(bao kv get -format=json -mount="$mount" "env/${env_name}" 2>/dev/null) || { + echo "!! cannot read ${mount}/env/${env_name}" >&2 + return 1 + } + + # Merge and emit. Python rather than jq because jq is not installed on these hosts + # and python3 is (the Centralis renderer already relies on it). + # + # This step also enforces the two invariants the dotenv format cannot express: + # * a value containing a newline is REJECTED, because it would silently become + # two lines and the second would parse as a bogus KEY=value or be dropped; + # * a value containing a shell metacharacter is REJECTED, because + # /etc/thermograph.env is sourced by bash in six places (deploy.sh:133, + # deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so + # `$(...)` or a backtick in a secret is a live command-execution path on the + # deploy host. The SOPS path has always had this hazard and has never tripped it + # only because every current value happens to be alphanumeric. Refusing here + # turns a latent code-execution bug into a loud render failure. + THERMOGRAPH_BAO_COMMON="$common_json" \ + THERMOGRAPH_BAO_ENV="$env_json" \ + python3 - <<'PY' +import json, os, re, sys + +def load(raw): + if not raw: + return {} + doc = json.loads(raw) + return doc.get("data", {}).get("data", {}) or {} + +merged = {} +# common first, environment second: last-wins, matching the SOPS concatenation order. +for src in ("THERMOGRAPH_BAO_COMMON", "THERMOGRAPH_BAO_ENV"): + merged.update(load(os.environ.get(src, ""))) + +KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +# Characters that change meaning when the file is sourced by bash. +UNSAFE = set("`$\\\"'") + +bad = [] +lines = [] +# Validate EVERYTHING before emitting a single byte. Printing as we go would leave +# partial output on stdout when a later key fails, and render-secrets.sh's whole +# discipline is that a failed source never produces a partial env file. +for key in sorted(merged): + if not KEY_RE.match(key): + bad.append(f"{key}: not a valid env var name") + continue + val = merged[key] + if val is None: + val = "" + if not isinstance(val, str): + val = json.dumps(val, separators=(",", ":")) + if "\n" in val or "\r" in val: + bad.append(f"{key}: value contains a newline (would corrupt the dotenv)") + continue + if UNSAFE & set(val): + bad.append( + f"{key}: value contains a shell metacharacter (one of ` $ \\ \" ') and " + f"/etc/thermograph.env is sourced by bash — refusing" + ) + continue + lines.append(f"{key}={val}") + +if bad: + sys.stderr.write("!! refusing to render; unsafe values:\n") + for line in bad: + sys.stderr.write(f"!! {line}\n") + sys.exit(1) + +if not lines: + sys.stderr.write("!! refusing to render: OpenBao returned zero keys\n") + sys.exit(1) + +sys.stdout.write("\n".join(lines) + "\n") +PY +} + +# thermograph_render_openbao +# +# The OpenBao equivalent of render_thermograph_secrets: source, then write. +# +# ON THE DUPLICATED WRITE PATH. The write logic below mirrors +# render-secrets.sh:130-154 rather than being factored out and shared. That is a +# deliberate, temporary choice, not an oversight: +# +# render-secrets.sh is on the deploy path for all three environments and there is no +# test suite in front of it (infra/CLAUDE.md: "Shell here runs as root over SSH +# against live hosts with no test suite in front of it"). Extracting the write path +# would mean the SOPS render — which currently works — starts flowing through newly +# moved code that cannot be tested end-to-end from here. An early-return branch keeps +# the SOPS path byte-identical and confines all new risk to the new backend. +# +# CONSOLIDATE THESE after prod has been on the OpenBao backend for a full release +# cycle and the SOPS path is being deleted anyway. Until then, a change to one write +# path must be made in both — which is exactly the cost being accepted here. +thermograph_render_openbao() { + local env_name="${1:?env name required}" + local out="${2:?output path required}" + + echo "==> Rendering $out from OpenBao (env=${env_name})" + + local tmp; tmp=$(mktemp) + # Holds DECRYPTED secrets. Removed on every exit path. Deliberately not a + # `trap ... RETURN` — see render-secrets.sh:82-91 for why that is actively wrong in + # a sourced function (the trap persists into the caller and re-fires on its next + # `source`, where the local is unset and `set -u` makes it fatal and silent). + if ! thermograph_openbao_source "$env_name" > "$tmp"; then + rm -f "$tmp" + return 1 + fi + + # An empty render must never reach $out. The source function already refuses a + # zero-key result, so this is a second line of defence rather than the only one: + # a truncated /etc/thermograph.env is the single worst outcome available here, + # because thermograph.service uses `EnvironmentFile=-` (missing is NOT fatal) and + # the app self-generates AUTH_SECRET and the VAPID pair when they are absent. The + # result would be a green deploy that silently invalidated every session and + # deleted every push subscription. + if [ ! -s "$tmp" ]; then + echo "!! OpenBao render produced an empty file; refusing to write $out" >&2 + rm -f "$tmp" + return 1 + fi + + local rc=0 + if [ -f "$out" ] && [ -w "$out" ]; then + cat "$tmp" > "$out" || rc=1 + elif install -m 0640 "$tmp" "$out" 2>/dev/null; then : + elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then : + else + echo "!! cannot write $out (need file write access or passwordless sudo)" >&2 + rc=1 + fi + + if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then + if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \ + || ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then + echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2 + rc=1 + fi + fi + + rm -f "$tmp" + return "$rc" +} diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index fb5731c..32093d9 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -31,6 +31,38 @@ render_thermograph_secrets() { local out="${3:-/etc/thermograph.env}" [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true) + # Backend dispatch. TG_SECRETS_BACKEND comes from env-topology.sh (per environment, + # because vps2 runs two); THERMOGRAPH_SECRETS_BACKEND overrides it for a by-hand run. + # Default is sops, so an unmigrated host and an unset caller both behave exactly as + # before this branch existed. + # + # The OpenBao path returns early rather than threading a conditional through the + # rest of this function. That is on purpose: everything below — the age-key sudo + # lift, the two-file last-wins concatenation, the three-way write, the mirror — stays + # on precisely the code path it has always been on, so migrating cannot regress the + # backend that is still authoritative for prod. + local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}" + if [ "$backend" = openbao ]; then + if [ -z "$env_name" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but no environment name was resolved" >&2 + return 1 + fi + local bao_lib="${repo}/deploy/render-secrets-openbao.sh" + if [ ! -f "$bao_lib" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2 + echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2 + return 1 + fi + # shellcheck source=/dev/null + . "$bao_lib" + thermograph_render_openbao "$env_name" "$out" + return $? + fi + if [ "$backend" != sops ]; then + echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2 + return 1 + fi + # The per-host file is required; common.yaml is optional so the initial cutover can # seed each host as an exact copy of its live env (byte-identical render, no value # changes) and factor out shared secrets into common.yaml later. diff --git a/infra/openbao/README.md b/infra/openbao/README.md new file mode 100644 index 0000000..196784a --- /dev/null +++ b/infra/openbao/README.md @@ -0,0 +1,254 @@ +# OpenBao — the secret store, phase 1 + +**Status: dormant.** Every file here is committed but nothing is cut over. SOPS is +still authoritative for dev, beta and prod, because `TG_SECRETS_BACKEND` defaults to +`sops` for all three in `infra/deploy/env-topology.sh`. Flipping an environment is a +one-line reviewed change, and it should not happen until `verify-parity.sh` passes. + +Verified against **OpenBao v2.6.1** (2026-07-22). The binary is `bao`. + +--- + +## Why do this at all + +Not for encryption, and not for audit — though the audit log is a real gain. The +decisive reason is one specific failure class that a file-based vault cannot close. + +`infra/CLAUDE.md` states the rule: *"vps1 must never hold prod credentials. It's the +box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of an +isolation boundary."* + +Today that rule is enforced by a **shell flag** — `THERMOGRAPH_SECRETS_SKIP_COMMON=1` +— set by the *caller*, at three separate call sites (`env-topology.sh:196` → +`deploy.sh:67`, `deploy-dev.sh:86`, `infra-sync.yml:93-95`). The renderer itself does +not know that `env=dev` means never-common. A fourth call site, or one by-hand +`render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3 keypairs +(one read-write on the bucket holding prod's database backups), the VAPID private key +that signs Web Push to real subscribers, `REGISTRY_TOKEN`, and the IndexNow and metrics +tokens onto the CI-runner box — **and exits 0**. + +Under `policies/tg-host-dev.hcl` that is not possible to get wrong. dev's identity +cannot read `thermograph/data/common`. A misconfigured caller gets a 403 instead of a +silent success. The failure class is *gone*, not guarded. That is what this migration +buys, and it is worth the cost of running a stateful service. + +The estate has already hardened this once by hand (commit `7583258`, "infra-sync: +refuse to render dev's vault onto a host that is not dev"). That is the shape of a +problem that wants an ACL, not another guard. + +### What it does *not* buy — stated plainly + +- **Rotation of provider-issued credentials.** Roughly half the credential count is + Discord, Contabo S3, the Forgejo registry token, VAPID, IndexNow. OpenBao cannot + rotate any of them; they stay static KV entries. Contabo Object Storage is + S3-compatible but exposes no IAM API, so the AWS secrets engine cannot issue against + it either. +- **Dynamic database credentials.** Inapplicable here for two independent reasons: the + apps read `THERMOGRAPH_DATABASE_URL` once at import and hold a pool, so a TTL'd + credential kills the pool at expiry; and there is no reload path anywhere in the + codebase — no SIGHUP handler, no config re-read. +- **Better review than git.** This is a real regression. Today a secret change is a + PR with a diff, backstopped by `secrets-guard` CI. Under OpenBao it is an + out-of-band API call with no diff and no review. Mitigated by committing a key-name + manifest so CI can still assert no key vanished, plus the audit log and + `bao kv metadata` as the change record — but not fully. +- **Fewer moving parts.** SOPS needs no server. This adds a stateful service to keep + alive, upgrade, back up and TLS-rotate, for ~40 secrets that change rarely. + +--- + +## Shape of the deployment + +| Decision | Choice | Why | +|---|---|---| +| Host | **vps2 only** | vps1 must never hold prod credentials, and with a static seal the box also holds the key that decrypts everything. Cost: vps1 needs vps2 up to deploy dev — acceptable, since when vps2 is down prod is down anyway. | +| Process | **native systemd**, not a container | A Swarm service is circular (`deploy-stack.sh` renders secrets to deploy the stack). A plain `docker run` makes the vault depend on the daemon deploys restart, and is one `prune -a` from gone. Native = one dependency, the disk. | +| Storage | **raft**, single node | 2.6.0 **deprecated the `file` backend** for removal in 2.7.0. Raft also has the only backup primitive (`operator raft snapshot save`). A *two*-node raft would be worse than one: quorum of two means losing either node loses writes. | +| Seal | **static auto-unseal** | See below. | +| TLS | self-signed, 10y, on disk | Must **not** come from Caddy/ACME — that would put DNS and the public internet in the boot chain of the secret store. | +| Auth | AppRole per environment, CIDR-bound | 5-minute tokens; secret-ids that only work from the right mesh address. | +| Consumption | keep rendering a dotenv file | Preserves every existing seam and keeps OpenBao a *deploy-time* dependency, never a runtime one. | + +### The unseal decision, which is the crux + +**Static seal**, key at `/etc/openbao/unseal.key` (0400), plus an off-box copy. + +Shamir (`-key-shares=5 -key-threshold=3`) means the vault is sealed after every process +restart — reboot, package upgrade, OOM kill — and a human must type three shares. +`render-secrets.sh` is on the deploy path for all three environments, so a sealed vault +blocks every deploy and every hotfix. With one operator the N-of-M threshold is +theatre: one person holds all the shares, so it buys nothing against the real threat +while costing an availability property the estate has today for free. A Contabo reboot +currently needs no human at all. + +OpenBao's docs hedge static seal — *"carefully evaluate"* — but that caveat is aimed at +multi-tenant enterprises. The argument here is that **static seal is not a downgrade +from the status quo**: `/etc/thermograph/age.key` is already one 0400 file per host +that decrypts everything forever with no audit trail. A static seal key is the +identical trust model. What changes is everything layered above it. + +**The one new single point of failure:** lose that key and the raft snapshots are +unrestorable. It must exist in ≥2 places, one not on vps2. Hard operator obligation. + +A `transit` seal against a second OpenBao on vps1 is genuinely better — vps2's disk +would no longer contain the unseal key — but the vps1 unsealer needs unsealing too, so +the circularity moves rather than dissolves, and it adds "vps1 must be up before vps2's +vault unseals" as a new failure mode. Revisit once the primary migration is boring. + +--- + +## Layout + +``` +thermograph/data/common the 16 shared beta+prod values (= common.yaml) +thermograph/data/env/prod prod's 16 (= prod.yaml) +thermograph/data/env/beta beta's 8 (= beta.yaml) +thermograph/data/env/dev dev's 12 — inherits NOTHING (= dev.yaml) +thermograph/data/centralis/prod Centralis' 9 (= centralis.prod.yaml) +thermograph/data/ops/backup S3 creds for ops-cron + the backup age recipient +thermograph/data/legacy/age the age PRIVATE key — see "the age key survives" +``` + +Inheritance lives in the render order (`common` then `env/`, last-wins); +isolation lives in the policy. Today both live in one shell variable. + +`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny +rule expressible as exactly one path, with no wildcard over `env/*` able to +accidentally re-include it. + +**~20 of the 61 keys are not secrets** (`PORT`, `WORKERS`, `APP_CPUS`, `TIMESCALEDB_TAG`, +`THERMOGRAPH_BASE_URL`, the Discord channel IDs, the VAPID *public* key, …). They stay +in the vault **for the cutover**, because byte-identical parity is the entire safety +property and splitting them would destroy it. Moving pure config back into the repo is +a clean follow-on that shrinks the vault-outage blast radius. + +--- + +## The age key survives this migration + +**This is the trap that would cause silent data loss.** + +`ops-cron.yml:142` and `:197` stream every off-box Postgres and Forgejo dump through +`pg_dump | age -r | rclone rcat`, with 30-day S3 retention. Restore uses +`/etc/thermograph/age.key`. So the age keypair is not just the SOPS transport — **it is +the backup encryption key.** + +Deleting it along with the SOPS vault files would destroy up to 30 days of database +recoverability, and nothing would notice until someone attempted a restore. + +So: keep the age private key at `thermograph/data/legacy/age` *and* in the password +manager, and keep it on disk until either the newest age-encrypted object in S3 has +aged out, or the backup path is re-pointed at a dedicated backup recipient. Retiring +SOPS and retiring age are two different projects. + +--- + +## Runbook + +### Stand it up (once, by the operator, on vps2) + +```sh +sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init +# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then: +export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt +export BAO_TOKEN= +sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles +bao token revoke -self +``` + +### Seed and prove (from your own machine — it needs your age key) + +```sh +infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing +infra/openbao/seed-from-sops.sh --all +infra/openbao/verify-parity.sh --all # MUST pass before any flip +``` + +`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md` +the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this +inherits that rule. + +### Cut an environment over + +One PR per hop, following the estate's own promotion model. + +```diff + dev) +- TG_SECRETS_BACKEND=sops ++ TG_SECRETS_BACKEND=openbao +``` + +Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended +gate before prod: **7 consecutive green parity runs on all three environments**, run +nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting +CI any vault access). + +`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the +SOPS path for a full release cycle after prod flips — it is the best mitigation +available for anything unforeseen. + +### Rollback + +Revert the one-line PR and redeploy. The SOPS path is untouched by this work — verified +by rendering dev (12 keys) and prod (32 keys) through it after the dispatch was added, +matching the live hosts exactly. + +--- + +## Failure modes + +| Failure | Effect | Mitigation | +|---|---|---| +| Vault down/sealed at deploy | Render returns 1, deploy aborts, **running stack unaffected** — temp-then-install means `/etc/thermograph.env` is never truncated | `Restart=always` + static seal so a reboot self-heals; alert on `/v1/sys/health`; `TG_SECRETS_BACKEND=sops` is a working revert | +| **Audit device wedges** | ⚠️ OpenBao **stops answering requests entirely** when no enabled audit device can record them — a full `/var` on vps2 blocks every deploy | Two devices (file + syslog); logrotate signals **SIGHUP** or bao writes to an unlinked inode; disk alert on vps2 | +| Static seal key lost | Raft snapshots unrestorable | ≥2 copies, one off-box. The one new SPOF | +| Raft corruption / disk loss | Total vault loss | Nightly `operator raft snapshot save`, age-encrypted to S3 beside the DB dumps. **Verify a restore end-to-end before any consumer depends on it** | +| Cold boot of vps2 | Nothing needs the vault | Swarm restarts from persisted specs; `stack.env` / `thermograph.env` / `centralis.env` all persist. **This property exists because we render a file, and would be destroyed by app-native reads** | +| Secret-id leak | Useless off the mesh | `secret_id_bound_cidrs`; revoke by accessor without knowing the value | +| beta credential reaches prod | Cross-env compromise | Separate AppRoles, separate policies, secret-ids owned by the separate deploy users (`agent` vs `deploy`) — a boundary that does **not** exist today, since both currently reach the same root-owned age key. Honest limit: root on vps2 defeats it, exactly as it defeats the single age key now | +| Upgrade to 2.7.0 | Removes the `file` backend and built-in cloud KMS seals | Already avoided by choosing raft + static. Pin the version | + +--- + +## Deliberately out of scope + +For ~40 credentials and one operator the real risk is over-engineering. Not adopted: +HA/multi-node raft, namespaces, dynamic database credentials, `bao agent`, cert auth, +PKI, identity groups, transit seal (revisit later), OIDC for the operator. + +**OIDC for CI is out of scope for a reason worth recording.** Forgejo Actions *does* +support OIDC — shipped in **Forgejo v15.0**, needing Runner > v12.5.0, enabled via +`enable-openid-connect: true` rather than GitHub's `permissions: id-token: write`. But +this estate runs `codeberg.org/forgejo/forgejo:9-rootless` +(`infra/deploy/forgejo/docker-stack.yml:59`). That is a six-major-version upgrade, and +coupling it to this migration would mean two large independent migrations at once with +no way to tell which one broke. Revisit after Forgejo reaches v15; then +`role_type=jwt` with `bound_claims` removes the last CI-side bearer credential. + +CI keeps its SSH keys and `REGISTRY_TOKEN` as Forgejo secrets. **CI gets no vault +access at all** — the SSH-in architecture already means the *host* renders, never the +runner, which is the least-privilege topology and should be preserved rather than +"upgraded". + +### Secret zero, honestly + +Nothing eliminates it; you choose where it sits and how little of it there is. After +this phase the chain terminates at **9 Forgejo Actions secrets + 1 static seal key + 4 +AppRole secret-ids**, versus today's **13 CI secrets + 1 omnipotent age key per host**. + +--- + +## Verification gaps to close on the box + +Flagged rather than guessed, because these could not be checked from a workstation: + +1. Whether a raft snapshot is restorable under a *different* seal key. Docs don't say. + **Test in phase 0, before anything depends on it.** +2. Exact `-wrap-ttl` flag spelling on `bao write` — documented in the response-wrapping + concept page but absent from the commands index. Check `bao write -h`. +3. Debian/Ubuntu package repo URL. The install docs claim `.deb` packages exist but + give no repository; `bootstrap.sh` uses the GitHub release tarball with checksum + verification, which is fine and also fixes a standing weakness — `provision-secrets.sh` + installs sops and age with a bare `sudo curl` and no verification at all. +4. Vault-era naming persists inside OpenBao config (`vault { }` stanza, + `X-Vault-Wrap-TTL` header). Expect it; don't "fix" it. diff --git a/infra/openbao/bootstrap-policies.sh b/infra/openbao/bootstrap-policies.sh new file mode 100755 index 0000000..81d07bd --- /dev/null +++ b/infra/openbao/bootstrap-policies.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Second half of bootstrap: KV mount, policies, AppRoles, host credentials. +# Run on vps2 with BAO_TOKEN set to a root token. Idempotent — safe to re-run. +# +# export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt +# export BAO_TOKEN= +# sudo -E bash infra/openbao/bootstrap-policies.sh +# +# Split from bootstrap.sh because everything here is safely automatable: none of it +# produces a credential a human has to custody. The one thing that does — the recovery +# material from `bao operator init` — is in bootstrap.sh and stops for the operator. +set -euo pipefail + +MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}" +SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +POLICY_DIR="$SELF_DIR/policies" + +: "${BAO_TOKEN:?BAO_TOKEN must be set to a root token}" +: "${BAO_ADDR:=https://127.0.0.1:8200}" +export BAO_ADDR +command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; } + +echo "==> KV v2 mount at ${MOUNT}/" +if bao secrets list -format=json | grep -q "\"${MOUNT}/\""; then + echo " already mounted" +else + bao secrets enable -path="$MOUNT" -version=2 kv +fi + +echo "==> policies" +for p in "$POLICY_DIR"/*.hcl; do + name="$(basename "$p" .hcl)" + bao policy write "$name" "$p" >/dev/null + echo " wrote $name" +done + +echo "==> approle auth" +bao auth list -format=json | grep -q '"approle/"' || bao auth enable approle + +# One role per environment, CIDR-bound to the host that legitimately renders it. +# +# secret_id_ttl=0 (never expires) is deliberate: a secret-id that silently lapses +# mid-quarter is a self-inflicted outage. The short lifetime lives on the TOKEN +# instead — it only has to outlive one render. +# +# The CIDR binding is the part that makes a leaked secret-id close to useless: it is +# only accepted from the correct WireGuard address. +add_role() { + local role="$1" policy="$2" cidr="$3" + bao write "auth/approle/role/${role}" \ + token_policies="$policy" \ + secret_id_bound_cidrs="$cidr" \ + token_bound_cidrs="$cidr" \ + secret_id_ttl=0 secret_id_num_uses=0 \ + token_ttl=5m token_max_ttl=15m token_num_uses=10 >/dev/null + echo " role ${role} -> ${policy} (bound ${cidr})" +} + +add_role tg-prod tg-host-prod 10.10.0.1/32 +add_role tg-beta tg-host-beta 10.10.0.1/32 +add_role tg-dev tg-host-dev 10.10.0.2/32 +add_role tg-centralis tg-centralis 10.10.0.1/32 + +# Install the local (vps2) credentials. prod and beta both render on this box. +# +# The OWNERSHIP here is a real boundary that does not exist today, and it is the one +# concrete isolation win available on a shared host. render-secrets.sh:119-124 records +# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the +# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation +# at all. Giving each environment its own 0400 secret-id owned by its own deploy user +# creates one. +# +# Be honest about the limit: root on vps2 reads both files, exactly as root today reads +# the one age key. This defends against MISCONFIGURATION, not against intrusion. +install_creds() { + local role="$1" dest="$2" owner="$3" + local rid sid + rid=$(bao read -field=role_id "auth/approle/role/${role}/role-id") + sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id") + # umask before creation, so the file is never briefly world-readable. + ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" ) + chown "$owner" "$dest" + chmod 0400 "$dest" + echo " installed $dest (0400 $owner)" +} + +if [ "$(id -u)" = 0 ]; then + install -d -o root -g root -m 0755 /etc/thermograph + install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent" + install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \ + || echo " (beta creds skipped — no 'deploy' user on this box?)" + install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root" +else + echo "!! not root; skipping credential install. Re-run with sudo -E." >&2 +fi + +cat < done. Policies and AppRoles are in place; the vault is still EMPTY. + +dev's credentials must be installed on vps1, not here. From your own machine: + + bao read -field=role_id auth/approle/role/tg-dev/role-id + bao write -f -field=secret_id auth/approle/role/tg-dev/secret-id + # then, on vps1, write both lines to /etc/thermograph/openbao-approle (0400 agent) + +Prefer response wrapping so the secret-id never lands in shell history or a log: + bao write -f -wrap-ttl=5m auth/approle/role/tg-dev/secret-id # gives a wrapping token + # on vps1: bao unwrap (single-use: a failed unwrap means interception) + +Next: seed, then verify parity. Neither cuts anything over. + infra/openbao/seed-from-sops.sh --dry-run + infra/openbao/seed-from-sops.sh --all + infra/openbao/verify-parity.sh --all +EOF diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh new file mode 100755 index 0000000..39fc5e6 --- /dev/null +++ b/infra/openbao/bootstrap.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Bring up OpenBao on vps2 from nothing. Run ONCE, as root, ON vps2, BY THE OPERATOR. +# +# sudo bash infra/openbao/bootstrap.sh +# +# ============================================================================ +# WHY THIS IS NOT AUTOMATED, AND MUST NOT BE +# ============================================================================ +# `bao operator init` mints the credentials that become the new root of trust for the +# entire estate: the recovery keys and the initial root token. Those have to be +# CUSTODIED BY A HUMAN. Any place a script could put them — a file on the box, the +# repo, a CI log, an agent transcript — is worse custody than the age key this +# migration is meant to improve on. +# +# So this script does everything up to and including init, prints the recovery +# material ONCE to the operator's terminal, and then stops and tells you what to do +# with it. It deliberately does not store it, mail it, or push it anywhere. +# +# Everything AFTER custody (policies, AppRoles, seeding, parity checks) is automated, +# because none of it produces a credential a human needs to hold. +set -euo pipefail + +BAO_VERSION="${BAO_VERSION:-2.6.1}" +MESH_ADDR="${MESH_ADDR:-10.10.0.1}" +SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +[ "$(id -u)" = 0 ] || { echo "!! run as root (sudo bash $0)" >&2; exit 1; } + +hostname_now="$(hostname)" +case "$hostname_now" in + vmi3453260) : ;; # vps2 + *) + # Refuse to bootstrap the secret store on the wrong box. Putting it on vps1 would + # invert the estate's central security decision: infra/CLAUDE.md states "vps1 must + # never hold prod credentials. It's the box that runs Forgejo, its CI runner, and + # dev's unreviewed branch — the opposite of an isolation boundary." With a static + # seal, the box also holds the key that decrypts everything. + # + # This mirrors the guard infra-sync.yml already has (commit 7583258, "refuse to + # render dev's vault onto a host that is not dev"). + echo "!! this host is '$hostname_now', not vps2 (vmi3453260)." >&2 + echo "!! OpenBao belongs on vps2. Refusing." >&2 + exit 1 + ;; +esac + +echo "==> 1/7 install the bao binary (v${BAO_VERSION})" +if ! command -v bao >/dev/null 2>&1; then + # NOTE: checksum verification. provision-secrets.sh installs sops and age with a bare + # `sudo curl` and no verification at all, which is a standing weakness (a compromised + # release URL yields a root binary that sees every plaintext). Do not copy that here. + # The checksums file is signed by the release; verify before installing. + tmpd="$(mktemp -d)" + base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}" + curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz" + curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS" + ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || { + echo "!! checksum mismatch on the bao tarball; refusing to install" >&2 + rm -rf "$tmpd"; exit 1 + } + tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao + install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao + rm -rf "$tmpd" +fi +bao version + +echo "==> 2/7 user, directories, TLS" +id -u openbao >/dev/null 2>&1 || useradd --system --home /var/lib/openbao --shell /usr/sbin/nologin openbao +install -d -o openbao -g openbao -m 0700 /var/lib/openbao +install -d -o openbao -g openbao -m 0750 /var/log/openbao +install -d -o root -g root -m 0755 /etc/openbao +install -d -o openbao -g openbao -m 0700 /etc/openbao/tls + +if [ ! -f /etc/openbao/tls/bao.crt ]; then + # Self-signed, 10 years, SAN on the mesh address. Deliberately NOT from Caddy/ACME: + # that would put DNS and the public internet in the boot chain of the secret store. + openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \ + -keyout /etc/openbao/tls/bao.key -out /etc/openbao/tls/bao.crt \ + -subj "/CN=openbao.thermograph.internal" \ + -addext "subjectAltName=IP:${MESH_ADDR},IP:127.0.0.1" >/dev/null 2>&1 + chown openbao:openbao /etc/openbao/tls/bao.key /etc/openbao/tls/bao.crt + chmod 0400 /etc/openbao/tls/bao.key + chmod 0444 /etc/openbao/tls/bao.crt +fi +# Consumers need the cert to trust the listener. +install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt + +echo "==> 3/7 static seal key" +if [ ! -f /etc/openbao/unseal.key ]; then + # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key. + ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key ) + chown openbao:openbao /etc/openbao/unseal.key + chmod 0400 /etc/openbao/unseal.key + echo " generated /etc/openbao/unseal.key (0400 openbao)" + echo " ⚠️ COPY THIS OFF THE BOX into your password manager before proceeding." + echo " Without an off-box copy, a raft snapshot is UNRESTORABLE." +else + echo " /etc/openbao/unseal.key already present, leaving alone" +fi + +echo "==> 4/7 config + systemd unit" +install -m 0640 -o root -g openbao "$SELF_DIR/config/openbao.hcl" /etc/openbao/config.hcl +install -m 0644 -o root -g root "$SELF_DIR/openbao.service" /etc/systemd/system/openbao.service +# logrotate MUST use SIGHUP: without it bao keeps writing to an unlinked inode and the +# audit log silently stops growing — and a wedged audit device makes OpenBao stop +# answering requests entirely, which would block every deploy on the estate. +cat > /etc/logrotate.d/openbao <<'ROTATE' +/var/log/openbao/audit.log { + daily + rotate 30 + compress + missingok + notifempty + create 0600 openbao openbao + postrotate + systemctl kill --signal=SIGHUP openbao.service 2>/dev/null || true + endscript +} +ROTATE +systemctl daemon-reload +systemctl enable --now openbao.service +sleep 3 +systemctl is-active --quiet openbao.service || { + echo "!! openbao.service failed to start; check: journalctl -u openbao -n 50" >&2 + exit 1 +} + +export BAO_ADDR="https://127.0.0.1:8200" +export BAO_CACERT=/etc/openbao/tls/bao.crt + +echo "==> 5/7 initialise" +if bao status -format=json 2>/dev/null | grep -q '"initialized": *true'; then + echo " already initialised, skipping" +else + echo + echo " ############################################################" + echo " # RECOVERY MATERIAL FOLLOWS. IT IS SHOWN EXACTLY ONCE. #" + echo " # Store the 3 recovery keys in 3 DIFFERENT places. #" + echo " # 2 of 3 are needed to rekey or mint a new root token. #" + echo " ############################################################" + echo + # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init + # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal + # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one + # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token. + bao operator init -recovery-shares=3 -recovery-threshold=2 + echo + echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are" + echo " stored off this machine." + read -r _ +fi + +bao status || true + +cat < 6/7 and 7/7 need a root token, which is NOT stored anywhere by design. + +Paste the initial root token (or one minted from recovery keys) and run: + + export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt + export BAO_TOKEN= + bash ${SELF_DIR}/bootstrap-policies.sh + +That script enables the KV mount, writes the policies, creates the AppRoles and +installs each host's credentials. Then, from YOUR OWN machine (it needs your age +key, and per infra/CLAUDE.md a script that reads production secrets is not for an +agent to run): + + infra/openbao/seed-from-sops.sh --dry-run # check first + infra/openbao/seed-from-sops.sh --all + infra/openbao/verify-parity.sh --all # must PASS before any flip + +Then revoke the root token: bao token revoke -self + +NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for +every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative +until you flip an environment in a reviewed PR — dev first. +EOF diff --git a/infra/openbao/config/openbao.hcl b/infra/openbao/config/openbao.hcl new file mode 100644 index 0000000..07f4d87 --- /dev/null +++ b/infra/openbao/config/openbao.hcl @@ -0,0 +1,118 @@ +// OpenBao server config for the Thermograph estate. Lives at /etc/openbao/config.hcl +// on vps2. Verified against OpenBao v2.6.1 (2026-07-22). +// +// ============================================================================ +// WHY THIS RUNS AS A NATIVE SYSTEMD SERVICE AND NOT A CONTAINER +// ============================================================================ +// The estate is Docker-native and this is the one thing that must not be. The rule +// is that OpenBao must not depend on anything it secures. +// +// * A Swarm service is fatal: deploy-stack.sh renders secrets in order to deploy +// the stack, so the stack cannot contain the thing that holds them. +// * A standalone `docker run --restart=always` is merely bad: it makes the vault's +// availability a function of the same Docker daemon that deploys restart, and +// leaves it one careless `docker system prune -a` from gone. The root CLAUDE.md +// already lists `prune -a` as a thing that eats the rollback image; it would +// eat this too. +// +// A native binary with Restart=always has exactly one dependency: the local disk. +// See infra/openbao/openbao.service. +// +// Corollary that is easy to get wrong: the listener cert must NOT come from the +// Caddy/ACME path. That would put DNS and the public internet in the boot chain of +// the estate's secret store. It is self-signed, long-lived, and on disk. +// ============================================================================ + +ui = false +cluster_name = "thermograph" +disable_mlock = true +log_level = "info" + +// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0 +// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys +// a migration within a release. Raft also has the only first-class backup primitive +// (`bao operator raft snapshot save`), which is the disaster-recovery story. +// +// A single-node raft is the honest shape for a one-operator estate. Note that a +// TWO-node raft would be strictly worse than one: quorum of two means losing either +// node loses write availability. Three voters would be needed, and the desktop +// (10.10.0.3) is intermittent, so it cannot be a reliable third. +storage "raft" { + path = "/var/lib/openbao" + node_id = "vps2" +} + +// Mesh-only. 10.10.0.1 is vps2's WireGuard address. There is deliberately no +// 0.0.0.0 listener and no public DNS record — vps2 is a public VPS, and reaching +// OpenBao should require being on the mesh, exactly like the Swarm control ports +// and the Loki endpoint. +listener "tcp" { + address = "10.10.0.1:8200" + tls_cert_file = "/etc/openbao/tls/bao.crt" + tls_key_file = "/etc/openbao/tls/bao.key" +} + +// Loopback, so a by-hand `bao` on the box works without the mesh address. +listener "tcp" { + address = "127.0.0.1:8200" + tls_cert_file = "/etc/openbao/tls/bao.crt" + tls_key_file = "/etc/openbao/tls/bao.key" +} + +api_addr = "https://10.10.0.1:8200" +cluster_addr = "https://10.10.0.1:8201" + +// ============================================================================ +// SEAL: static auto-unseal. This is the crux of the whole design. +// ============================================================================ +// The `static` seal (OpenBao v2.4.0+) auto-unseals from a local key with no cloud +// KMS and no HSM. OpenBao's own docs hedge it — "carefully evaluate use of Static +// Key Auto Unseal" — but that caveat is written for multi-tenant enterprises. +// +// The decisive argument here is that static seal is NOT A DOWNGRADE FROM THE STATUS +// QUO. Today /etc/thermograph/age.key is one file, 0400, root-owned, on each +// rendering host, which decrypts every secret in the estate forever with no audit +// trail. A static seal key at 0400 on vps2 is the IDENTICAL trust model: one file on +// one box whose compromise is total compromise. What changes is everything layered +// above it — per-consumer ACLs, an audit log, five-minute tokens, versioned rollback. +// +// WHY NOT SHAMIR: `bao operator init -key-shares=5 -key-threshold=3` means the vault +// is sealed after every process restart — reboot, package upgrade, OOM kill — and a +// human must type three shares. render-secrets.sh is on the deploy path for all three +// environments, so a sealed vault blocks every deploy and every hotfix. With one +// operator the N-of-M threshold is theatre: one person holds all the shares, so it +// buys nothing against the real threat while costing an availability property the +// estate currently has for free. Today a Contabo reboot needs no human at all. +// Introducing a human-in-the-loop requirement that does not exist today is a +// regression this estate cannot absorb. +// +// THE ONE NEW SINGLE POINT OF FAILURE this migration introduces: lose this key and +// the raft snapshots become unrestorable. It MUST exist in at least two places, one +// of them not on vps2. That is a hard operator obligation, not a suggestion. +// +// Rotation is supported n-1 via previous_key / previous_key_id. +seal "static" { + current_key_id = "20260801-1" + current_key = "file:///etc/openbao/unseal.key" +} + +// ============================================================================ +// AUDIT: two devices, deliberately. +// ============================================================================ +// ⚠️ OpenBao STOPS ANSWERING REQUESTS ENTIRELY when no enabled audit device can +// record them. That is correct behaviour for an audit log and a catastrophic way to +// discover a full disk: a full /var on vps2 would block every deploy on the estate. +// Two independent devices mean one wedging is survivable. +// +// Values are HMAC-SHA256'd in the log, so it is safe to ship to Loki through the +// existing Alloy pipeline — which finally makes "who read which secret, when" a +// queryable question. Today reading age.key leaves no trace whatsoever. +// +// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an +// unlinked inode and the log silently stops growing. +audit "file" { + file_path = "/var/log/openbao/audit.log" + mode = "0600" +} + +audit "syslog" {} diff --git a/infra/openbao/openbao.service b/infra/openbao/openbao.service new file mode 100644 index 0000000..a136c4f --- /dev/null +++ b/infra/openbao/openbao.service @@ -0,0 +1,50 @@ +# OpenBao, as a native systemd service on vps2. Install to +# /etc/systemd/system/openbao.service. +# +# Native, not containerised, on purpose — see the header of config/openbao.hcl. The +# whole point is that this unit's only dependency is the local disk: not the Docker +# daemon that deploys restart, not the Swarm it would otherwise be deployed by, not +# Caddy/ACME for its certificate, and not TimescaleDB. +# +# Note there is currently NO systemd unit of any kind loaded on either host for +# Thermograph (infra/deploy/thermograph.service exists in the repo but is not +# installed), so this is the first. That is deliberate: it is the one component that +# must survive the container layer being broken. + +[Unit] +Description=OpenBao secret store +Documentation=https://openbao.org/docs/ +# network-online rather than plain network: the mesh listener binds 10.10.0.1, and +# binding a WireGuard address before the interface is up fails the unit outright. +After=network-online.target +Wants=network-online.target +# Explicitly NOT After=docker.service. Adding it would recreate the dependency this +# design exists to avoid. +ConditionFileNotEmpty=/etc/openbao/config.hcl + +[Service] +User=openbao +Group=openbao +ExecStart=/usr/local/bin/bao server -config=/etc/openbao/config.hcl +ExecReload=/bin/kill --signal HUP $MAINPID +KillSignal=SIGINT + +# Restart always, which together with the static seal is what makes a host reboot +# self-healing. This pair is the reason deploys are not blocked by a reboot. +Restart=always +RestartSec=5 + +# mlock is disabled in the config (disable_mlock = true), so no IPC_LOCK capability +# is needed. If mlock is ever enabled, add: AmbientCapabilities=CAP_IPC_LOCK +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=full +ProtectHome=yes +# The three paths the service legitimately writes. +ReadWritePaths=/var/lib/openbao /var/log/openbao +# Raft + audit both do a lot of small writes; the default is too low for a store. +LimitNOFILE=65536 +LimitCORE=0 + +[Install] +WantedBy=multi-user.target diff --git a/infra/openbao/policies/tg-centralis.hcl b/infra/openbao/policies/tg-centralis.hcl new file mode 100644 index 0000000..6518ba8 --- /dev/null +++ b/infra/openbao/policies/tg-centralis.hcl @@ -0,0 +1,67 @@ +// Policy for the Centralis control-plane container on vps2. +// +// Centralis is the tool an operator drives a rotation THROUGH, so it needs more than +// a host render does: it must enumerate key names across every environment to build +// `secrets_inventory`, and write values to perform `secrets_rotate`. +// +// The split below is the important part. Centralis gets: +// * METADATA read/list on every path -> it can build a names-and-versions +// inventory across all three environments WITHOUT being able to read a value. +// * DATA write on every path -> it can rotate. +// * DATA read on NOTHING. +// +// That is a genuine improvement on the SOPS design, not a port of it. Today +// `secrets_inventory` can list key names without a decryption key only because SOPS +// happens to leave key names as plaintext in the YAML — a property of the file +// format, relied on deliberately (tools/secrets.ts:199 `extractKeyNames`). Under +// OpenBao the same "names, never values" guarantee becomes an ENFORCED capability +// boundary instead of a fortunate accident. Centralis cannot print a secret it is +// not able to fetch. +// +// The `read` omission is load-bearing: Centralis holds the Docker socket (root +// -equivalent on prod) and an SSH private key to beta and dev. Granting it data read +// on prod's secrets would make a Centralis compromise equivalent to full estate +// credential disclosure. It already effectively is via the Docker socket — but there +// is no reason to hand it a second, easier path. + +// Names and version history across the whole tree — no values. +path "thermograph/metadata/*" { + capabilities = ["read", "list"] +} + +// Rotation: write a new version. Note "create" + "update" but NOT "read": +// OpenBao permits a blind write, which is exactly the shape secrets_rotate wants — +// it mints or receives a new value and stores it, and never needs the old one. +path "thermograph/data/*" { + capabilities = ["create", "update"] +} + +// Its OWN configuration is the one place Centralis may read, because it must render +// /etc/centralis.env for itself. This is the direct analogue of centralis.prod.yaml. +path "thermograph/data/centralis/prod" { + capabilities = ["read", "create", "update"] +} + +// Version rollback, so a bad rotation is recoverable through the control plane +// rather than requiring a shell on the box. This is the capability that replaces +// "revert the PR" in the SOPS model. +path "thermograph/undelete/*" { + capabilities = ["update"] +} + +path "thermograph/destroy/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +// Read its own AppRole role-id for self-diagnosis, but never the secret-id. +path "auth/approle/role/tg-centralis/role-id" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/tg-host-beta.hcl b/infra/openbao/policies/tg-host-beta.hcl new file mode 100644 index 0000000..7d8ae67 --- /dev/null +++ b/infra/openbao/policies/tg-host-beta.hcl @@ -0,0 +1,37 @@ +// Policy for BETA's render identity on vps2. Mirror of tg-host-prod.hcl. +// +// Beta legitimately reads the shared set: unlike dev, beta is on vps2 and already +// co-resident with prod, so withholding `shared` from it would buy nothing while +// breaking the render (beta's 24 live keys = beta.yaml's 8 + shared's 16). + +path "thermograph/data/common" { + capabilities = ["read"] +} + +path "thermograph/metadata/common" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/beta" { + capabilities = ["read"] +} + +path "thermograph/metadata/env/beta" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/prod" { + capabilities = ["deny"] +} + +path "thermograph/data/env/dev" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/infra/openbao/policies/tg-host-dev.hcl b/infra/openbao/policies/tg-host-dev.hcl new file mode 100644 index 0000000..2d7b8f9 --- /dev/null +++ b/infra/openbao/policies/tg-host-dev.hcl @@ -0,0 +1,72 @@ +// Policy for the DEV host's render identity (vps1). +// +// THIS FILE IS THE POINT OF THE MIGRATION. +// +// infra/CLAUDE.md states the rule: "vps1 must never hold prod credentials. It's the +// box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of +// an isolation boundary." +// +// Today that rule is enforced by a SHELL FLAG (THERMOGRAPH_SECRETS_SKIP_COMMON=1) set +// by the *caller*, at three separate call sites: env-topology.sh:196 -> deploy.sh:67, +// deploy-dev.sh:86, and infra-sync.yml:93-95. The renderer itself does not know that +// env=dev means never-common. A fourth call site, or one by-hand +// `render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3 +// keypairs (one read-write on the bucket holding prod's database backups), the VAPID +// private key that signs Web Push to real subscribers, REGISTRY_TOKEN, and the +// IndexNow and metrics tokens onto the Forgejo CI-runner box — and exits 0. +// +// Under this policy that is no longer possible to get wrong. dev's identity cannot +// read the common path. A misconfigured caller gets a 403 from OpenBao, not a silent +// success with production credentials on disk. The failure class is gone, not guarded. + +// dev's own values. This is the ONLY secret path dev can read. +path "thermograph/data/env/dev" { + capabilities = ["read"] +} + +path "thermograph/metadata/env/dev" { + capabilities = ["read", "list"] +} + +// EXPLICIT DENY on the shared production credential set. +// +// A deny rule is redundant with OpenBao's default-deny — anything not granted is +// already refused. It is here anyway, and must stay, for two reasons: +// 1. It is documentation that survives refactoring. Someone widening dev's grants +// has to delete an explicit deny to break the rule, which is a much louder edit +// than adding a path to an allow list. +// 2. In OpenBao, a deny on a path beats any grant at that path regardless of rule +// order or specificity. So if dev's identity is ever accidentally given a +// broader policy alongside this one, this still wins. +path "thermograph/data/common" { + capabilities = ["deny"] +} + +path "thermograph/metadata/common" { + capabilities = ["deny"] +} + +// Deny the other environments outright. Same argument: dev has no business reading +// beta or prod, and being explicit means a widened grant elsewhere cannot silently +// re-open it. +path "thermograph/data/env/prod" { + capabilities = ["deny"] +} + +path "thermograph/data/env/beta" { + capabilities = ["deny"] +} + +path "thermograph/data/centralis/*" { + capabilities = ["deny"] +} + +// Token self-management, so the render can look up and renew its own lease without +// needing a broader grant. +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/infra/openbao/policies/tg-host-prod.hcl b/infra/openbao/policies/tg-host-prod.hcl new file mode 100644 index 0000000..bb2566d --- /dev/null +++ b/infra/openbao/policies/tg-host-prod.hcl @@ -0,0 +1,44 @@ +// Policy for PROD's render identity on vps2. +// +// Reads the shared set plus prod's own overrides — the OpenBao equivalent of +// "common.yaml then prod.yaml, last-wins". +// +// Note what this policy does NOT grant: beta. vps2 runs prod and beta side by side +// with one filesystem and one SSH credential, so the host is emphatically not an +// isolation boundary between them (infra/CLAUDE.md is explicit about this). Separate +// identities with separate policies do not fix that — a foothold on the box can read +// both AppRole credentials — but they do mean a *mistake* cannot cross the line: +// prod's render cannot accidentally pull beta's database password, and the audit log +// attributes every read to one environment or the other. + +path "thermograph/data/common" { + capabilities = ["read"] +} + +path "thermograph/metadata/common" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/prod" { + capabilities = ["read"] +} + +path "thermograph/metadata/env/prod" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/beta" { + capabilities = ["deny"] +} + +path "thermograph/data/env/dev" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/infra/openbao/policies/tg-ops-backup.hcl b/infra/openbao/policies/tg-ops-backup.hcl new file mode 100644 index 0000000..670f102 --- /dev/null +++ b/infra/openbao/policies/tg-ops-backup.hcl @@ -0,0 +1,50 @@ +// Policy for the ops-cron backup job (.forgejo/workflows/ops-cron.yml). +// +// Exists to DELETE FOUR CI SECRETS. Today S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY and +// S3_SECRET_KEY are Forgejo repo secrets, even though ops-cron only ever uses them +// INSIDE the script it SSHes onto the host — they are passed through `envs:` and +// consumed as RCLONE_CONFIG_ARCHIVE_*. Nothing needs them on the runner. Moving them +// here means the host reads them directly and CI holds four fewer credentials. +// +// It also collapses a genuine duplication: the same values already live in the SOPS +// vault as THERMOGRAPH_S3_* in common.yaml, so there are currently two sources of +// truth for one credential pair, free to drift. +// +// The backup recipient belongs here too — see thermograph/data/ops/backup below. +// ops-cron.yml:142 and :197 currently HARDCODE the age recipient as a literal, in two +// places, and that same public key is duplicated in eleven places across the repo. + +path "thermograph/data/ops/backup" { + capabilities = ["read"] +} + +path "thermograph/metadata/ops/backup" { + capabilities = ["read", "list"] +} + +// ============================================================================ +// THE AGE KEY IS NOT RETIRED BY THIS MIGRATION. This path is why. +// ============================================================================ +// ops-cron.yml streams every off-box database dump through +// `pg_dump | age -r | rclone rcat`, with 30-day S3 retention, and restore +// depends on the private half at /etc/thermograph/age.key. The Forgejo backup path is +// the same shape. +// +// So the age keypair is not merely the SOPS transport — it is the BACKUP ENCRYPTION +// KEY. Deleting it alongside the SOPS vault files would silently destroy up to 30 days +// of database recoverability, and nothing would notice until a restore was attempted. +// +// Therefore: the age private key is stored at thermograph/data/legacy/age (operator +// access only, deliberately NOT granted by this policy) and kept on disk until either +// the newest age-encrypted object in S3 has aged out, or the backup path is re-pointed +// at a dedicated backup recipient. Only then may /etc/thermograph/age.key be removed. +// +// The `deny` is explicit so that widening this policy cannot hand the backup job the +// key to read everything else in the estate. +path "thermograph/data/legacy/age" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} diff --git a/infra/openbao/seed-from-sops.sh b/infra/openbao/seed-from-sops.sh new file mode 100755 index 0000000..09f32cc --- /dev/null +++ b/infra/openbao/seed-from-sops.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# Seed OpenBao from the SOPS vault. One-shot, idempotent, run BY THE OPERATOR. +# +# infra/openbao/seed-from-sops.sh --dry-run # key names only, writes nothing +# infra/openbao/seed-from-sops.sh --env dev # seed one environment +# infra/openbao/seed-from-sops.sh --all # seed everything +# +# ============================================================================ +# THIS SCRIPT READS EVERY PRODUCTION SECRET IN THE ESTATE IN PLAINTEXT. +# ============================================================================ +# infra/CLAUDE.md and deploy/secrets/README.md both say that +# deploy/secrets/seed-from-live.sh "reads production secrets and is explicitly NOT for +# an agent to run". This script is in the same category and inherits the same rule: +# run it yourself, from your own terminal, with your own age key. +# +# It never prints a value. Every diagnostic is key NAMES and counts, so the output is +# safe to paste into an issue. --dry-run makes that guarantee mechanical: it resolves +# and validates everything, then reports what it WOULD write without contacting +# OpenBao at all. +# +# Why seed from SOPS rather than re-entering values by hand: the safety property of +# this entire migration is that the OpenBao render can be diffed BYTE-FOR-BYTE against +# the SOPS render (see verify-parity.sh). That only works if the values are identical. +# Re-typing 61 values would silently rotate whichever ones were mistyped, and the +# self-generating ones (AUTH_SECRET, the VAPID pair) fail SILENTLY when wrong — the app +# mints a replacement, comes up green, and invalidates every session and push +# subscription. So: copy exactly now, rotate deliberately later. +set -euo pipefail + +MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}" +AGE_KEY="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}" +DRY_RUN=0 +TARGETS=() + +# Resolve the repo's infra/ directory from this script's own location, so the script +# works from any cwd. This is the same mistake terraform's remote-exec makes +# (main.tf:256 passes app_dir where app_dir/infra is wanted), so it is worth being +# explicit rather than clever. +SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)" +VAULT_DIR="$INFRA_DIR/deploy/secrets" + +usage() { + cat >&2 <<'EOF' +usage: seed-from-sops.sh [--dry-run] (--all | --env ...) + + --dry-run validate and report key names/counts; contact OpenBao not at all + --all seed common, dev, beta, prod and centralis.prod + --env seed one of: common | dev | beta | prod | centralis.prod + +Environment: + SOPS_AGE_KEY_FILE age private key (default ~/.config/sops/age/keys.txt) + BAO_ADDR OpenBao address (default https://10.10.0.1:8200) + BAO_TOKEN an operator token with write on /data/* + THERMOGRAPH_BAO_MOUNT KV v2 mount name (default thermograph) +EOF + exit 2 +} + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN=1; shift ;; + --all) TARGETS=(common dev beta prod centralis.prod); shift ;; + --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;; + -h|--help) usage ;; + *) echo "!! unknown argument: $1" >&2; usage ;; + esac +done + +[ ${#TARGETS[@]} -gt 0 ] || usage + +# The KV v2 path for a given vault file. common.yaml and centralis.prod.yaml are +# special-cased; everything else is an environment overlay. +# +# Keeping `common` at its own top-level path rather than under env/ is what makes the +# dev policy expressible: tg-host-dev.hcl denies exactly one path, and no wildcard +# over env/* can accidentally re-include it. +bao_path_for() { + case "$1" in + common) printf 'common\n' ;; + centralis.prod) printf 'centralis/prod\n' ;; + dev|beta|prod) printf 'env/%s\n' "$1" ;; + *) echo "!! unknown vault target: $1" >&2; return 1 ;; + esac +} + +[ -f "$AGE_KEY" ] || { echo "!! age key not found at $AGE_KEY" >&2; exit 1; } +command -v sops >/dev/null 2>&1 || { echo "!! sops not on PATH" >&2; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "!! python3 not on PATH" >&2; exit 1; } + +if [ "$DRY_RUN" = 0 ]; then + command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; } + [ -n "${BAO_TOKEN:-}" ] || { echo "!! BAO_TOKEN is not set" >&2; exit 1; } + export BAO_ADDR="${BAO_ADDR:-https://10.10.0.1:8200}" +fi + +rc=0 +for target in "${TARGETS[@]}"; do + src="$VAULT_DIR/${target}.yaml" + if [ ! -f "$src" ]; then + echo "!! no vault file at $src" >&2 + rc=1 + continue + fi + + path="$(bao_path_for "$target")" || { rc=1; continue; } + + # Decrypt to JSON, not dotenv. JSON is lossless: the dotenv writer flattens a value + # containing a newline into a literal \n that is indistinguishable from a backslash + # followed by n (render-secrets.sh:198 documents this). Seeding through dotenv would + # bake that ambiguity into OpenBao permanently. + plain_json="$(SOPS_AGE_KEY_FILE="$AGE_KEY" \ + sops -d --input-type yaml --output-type json "$src" 2>/dev/null)" || { + echo "!! failed to decrypt $src" >&2 + rc=1 + continue + } + + # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe + # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at + # SEED time — before anything depends on it — rather than at deploy time. + reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY' +import json, os, re, sys + +doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"]) +if not isinstance(doc, dict) or not doc: + sys.stderr.write("!! vault did not decrypt to a non-empty object\n") + sys.exit(1) + +KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +UNSAFE = set("`$\\\"'") + +out, bad = {}, [] +for key, val in doc.items(): + if key == "sops": + continue + if not KEY_RE.match(key): + bad.append(f"{key}: not a valid env var name") + continue + if val is None: + val = "" + if not isinstance(val, str): + val = json.dumps(val, separators=(",", ":")) + if "\n" in val or "\r" in val: + bad.append(f"{key}: contains a newline") + continue + if UNSAFE & set(val): + bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") + continue + out[key] = val + +if bad: + sys.stderr.write("!! unsafe values, refusing:\n") + for line in bad: + sys.stderr.write(f"!! {line}\n") + sys.exit(1) + +# stdout line 1: space-separated key NAMES, for the operator-visible log. +# stdout line 2+: the JSON payload, fed to `bao kv put` on stdin. +sys.stdout.write(" ".join(sorted(out)) + "\n") +sys.stdout.write(json.dumps(out) + "\n") +PY +)" || { echo "!! validation failed for $target" >&2; rc=1; continue; } + + names="$(printf '%s\n' "$reshaped" | sed -n '1p')" + payload="$(printf '%s\n' "$reshaped" | sed -n '2p')" + count="$(printf '%s\n' "$names" | wc -w | tr -d ' ')" + + if [ "$DRY_RUN" = 1 ]; then + echo "== ${target}.yaml -> ${MOUNT}/${path} (${count} keys, DRY RUN, nothing written)" + printf ' %s\n' "$names" + continue + fi + + # `bao kv put -` reads a JSON object from stdin, so no value ever appears in argv + # (where it would be visible in /proc and in the shell's history). + if printf '%s' "$payload" | bao kv put -mount="$MOUNT" "$path" - >/dev/null; then + echo "== ${target}.yaml -> ${MOUNT}/${path} (${count} keys written)" + else + echo "!! failed writing ${MOUNT}/${path}" >&2 + rc=1 + fi +done + +if [ "$rc" = 0 ] && [ "$DRY_RUN" = 0 ]; then + cat <&2 <<'EOF' +usage: verify-parity.sh (--all | --env ...) + +Renders each environment through BOTH backends and compares the resulting +KEY=value sets. Prints key names and counts only, never values. + +Exit status: 0 = every requested environment is at parity; 1 = a mismatch. +Suitable as a CI / cron gate. +EOF + exit 2 +} + +while [ $# -gt 0 ]; do + case "$1" in + --all) TARGETS=(dev beta prod); shift ;; + --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;; + -h|--help) usage ;; + *) echo "!! unknown argument: $1" >&2; usage ;; + esac +done +[ ${#TARGETS[@]} -gt 0 ] || usage + +# shellcheck source=/dev/null +. "$INFRA_DIR/deploy/render-secrets-openbao.sh" + +overall=0 +for env_name in "${TARGETS[@]}"; do + echo "== parity check: $env_name" + + # dev renders WITHOUT common on both backends. On the SOPS side that is a caller + # flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps + # the comparison apples-to-apples — and if the flag were wrong, the OpenBao side + # would fail closed with a 403 rather than quietly diverge, which is exactly the + # improvement being verified. + skip_common=0 + [ "$env_name" = dev ] && skip_common=1 + + sops_out=$(mktemp); bao_out=$(mktemp) + # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a + # trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a + # sourced context persists into the caller's shell and re-fires on the next source). + cleanup() { rm -f "$sops_out" "$bao_out"; } + + # --- SOPS side: common (unless dev) then , appended, last-wins --- + key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" + [ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}" + if [ ! -r "$key" ]; then + echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2 + cleanup; overall=1; continue + fi + + if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then + SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \ + "$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || { + echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; } + fi + SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \ + "$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || { + echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; } + + # --- OpenBao side --- + if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \ + thermograph_openbao_source "$env_name" > "$bao_out"; then + echo "!! OpenBao render failed for $env_name" >&2 + cleanup; overall=1; continue + fi + + # --- Compare --- + # The SOPS output can legitimately contain the SAME key twice (common then env), and + # every consumer takes the LAST occurrence — `env_file:` and `source` both do. So + # collapsing to last-wins is not a normalisation for convenience, it is what the + # consumers actually see. Comparing raw would report a false mismatch the moment a + # key is overridden. + norm() { + python3 -c ' +import sys +seen = {} +order = [] +for line in open(sys.argv[1], encoding="utf-8", errors="replace"): + line = line.rstrip("\n") + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if k not in seen: + order.append(k) + seen[k] = v # last occurrence wins +for k in sorted(order): + print(f"{k}={seen[k]}") +' "$1" + } + + a=$(mktemp); b=$(mktemp) + norm "$sops_out" > "$a"; norm "$bao_out" > "$b" + + n_sops=$(wc -l < "$a" | tr -d ' ') + n_bao=$(wc -l < "$b" | tr -d ' ') + + if cmp -s "$a" "$b"; then + echo " PASS — ${n_sops} keys, byte-identical after last-wins collapse" + else + overall=1 + echo " FAIL — sops=${n_sops} keys, openbao=${n_bao} keys" + # Report only NAMES. Three buckets, because the fix differs per bucket: + # only-in-sops -> the seed missed a key (re-run seed-from-sops.sh) + # only-in-bao -> OpenBao has something the vault does not (stale/extra write) + # value-differs -> the seed captured a different value; DO NOT flip the backend + comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in SOPS: /' + comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in OpenBao: /' + join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \ + | awk -F= '$2 != $3 { print " value differs: " $1 }' | sort -u + fi + + rm -f "$a" "$b" + cleanup +done + +if [ "$overall" = 0 ]; then + echo + echo "PARITY OK for: ${TARGETS[*]}" + echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments." + echo "Recommended gate before prod: 7 consecutive green runs on all three." +else + echo + echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2 +fi +exit "$overall" -- 2.45.2 From 7878763b66c06789760bb15bba5c44f253e421de Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Wed, 29 Jul 2026 21:42:01 -0700 Subject: [PATCH 31/37] forgejo: restart the db service on a clean exit, not just on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forgejo_db has been at 0/1 replicas since 2026-07-29 01:24 UTC. Postgres hit an invalid data-directory lock file ("could not open file postmaster.pid ... performing immediate shutdown because data directory lock file is invalid") and exited 0. With restart_policy.condition=on-failure, Swarm read the zero status as successful completion, marked the task Complete, and never rescheduled it. The forgejo service itself stayed Up and kept serving its homepage, so the outage presented as every repository page, the whole API and all CI returning 500 with "dial tcp: lookup db on 127.0.0.11:53: no such host" — including the auth path, which is why API calls reported "user does not exist [uid: 0]" rather than a database error. on-failure cannot distinguish "finished successfully" from "shut itself down and should be restarted", and Postgres exits 0 on several such paths, so it is the wrong policy for an always-on stateful service. This is the durable fix; it does not restart the currently stopped task. --- infra/deploy/forgejo/docker-stack.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/infra/deploy/forgejo/docker-stack.yml b/infra/deploy/forgejo/docker-stack.yml index a86d41b..b939025 100644 --- a/infra/deploy/forgejo/docker-stack.yml +++ b/infra/deploy/forgejo/docker-stack.yml @@ -53,7 +53,19 @@ services: cpus: "${FORGEJO_DB_CPUS:-1}" memory: ${FORGEJO_DB_MEMORY:-1g} restart_policy: - condition: on-failure + # `any`, NOT `on-failure`. This took Forgejo down for 27 hours on 2026-07-29: + # Postgres hit an invalid data-directory lock file ("could not open file + # postmaster.pid ... performing immediate shutdown") and exited **0**. A clean + # exit is not a failure, so Swarm considered the task Complete, dropped the + # service to 0/1 replicas, and never rescheduled it. Forgejo itself stayed Up + # and served its homepage while every repo page, the API and all CI returned + # 500 with `dial tcp: lookup db ... no such host`. + # + # `on-failure` is the wrong policy for any always-on stateful service: it + # cannot distinguish "finished successfully" from "shut itself down and should + # be restarted", and Postgres does the latter with status 0 on several paths. + condition: any + delay: 5s forgejo: image: codeberg.org/forgejo/forgejo:9-rootless -- 2.45.2 From 8a2c838663a1012996ac35121c48c6dfd8a3632b Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Wed, 29 Jul 2026 21:42:01 -0700 Subject: [PATCH 32/37] forgejo: restart the db service on a clean exit, not just on failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forgejo_db has been at 0/1 replicas since 2026-07-29 01:24 UTC. Postgres hit an invalid data-directory lock file ("could not open file postmaster.pid ... performing immediate shutdown because data directory lock file is invalid") and exited 0. With restart_policy.condition=on-failure, Swarm read the zero status as successful completion, marked the task Complete, and never rescheduled it. The forgejo service itself stayed Up and kept serving its homepage, so the outage presented as every repository page, the whole API and all CI returning 500 with "dial tcp: lookup db on 127.0.0.11:53: no such host" — including the auth path, which is why API calls reported "user does not exist [uid: 0]" rather than a database error. on-failure cannot distinguish "finished successfully" from "shut itself down and should be restarted", and Postgres exits 0 on several such paths, so it is the wrong policy for an always-on stateful service. This is the durable fix; it does not restart the currently stopped task. --- infra/deploy/forgejo/docker-stack.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/infra/deploy/forgejo/docker-stack.yml b/infra/deploy/forgejo/docker-stack.yml index a86d41b..b939025 100644 --- a/infra/deploy/forgejo/docker-stack.yml +++ b/infra/deploy/forgejo/docker-stack.yml @@ -53,7 +53,19 @@ services: cpus: "${FORGEJO_DB_CPUS:-1}" memory: ${FORGEJO_DB_MEMORY:-1g} restart_policy: - condition: on-failure + # `any`, NOT `on-failure`. This took Forgejo down for 27 hours on 2026-07-29: + # Postgres hit an invalid data-directory lock file ("could not open file + # postmaster.pid ... performing immediate shutdown") and exited **0**. A clean + # exit is not a failure, so Swarm considered the task Complete, dropped the + # service to 0/1 replicas, and never rescheduled it. Forgejo itself stayed Up + # and served its homepage while every repo page, the API and all CI returned + # 500 with `dial tcp: lookup db ... no such host`. + # + # `on-failure` is the wrong policy for any always-on stateful service: it + # cannot distinguish "finished successfully" from "shut itself down and should + # be restarted", and Postgres does the latter with status 0 on several paths. + condition: any + delay: 5s forgejo: image: codeberg.org/forgejo/forgejo:9-rootless -- 2.45.2 From 702bcd920cb4bce62ca654c40830bc8e5cd6d69b Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 30 Jul 2026 19:47:22 -0700 Subject: [PATCH 33/37] openbao: fix bootstrap so it completes on 2.6.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bootstrap.sh could not run to completion on OpenBao 2.6.1: - Release asset names were wrong. `bao__linux_amd64.tar.gz` and `bao__SHA256SUMS` both 404; the assets are `openbao__...` and `checksums.txt`. The tarball is now saved under its real name and the checksum grep anchored to end-of-line, because checksums.txt also lists a .sbom.json and `sha256sum -c` resolves each line by the filename inside it. - `openssl rand -base64 32` appends a newline and the static seal reads the key file raw, so the service failed with `Error configuring seal "static": unknown encoding for AES-256 key`. - The audit stanza relied on the block label being the device type. 2.6.1 requires explicit `type` and `path` with device settings under `options`. Worth noting the intermediate state: with type and path but a bare `file_path`, the server starts and silently ignores the log location, which is the worse failure given a wedged audit device stops OpenBao answering at all. - `disable_mlock` is unsupported in 2.6.1 and warned on every start. - Nothing opened port 8200 and ufw defaults to deny(incoming), so vps1 could not reach the vault. dev renders on vps1, so this would have surfaced as a timeout during cutover rather than here. bootstrap-policies.sh installed beta's credentials as `deploy:deploy`, but vps2 has no `deploy` user and both environments deploy as `agent` (env-topology.sh sets TG_SSH_TARGET=agent@ for both; deploy.yml uses one VPS2_SSH_USER). install_creds also printed a success line after a failed chown: the call site wrapped it in `|| echo`, which suppresses `set -e` for everything inside the function, so it fell through to chmod and reported an ownership it never applied. It now removes the half-written file and returns non-zero. Corrects the isolation rationale accordingly — separate credentials buy audit attribution and a policy boundary against mistakes, not deploy-user isolation. Documents the 2.6.0 root-token change (HCSEC-2026-08): `generate-root` now targets an authenticated endpoint, so recovery keys cannot mint a replacement token and revoking the last root token before a second admin identity exists is a one-way door. Also corrects the runbook's `verify-parity.sh --all`, which cannot work from a single host given the AppRole CIDR bindings. --- infra/openbao/README.md | 41 +++++++++++++-- infra/openbao/bootstrap-policies.sh | 37 +++++++++---- infra/openbao/bootstrap.sh | 80 +++++++++++++++++++++++------ infra/openbao/config/openbao.hcl | 24 +++++++-- 4 files changed, 147 insertions(+), 35 deletions(-) diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 196784a..037bfcb 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -148,22 +148,53 @@ SOPS and retiring age are two different projects. ### Stand it up (once, by the operator, on vps2) ```sh -sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init +sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, ufw, unit, init # custody the recovery keys + /etc/openbao/unseal.key OFF the box, then: -export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt +export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt export BAO_TOKEN= sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles -bao token revoke -self ``` -### Seed and prove (from your own machine — it needs your age key) +> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated +> `/sys/generate-root` with an authenticated `/sys/generate-root-token` +> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so +> minting a root token requires a token you already have. **The recovery keys are not +> a way back in**; they rekey, they do not authenticate. Unless +> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking +> the last root token locks you out of an otherwise healthy, unsealed vault, and the +> only remedy is re-initialising from scratch. Revoke only once a second admin +> identity exists. + +Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than +`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700 +openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails +with `certificate signed by unknown authority`. + +### Seed and prove (on vps2 — the age key is already there) ```sh +export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing infra/openbao/seed-from-sops.sh --all -infra/openbao/verify-parity.sh --all # MUST pass before any flip ``` +Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs` +to the host that legitimately renders it, so an environment can only be verified from +its own host: + +```sh +# on vps2 +infra/openbao/verify-parity.sh --env prod # expect 32 keys +THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \ + infra/openbao/verify-parity.sh --env beta # expect 24 keys +# on vps1 +cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys +``` + +The explicit `THERMOGRAPH_BAO_APPROLE` for beta is a stopgap: `env-topology.sh` does +not yet derive the AppRole path per environment, so the renderer falls back to prod's +credentials and `tg-host-prod.hcl` correctly denies `env/beta`. + `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md` the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this inherits that rule. diff --git a/infra/openbao/bootstrap-policies.sh b/infra/openbao/bootstrap-policies.sh index 81d07bd..2104e51 100755 --- a/infra/openbao/bootstrap-policies.sh +++ b/infra/openbao/bootstrap-policies.sh @@ -63,15 +63,21 @@ add_role tg-centralis tg-centralis 10.10.0.1/32 # Install the local (vps2) credentials. prod and beta both render on this box. # -# The OWNERSHIP here is a real boundary that does not exist today, and it is the one -# concrete isolation win available on a shared host. render-secrets.sh:119-124 records -# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the -# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation -# at all. Giving each environment its own 0400 secret-id owned by its own deploy user -# creates one. +# Both files are owned by `agent`, because that is the single identity both +# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for +# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no +# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's +# file to `deploy:deploy` on the theory that beta and prod deploy as different users; +# they do not, and the chown simply failed. # -# Be honest about the limit: root on vps2 reads both files, exactly as root today reads -# the one age key. This defends against MISCONFIGURATION, not against intrusion. +# So be accurate about what separate credentials buy on this box: NOT deploy-user +# isolation — there is one deploy user, and it can read both files. What they buy is +# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a +# *mistake* crossing between environments: a beta render authenticating with beta's +# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong +# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's +# database password. Real deploy-user isolation would need a `deploy` user on vps2 and +# a separate SSH credential for beta in deploy.yml — a larger change than this script. install_creds() { local role="$1" dest="$2" owner="$3" local rid sid @@ -79,7 +85,17 @@ install_creds() { sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id") # umask before creation, so the file is never briefly world-readable. ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" ) - chown "$owner" "$dest" + # Abort the function if chown fails, and remove the half-installed file. The call + # site wraps this in `|| echo`, which suppresses `set -e` for everything inside — + # so without this check a failed chown falls straight through to chmod and prints a + # line asserting an ownership that was never applied. A root-owned credential the + # renderer cannot read, reported as installed, is worse than no credential at all: + # it fails at deploy time instead of here. + if ! chown "$owner" "$dest"; then + rm -f "$dest" + echo " !! chown $owner failed for $dest — removed, nothing installed" >&2 + return 1 + fi chmod 0400 "$dest" echo " installed $dest (0400 $owner)" } @@ -87,8 +103,7 @@ install_creds() { if [ "$(id -u)" = 0 ]; then install -d -o root -g root -m 0755 /etc/thermograph install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent" - install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \ - || echo " (beta creds skipped — no 'deploy' user on this box?)" + install_creds tg-beta /etc/thermograph/openbao-approle-beta "agent:agent" install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root" else echo "!! not root; skipping credential install. Re-run with sudo -E." >&2 diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh index 39fc5e6..f3d0f69 100755 --- a/infra/openbao/bootstrap.sh +++ b/infra/openbao/bootstrap.sh @@ -52,13 +52,19 @@ if ! command -v bao >/dev/null 2>&1; then # The checksums file is signed by the release; verify before installing. tmpd="$(mktemp -d)" base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}" - curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz" - curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS" - ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || { + # Asset names are `openbao__...`, and the checksums file is `checksums.txt`. + # The previous `bao__...` / `bao__SHA256SUMS` names both 404. + tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz" + curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}" + curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt" + # Anchor to end-of-line and save under the real asset name: checksums.txt also lists + # .sbom.json, and `sha256sum -c` resolves each line by the filename IN it, + # so an unanchored match fails on a file that was never fetched. + ( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || { echo "!! checksum mismatch on the bao tarball; refusing to install" >&2 rm -rf "$tmpd"; exit 1 } - tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao + tar -xzf "$tmpd/$tarball" -C "$tmpd" bao install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao rm -rf "$tmpd" fi @@ -88,7 +94,11 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt echo "==> 3/7 static seal key" if [ ! -f /etc/openbao/unseal.key ]; then # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key. - ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key ) + # tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the + # static seal reads this file RAW. That one trailing byte makes the key neither + # 32 raw bytes nor clean base64, so bao refuses to start with + # `Error configuring seal "static": unknown encoding for AES-256 key`. + ( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key ) chown openbao:openbao /etc/openbao/unseal.key chmod 0400 /etc/openbao/unseal.key echo " generated /etc/openbao/unseal.key (0400 openbao)" @@ -125,6 +135,24 @@ systemctl is-active --quiet openbao.service || { exit 1 } +# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but +# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without +# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao +# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the +# AppRole CIDR bindings are the real control, and nothing else on the mesh — the +# desktop, the phone — has any business reaching the secret store. +if command -v ufw >/dev/null 2>&1; then + if ufw status | grep -q '8200.*10\.10\.0\.2'; then + echo " ufw: 8200/tcp from 10.10.0.2 already allowed" + else + ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \ + comment 'vps1/dev -> openbao' >/dev/null + echo " ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)" + fi +else + echo " !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2 +fi + export BAO_ADDR="https://127.0.0.1:8200" export BAO_CACERT=/etc/openbao/tls/bao.crt @@ -140,9 +168,16 @@ else echo " ############################################################" echo # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init - # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal - # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one - # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token. + # yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here + # is redundancy against loss, not separation of duty — there is one operator. + # + # These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced + # the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token + # (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so + # minting a root token requires a token you already have. The deprecated endpoint is + # off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl. + # Consequence: revoking the last root token before another admin identity exists is + # a one-way door. See the ordering note in the handoff below. bao operator init -recovery-shares=3 -recovery-threshold=2 echo echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are" @@ -156,22 +191,37 @@ cat < 6/7 and 7/7 need a root token, which is NOT stored anywhere by design. -Paste the initial root token (or one minted from recovery keys) and run: +Paste the initial root token printed above and run: export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt export BAO_TOKEN= bash ${SELF_DIR}/bootstrap-policies.sh +KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys +cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature +\`bao token revoke -self\` locks you out of an otherwise healthy vault. + That script enables the KV mount, writes the policies, creates the AppRoles and -installs each host's credentials. Then, from YOUR OWN machine (it needs your age -key, and per infra/CLAUDE.md a script that reads production secrets is not for an -agent to run): +installs each host's credentials. Then seed — on THIS box, where the age key already +lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production +secrets is not for an agent to run: - infra/openbao/seed-from-sops.sh --dry-run # check first + export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key + infra/openbao/seed-from-sops.sh --dry-run # check first: 16/12/8/16 keys infra/openbao/seed-from-sops.sh --all - infra/openbao/verify-parity.sh --all # must PASS before any flip -Then revoke the root token: bao token revoke -self +Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host +that legitimately renders it, so an environment can only be verified from its own host. + + # here (vps2) -- beta needs its own AppRole, prod uses the default + infra/openbao/verify-parity.sh --env prod # expect 32 keys + THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \\ + infra/openbao/verify-parity.sh --env beta # expect 24 keys + # on vps1 + cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys + +Only once all three PASS, and only after you have a second admin identity that is not +the root token, revoke it: bao token revoke -self NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative diff --git a/infra/openbao/config/openbao.hcl b/infra/openbao/config/openbao.hcl index 07f4d87..b9175de 100644 --- a/infra/openbao/config/openbao.hcl +++ b/infra/openbao/config/openbao.hcl @@ -25,8 +25,12 @@ ui = false cluster_name = "thermograph" -disable_mlock = true log_level = "info" +// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and +// logs `unknown or unsupported field disable_mlock` on every start. Carrying a +// setting the server ignores only trains the operator to skim startup warnings, +// which is where a real one will eventually hide. The corresponding note about +// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason. // STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0 // DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys @@ -110,9 +114,21 @@ seal "static" { // // logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an // unlinked inode and the log silently stops growing. +// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the +// device NAME, not its type — and device settings go in `options`. Written as +// `audit "file" { file_path = ... }` the server refuses to start ("audit type must +// be specified"); with type+path but a bare file_path it starts but silently +// IGNORES the path and mode, which is the worse failure of the two. audit "file" { - file_path = "/var/log/openbao/audit.log" - mode = "0600" + type = "file" + path = "file/" + options = { + file_path = "/var/log/openbao/audit.log" + mode = "0600" + } } -audit "syslog" {} +audit "syslog" { + type = "syslog" + path = "syslog/" +} -- 2.45.2 From 5001269b374c35cab5cd2b6984680dd9c29bbad7 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 30 Jul 2026 19:49:28 -0700 Subject: [PATCH 34/37] openbao: derive the AppRole credential path per environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit render-secrets-openbao.sh resolved the AppRole file to a single host-wide default, /etc/thermograph/openbao-approle — which is prod's. On vps2 that made beta's render authenticate as tg-prod, and tg-host-prod.hcl denies thermograph/data/env/beta, so beta could never render from OpenBao. It surfaces as `cannot read thermograph/env/beta`: the policy working correctly against the wrong identity. env-topology.sh exists precisely because vps2 runs two environments and one host-wide marker cannot answer "which environment is this". The AppRole path is the same question and had been given the same host-wide answer. It is now derived per environment alongside host, checkout, branch, ports and DB role, and the renderer resolves THERMOGRAPH_BAO_APPROLE, then TG_BAO_APPROLE, then the conventional path — the same precedence render-secrets.sh:44 already uses for the backend selector. Only beta differs. prod and dev keep the conventional path; dev is alone on vps1 so there is no collision there. Beta parity passes when the path is supplied by hand (24 keys = 8 + 16 shared); this makes it automatic. Derivation confirmed by sourcing env-topology.sh for all three environments. --- infra/deploy/env-topology.sh | 16 +++++++++++++++- infra/deploy/render-secrets-openbao.sh | 8 +++++++- infra/openbao/README.md | 9 ++++----- infra/openbao/bootstrap.sh | 5 ++--- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh index 9e2b7c9..d57d148 100644 --- a/infra/deploy/env-topology.sh +++ b/infra/deploy/env-topology.sh @@ -76,6 +76,16 @@ thermograph_topology() { # for the reason that marker was demoted to a fallback in the first place: vps2 runs # prod and beta side by side, and one host-wide file cannot name two backends. TG_SECRETS_BACKEND=sops + # Which AppRole credential file render-secrets-openbao.sh authenticates with. Same + # reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO + # environments, so a single host-wide default cannot serve both. prod and dev each + # get the conventional path (dev is alone on vps1, so there is no collision there); + # beta is the one that must differ, because it shares a filesystem with prod. + # + # Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl + # correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that + # runs prod. + TG_BAO_APPROLE=/etc/thermograph/openbao-approle case "$env_name" in prod) @@ -116,6 +126,10 @@ thermograph_topology() { # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh). TG_SSH_HOST=169.58.46.181 TG_SSH_TARGET=agent@169.58.46.181 + # The one environment that cannot use the default AppRole path: it shares a + # filesystem with prod, so it needs its own credential file to authenticate as + # tg-beta rather than tg-prod. bootstrap-policies.sh installs it here. + TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta # A SECOND checkout on the same box. Separate from prod's so the two can # sit on different commits of this repo, and so `git reset --hard` in one # deploy can never yank the tree out from under the other's running @@ -228,7 +242,7 @@ thermograph_topology() { export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY - export TG_SECRETS_BACKEND + export TG_SECRETS_BACKEND TG_BAO_APPROLE export TG_SSH_HOST TG_SSH_TARGET } diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh index 82c449c..aaf606d 100644 --- a/infra/deploy/render-secrets-openbao.sh +++ b/infra/deploy/render-secrets-openbao.sh @@ -27,7 +27,13 @@ thermograph_openbao_source() { local env_name="${1:?env name required}" local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" - local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}" + # Explicit override first, then the per-environment value env-topology.sh derives, + # then the conventional path. Same precedence shape as render-secrets.sh:44's + # THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the + # THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the + # deploy path sets. Falling straight through to the bare default is what made beta + # authenticate as tg-prod and get denied its own path. + local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}" local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" command -v bao >/dev/null 2>&1 || { diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 037bfcb..855e4d7 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -185,15 +185,14 @@ its own host: ```sh # on vps2 infra/openbao/verify-parity.sh --env prod # expect 32 keys -THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \ - infra/openbao/verify-parity.sh --env beta # expect 24 keys +infra/openbao/verify-parity.sh --env beta # expect 24 keys # on vps1 cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys ``` -The explicit `THERMOGRAPH_BAO_APPROLE` for beta is a stopgap: `env-topology.sh` does -not yet derive the AppRole path per environment, so the renderer falls back to prod's -credentials and `tg-host-prod.hcl` correctly denies `env/beta`. +Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per +environment, so beta authenticates as `tg-beta` rather than falling back to prod's +credentials and being denied its own path by `tg-host-prod.hcl`. `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md` the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh index f3d0f69..6fa199d 100755 --- a/infra/openbao/bootstrap.sh +++ b/infra/openbao/bootstrap.sh @@ -213,10 +213,9 @@ secrets is not for an agent to run: Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host that legitimately renders it, so an environment can only be verified from its own host. - # here (vps2) -- beta needs its own AppRole, prod uses the default + # here (vps2) infra/openbao/verify-parity.sh --env prod # expect 32 keys - THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \\ - infra/openbao/verify-parity.sh --env beta # expect 24 keys + infra/openbao/verify-parity.sh --env beta # expect 24 keys # on vps1 cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys -- 2.45.2 From 657e27ba9f19b9c8e18f3a85677995b56df5d307 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 30 Jul 2026 19:55:17 -0700 Subject: [PATCH 35/37] openbao: render Centralis from the vault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Centralis could not migrate at all. `seed-from-sops.sh --all` refused centralis.prod with `CENTRALIS_TOKENS: contains a shell metacharacter`, and render-secrets-openbao.sh had no Centralis path whatsoever — so the seeded data would have had nothing to read it. Both halves come from applying the app stack's rules to a file that does not share its shape. /etc/thermograph.env is raw unquoted KEY=value. /etc/centralis.env is consumed by exactly one thing and that thing is a shell: sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up' CENTRALIS_TOKENS is JSON, so it can never satisfy a no-metacharacters rule, and rendering it unquoted is precisely the 2026-07-24 failure: bash strips the quotes, Centralis fails closed on the malformed registry and serves a single `shared` identity, indistinguishable from the file never being written. So: - seed-from-sops.sh validates against the renderer that will consume the path. The dotenv rules still apply to common and env/*; centralis/* is exempt because it is single-quoted downstream. - render-secrets-openbao.sh gains thermograph_render_openbao_centralis, mirroring render_centralis_secrets: fetch as JSON, emit POSIX single-quoted assignments, then PROVE the file by sourcing it under `env -i` with `set -a` and comparing every value back before anything touches the destination. Mirrored rather than shared, per the existing note above thermograph_render_openbao — consolidate when the SOPS path is deleted. - render_centralis_secrets gains the same early-return backend dispatch the app path already uses, so the new function is reachable. Without it the SOPS path was the only path regardless of TG_SECRETS_BACKEND. Verified against a synthetic registry containing a JSON token map, an apostrophe, and `$(...)`/backtick/backslash/double-quote payloads: 5 keys round-trip byte-for-byte through `set -a; . file`, CENTRALIS_TOKENS parses to the right subjects, and the command substitution does not execute. The negative case is covered too — fed the unquoted 2026-07-24 shape, the round-trip check reports the value as changed (by length, never content) and refuses to write. Also corrects the claim in render-secrets-openbao.sh that the SOPS path never tripped the metacharacter hazard "because every current value happens to be alphanumeric". CENTRALIS_TOKENS is the counter-example; it never tripped those paths because it was never in them. --- infra/deploy/render-secrets-openbao.sh | 260 ++++++++++++++++++++++++- infra/deploy/render-secrets.sh | 24 +++ infra/openbao/README.md | 16 ++ infra/openbao/seed-from-sops.sh | 28 ++- 4 files changed, 315 insertions(+), 13 deletions(-) diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh index aaf606d..518f999 100644 --- a/infra/deploy/render-secrets-openbao.sh +++ b/infra/deploy/render-secrets-openbao.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash # OpenBao source backend for render-secrets.sh. Sourced, never run directly. # -# Two functions: -# thermograph_openbao_source -> merged dotenv on STDOUT -# thermograph_render_openbao -> source, then write +# Functions: +# thermograph_openbao_source -> merged dotenv on STDOUT +# thermograph_render_openbao -> source, then write +# thermograph_render_openbao_centralis [out] -> quoted /etc/centralis.env +# +# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The +# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted — +# a different shape for a different consumer, not an inconsistency. # # render_thermograph_secrets in render-secrets.sh dispatches to the second when # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path @@ -113,9 +118,16 @@ thermograph_openbao_source() { # /etc/thermograph.env is sourced by bash in six places (deploy.sh:133, # deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so # `$(...)` or a backtick in a secret is a live command-execution path on the - # deploy host. The SOPS path has always had this hazard and has never tripped it - # only because every current value happens to be alphanumeric. Refusing here - # turns a latent code-execution bug into a loud render failure. + # deploy host. Refusing here turns a latent code-execution bug into a loud + # render failure. + # + # An earlier version of this comment claimed the SOPS path had never tripped + # this "only because every current value happens to be alphanumeric". That is + # false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never + # tripped THESE paths because it is not in them — it lives at + # centralis/ and renders to /etc/centralis.env, which is single-quoted + # precisely because raw dotenv cannot carry it. See + # thermograph_render_openbao_centralis at the end of this file. THERMOGRAPH_BAO_COMMON="$common_json" \ THERMOGRAPH_BAO_ENV="$env_json" \ python3 - <<'PY' @@ -243,3 +255,239 @@ thermograph_render_openbao() { rm -f "$tmp" return "$rc" } + +# --------------------------------------------------------------------------- +# CENTRALIS +# --------------------------------------------------------------------------- +# /etc/centralis.env is a different file, with a different consumer, and therefore a +# different output shape. render-secrets.sh:206-233 carries the full argument; the +# short version is that it is consumed by exactly one thing and that thing is a shell: +# +# sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up' +# +# `set -a; . file` runs every line through bash's whole expansion pipeline — quote +# removal, parameter expansion, command substitution, field splitting. Raw dotenv is +# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered +# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result +# by serving a single `shared` identity — indistinguishable from the file never having +# been written. That is the 2026-07-24 incident. So this path emits POSIX +# single-quoted assignments and PROVES them by sourcing the result in a clean shell +# before anything touches $dest. +# +# This mirrors render_centralis_secrets rather than sharing with it, for the same +# reason given above thermograph_render_openbao: the SOPS path stays byte-identical +# and all new risk stays in this file. Consolidate the two when the SOPS path is being +# deleted anyway, not before. + +# _thermograph_openbao_login -> token on STDOUT +# +# Separate from the login inline in thermograph_openbao_source deliberately: that +# function is now exercised by prod and beta parity and is left untouched. +_thermograph_openbao_login() { + local approle="$1" addr="$2" ca="$3" + local creds role_id secret_id + if [ -r "$approle" ]; then + creds=$(cat "$approle") + else + creds=$(sudo cat "$approle" 2>/dev/null || true) + fi + [ -n "$creds" ] || { + echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2 + return 1 + } + role_id=$(printf '%s\n' "$creds" | sed -n '1p') + secret_id=$(printf '%s\n' "$creds" | sed -n '2p') + [ -n "$role_id" ] && [ -n "$secret_id" ] || { + echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2 + return 1 + } + export BAO_ADDR="$addr" + [ -f "$ca" ] && export BAO_CACERT="$ca" + bao write -field=token auth/approle/login \ + role_id="$role_id" secret_id="$secret_id" 2>/dev/null +} + +# thermograph_render_openbao_centralis [dest] +thermograph_render_openbao_centralis() { + local env_name="${1:?env name required}" + local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}" + local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" + local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" + local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" + local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}" + + command -v bao >/dev/null 2>&1 || { + echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2 + return 1 + } + command -v python3 >/dev/null 2>&1 || { + echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2 + return 1 + } + + echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})" + + local token + token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1 + [ -n "$token" ] || { + echo "!! OpenBao AppRole login failed for centralis" >&2 + echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2 + return 1 + } + export BAO_TOKEN="$token" + + # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every + # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap + # persists into the caller's shell and re-fires on its next `source`. + local work rc=0 + work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; } + chmod 700 "$work" + + # Subshell so no failure path can skip the cleanup, and so umask cannot leak out. + # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound + # command that is the left operand of `||`, which this subshell is. + ( + umask 077 + + bao kv get -format=json -mount="$mount" "centralis/${env_name}" \ + > "$work/kv.json" 2>/dev/null || { + echo "!! cannot read ${mount}/centralis/${env_name}" >&2 + exit 1 + } + + python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1 +import json, sys +doc = json.load(open(sys.argv[1], encoding="utf-8")) +data = doc.get("data", {}).get("data", {}) or {} +if not data: + sys.stderr.write("!! OpenBao returned zero keys for centralis\n") + sys.exit(1) +json.dump(data, open(sys.argv[2], "w", encoding="utf-8")) +UNWRAP + + python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1 +import json, re, sys + +src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3] +NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + + +def shell_quote(v): + """POSIX single-quoting. Inside '...' every byte is literal except ' itself, + which is closed, backslash-escaped and reopened. There is no escape sequence + to get wrong and no character class to keep up to date: it is total, and + total is the only property worth having for a file bash will expand.""" + return "'" + v.replace("'", "'\\''") + "'" + + +def scalar(k, v): + if isinstance(v, str): + return v + if isinstance(v, bool): # before int: bool is an int in Python + return "true" if v else "false" + if isinstance(v, (int, float)): + return json.dumps(v) + if v is None: + return "" + sys.stderr.write( + "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__)) + sys.exit(1) + + +data = json.load(open(src, encoding="utf-8")) +want, lines = {}, [] +for k, v in data.items(): + if not NAME.match(k): + sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,)) + sys.exit(1) + want[k] = scalar(k, v) + lines.append("%s=%s\n" % (k, shell_quote(want[k]))) + +with open(out_env, "w", encoding="utf-8") as fh: + fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n") + fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n") + fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n") + fh.write("# Rotate with `bao kv put` against thermograph/centralis/ instead.\n") + fh.writelines(lines) +with open(want_json, "w", encoding="utf-8") as fh: + json.dump(want, fh) +sys.stderr.write(" %d keys quoted\n" % len(want)) +EMIT + + # PROVE IT. Source the rendered file exactly the way the deploy does — clean + # environment, `set -a`, nothing inherited that could mask a dropped key — and + # read every value back. This is the check the 2026-07-24 file could not pass. + # shellcheck disable=SC2016 # single quotes are the point: "$1" must be expanded + # by the inner `bash -c` against the argument after `_`, not by this shell. + env -i PATH="$PATH" bash -c \ + 'set -a; . "$1"; set +a; exec python3 -c " +import json, os, sys +sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \ + > "$work/got.json" || exit 1 + + python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1 +import json, sys + +want = json.load(open(sys.argv[1], encoding="utf-8")) +got = json.load(open(sys.argv[2], encoding="utf-8")) + +bad = [] +for k, v in want.items(): + if k not in got: + bad.append("%s: not set at all after sourcing" % k) + elif got[k] != v: + # NEVER print either value. The difference is the finding; the content is + # not, and this runs inside a deploy log. + bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)" + % (k, len(v), len(got[k]))) +if bad: + sys.stderr.write("!! the rendered file does not round-trip through bash:\n") + for b in bad: + sys.stderr.write("!! %s\n" % b) + sys.exit(1) + +# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes, +# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the +# single `shared` identity, which looks exactly like the registry never having been +# configured. Refuse to write a file that would do that. +raw = got.get("CENTRALIS_TOKENS", "") +if raw: + try: + reg = json.loads(raw) + except ValueError as exc: + sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc) + sys.exit(1) + if not isinstance(reg, dict) or not reg: + sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n") + sys.exit(1) + if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()): + sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n") + sys.exit(1) + # Subjects are names on audit lines, not credentials. Printing them is the whole + # point: it is how an operator sees at a glance that nobody was lost. + sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n" + % (len(reg), ", ".join(sorted(reg)))) + +sys.stderr.write(" %d keys survive `set -a; . ` byte-for-byte\n" % len(want)) +VERIFY + ) || rc=1 + + # Only now, with a file read back through bash and matched value for value, does + # anything touch $dest. 0600: four bearer credentials for the estate's control + # plane, with no group that needs them. + if [ "$rc" -eq 0 ]; then + if [ -f "$dest" ] && [ -w "$dest" ]; then + cat "$work/out.env" > "$dest" || rc=1 + elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then : + elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then : + else + echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2 + rc=1 + fi + else + echo "!! render aborted; ${dest} left untouched" >&2 + fi + + rm -rf "$work" + return "$rc" +} diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index 32093d9..ea7de00 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -272,6 +272,30 @@ render_centralis_secrets() { echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2 return 1 fi + + # Same early-return dispatch as render_thermograph_secrets, for the same reason + # given there: everything below — the age-key sudo lift, the JSON decrypt, the + # quote-and-prove pipeline, the three-way write — stays on exactly the code path it + # has always been on, so this cannot regress the backend still authoritative for + # Centralis. The OpenBao path needs no age key, so it returns before that check. + local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}" + if [ "$backend" = openbao ]; then + local bao_lib="${repo}/deploy/render-secrets-openbao.sh" + if [ ! -f "$bao_lib" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2 + echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2 + return 1 + fi + # shellcheck source=/dev/null + . "$bao_lib" + thermograph_render_openbao_centralis "$env_name" "$dest" + return + fi + if [ "$backend" != sops ]; then + echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2 + return 1 + fi + if [ ! -f "$key" ]; then echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2 return 1 diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 855e4d7..14c6ac8 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -112,6 +112,22 @@ thermograph/data/legacy/age the age PRIVATE key — see "the age key surviv Inheritance lives in the render order (`common` then `env/`, last-wins); isolation lives in the policy. Today both live in one shell variable. +**`centralis/` renders differently from everything above it, and must.** The +app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env` +is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion +pipeline, so it is emitted as POSIX single-quoted assignments and then verified by +sourcing it in a clean shell and comparing every value back before the file is +written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash +strips its quotes, Centralis fails closed on the malformed result and serves a +single `shared` identity, which is indistinguishable from the file never having +been written. That is the 2026-07-24 incident, and the round-trip check exists so a +render carrying it cannot reach `/etc`. + +Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell +metacharacters, no newlines) only to `common` and `env/*`. Applying them to +`centralis/*` would reject data that renders perfectly well — the validator matches +the renderer that will consume the path, not a single global rule. + `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny rule expressible as exactly one path, with no wildcard over `env/*` able to accidentally re-include it. diff --git a/infra/openbao/seed-from-sops.sh b/infra/openbao/seed-from-sops.sh index 09f32cc..3988ca7 100755 --- a/infra/openbao/seed-from-sops.sh +++ b/infra/openbao/seed-from-sops.sh @@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at # SEED time — before anything depends on it — rather than at deploy time. - reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY' + # Which renderer will consume this path decides which values are legal, so the + # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED + # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter + # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by + # render_centralis_secrets and its OpenBao counterpart, where every byte inside + # '...' is literal — so applying the dotenv rules to it rejects data that renders + # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them. + quoted=0 + case "$target" in centralis.*) quoted=1 ;; esac + + reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY' import json, os, re, sys doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"]) @@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc: sys.stderr.write("!! vault did not decrypt to a non-empty object\n") sys.exit(1) +# 1 => destined for a single-quoted env file; 0 => raw dotenv. +QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1" + KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") UNSAFE = set("`$\\\"'") @@ -141,12 +154,13 @@ for key, val in doc.items(): val = "" if not isinstance(val, str): val = json.dumps(val, separators=(",", ":")) - if "\n" in val or "\r" in val: - bad.append(f"{key}: contains a newline") - continue - if UNSAFE & set(val): - bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") - continue + if not QUOTED: + if "\n" in val or "\r" in val: + bad.append(f"{key}: contains a newline") + continue + if UNSAFE & set(val): + bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") + continue out[key] = val if bad: -- 2.45.2 From 11f1275998254d445ba2935cf4be269461f2e24a Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 30 Jul 2026 21:24:18 -0700 Subject: [PATCH 36/37] rotate Contabo S3 credentials --- infra/deploy/secrets/common.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/infra/deploy/secrets/common.yaml b/infra/deploy/secrets/common.yaml index 83ec094..247789d 100644 --- a/infra/deploy/secrets/common.yaml +++ b/infra/deploy/secrets/common.yaml @@ -4,13 +4,13 @@ THERMOGRAPH_BASE: ENC[AES256_GCM,data:sQ==,iv:Pop6S2qU0o2+2xMKWQD2P3Vjdh4rUafWF7 THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:CA==,iv:TM6XiygrDQn2qps4lO6hY2ldjW8LWlO3D/R/DBPqvGQ=,tag:+thBT7fMzH+DigecgxF4CQ==,type:str] THERMOGRAPH_INDEXNOW_KEY: ENC[AES256_GCM,data:XZEMMJfmrSjc/sKEePaigTqEkh3ob/E25R25DvOAWwY=,iv:fqdaegzG1Na0zW+UgOh92RS4FP22pxStdD1/Jq5DRjs=,tag:/8ORtVkGUWjqgGXG/xZHmg==,type:str] THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:5pL+8tZbs8TZ5ezJpkPUmMss0gLcte73lU+au/476Ws=,iv:ujU3G+rbPZQx/igg50GeIPQxqgd+szIsjc3AqGhGLBU=,tag:hrkgn2DPkmBGThlsdMplqg==,type:str] -THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:XG+6wizXMVVFixhqLhg72HoKw9kEmH+Tty8jYMycvVk=,iv:tNz/WrkMYip2QyAr6bRy6PaIee56ZZ64BELBgvidVT0=,tag:HiQ55DGbeDAOSF0vpNWpWw==,type:str] +THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:Ui5dcmhzz84nTp1gYhJ89VOJ2eOBFyLjf0WkZ7Y3sI0=,iv:WI0QqbUW7BHo7Zz0082zxpYNIYpVIhq1NeBGbQeDc/I=,tag:urNsE8DPaldJcw9aqX1PYA==,type:str] THERMOGRAPH_METRICS_TOKEN: ENC[AES256_GCM,data:DmOZU3HWQTuoSvQMb7iPkClJbDH48NB3OcRIYRkqNz8=,iv:acgGh0w2mc5ZD7rB7m/GW3Awsx3RWK+02L6YkAv5Rw0=,tag:x3U9F0Dekw/r9euRMjVQAA==,type:str] THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:hTSSUIfOlY0GYI8gZ7FcOi7c89iye0Ym73l4M22zkNk=,iv:aPdbXKb4pOqigo9J3a7oM3qTip6HQcHUQDlAwg6XfoE=,tag:C4p+Ob/iSElNdF4m9E+M7A==,type:str] THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:vGJDaElN6JRUawlD55Vr5Q==,iv:4ZAQ5NXb9xuMN+qPspM17W6zm9NIDXLX9hTJ/etSL0E=,tag:Ss7xaUl+kOiVF7oDs9FiYw==,type:str] THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:PtZg52Uu70Ndx7L/dRTCZTmqv80NXiohOg+gfrzG,iv:hKQTy3Twd5uWwnS4UIs9oqT4NzpYUWQxWO1lUqrzpAc=,tag:kYwYp58fxkzRe5yQPSXa3g==,type:str] -THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:1xuZRaC4p+ZM/oFAUuGDZb6AAu2PbjFrHQY3zmczRpI=,iv:xzI24qlUOvYiThBLOue0odvopuxdHCwDlwI4JwiP9EU=,tag:yRS9GjGLBEYk/kZ6miFudg==,type:str] -THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:N9mPLx6Mcgqm5TlgqIMFhNuZsBZxVPXf3LplO9k+VA==,iv:y5+MLI0zC5Kb6pRdORTMVJ1iMMoFrVRfv99mKWMRjp8=,tag:7VvKCLU+1TJtQcHWlDwybg==,type:str] +THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:4u5d3RHHBeIDfRIwQVDldWUhH7PyTcNP9s3xTrL4N74=,iv:RXWu+4ZdTmWx3PGw6nWhN2587byy4wqjY74Ldu4xvuk=,tag:dwS6htP2Dv2OQyNzmjZZIA==,type:str] +THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:hg5HqBua81dG3d4KaxLxljRkeGnt4h4=,iv:q+4tQGrlsQaRGUTJuqByqss+twR6v6NQrI/qSPjCbsQ=,tag:2D1+hv/KSDJu+4+J9S0hww==,type:str] THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:hTxKlgPWeW4HGBf08SxdAdK+ce6T5Fl9f+5tSGV6WpS+za5EI3zsK8BgRw==,iv:l+omFaCyL2+PHWdchadkmED2gIAoj5T0u/Ltzaw8mok=,tag:AQ3wT/wD5JAouX1M+Tjjmw==,type:str] THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:bgtqZFfTzYz5llh/YhAyGwGtRedK3/ze8SAWj8YdldY6s4nt2F3XeTANVO2Y9+NzvXc16PB3WWPnYEzCxnrG2KciIeYpOYZMoPpGldENPBLJtLa76Ej9,iv:HuZ7XtlXIt4wyJ79k3IYjHfWrQp7lnak4uCspyzS4BE=,tag:uPZvba8LVugDy76pKat6TA==,type:str] TIMESCALEDB_TAG: ENC[AES256_GCM,data:3mDU/k5fx0894+E=,iv:/iw3BZPF3mZ9M3bFNJSTzP9t15YZWCJwzoVXYL9Vj9o=,tag:Y4ez3+XfLJN1PtJyZlJj+g==,type:str] @@ -25,7 +25,7 @@ sops: bncE6yn10QK5kVcF9dDvpQ6RbaG+ESpNZTnuhSL5PDqrKtjnsV0Iqw== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-25T00:38:24Z" - mac: ENC[AES256_GCM,data:C12PnFE7MO0Ge5dCAHCcyXh650hFtRuexl5d3DL0IqTf8nLyNtRJzrPyl9whIs+S3HulsXh3xJOwwPdg6aTqdCfGSQ4Kp5qV+OHxH9lfpHs2yH+exa/eX+7Khpjk0OfvX5beC0GSPpYf9c01buaUTWcIh3pJXpDMdqr+aLC+Zd0=,iv:6VeaaEEkWU5rdTHy75rc2emb2u/HmOmLzO11D38ZXHc=,tag:iGRevzdtqZF9kyH0wxqG3w==,type:str] + lastmodified: "2026-07-31T04:23:12Z" + mac: ENC[AES256_GCM,data:KkItyA0iw2j4w9pf/efFcpyOP8RmO8XzfMoQhJcRXLZbrUC3/xltbxkfTIe2hRgAqKyz5d7EuAHBPwwSCubue0O2gzlG0nCnRQH7YyO4cUrP24XYb+aI+tBM2jdZIuZrGDNcq04cwKiecC6MyOOp8uQPZ41AOPH3sONGWj1VH2s=,iv:+j5uv4QbuWpXAyElvbYGKWhe9vmGTfs3Hpy5I5FetJo=,tag:wcfPuugefeQAyxdb2HV1Fg==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 -- 2.45.2 From b2077b52946a58163895d90e00d77f2e27597cdd Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Thu, 30 Jul 2026 20:26:37 -0700 Subject: [PATCH 37/37] secrets: make the two-host age key quorum a checked policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The copies of /etc/thermograph/age.key on vps1 and vps2 have always read as provisioning residue — a side effect of provision-secrets.sh rather than a decision. They are in fact the estate's recovery quorum: that key is the single recovery root for all five SOPS vaults AND for every off-box backup, since ops-cron.yml:142,197 pipe those dumps through `age -r` to the same recipient. On 2026-07-30 the operator's copy was shredded from the desktop before it had been copied to its replacement machine, leaving zero operator-side copies. These two were the only surviving material and the key was restored from vps2. Documenting the ordering rule that would have prevented it, and adding the check that makes the quorum verifiable rather than assumed. verify-age-quorum.sh asserts presence, permissions and hash equality across both hosts. Divergence is treated as seriously as absence: two hosts holding different keys means one renders secrets the other cannot read, and a restore silently picks the wrong one. Prints SHA-256 digests and modes only — a digest of a 32-byte random key is not reversible, so it is safe in a CI log, and the key is never read or copied anywhere. Exit status makes it usable as a cron gate. Also replaces the "wasn't re-verified for this pass" hedge in the README with what was actually measured: vps1 0440 root:deploy, vps2 0400 root:root, digests equal. The dev render does not need vps1's group bit — it runs as `agent`, which is not in group `deploy` and reaches the key through passwordless sudo, with /opt/thermograph-dev owned agent:agent. Tightening vps1 to 0400 root:root is therefore expected to be safe, and is called out as the follow-up rather than done here because it changes a live host on the deploy path. The check reports that group read as an advisory rather than a failure. It is the current provisioned state, and a check that fails on day one is a check that is ignored by day three. --- infra/deploy/secrets/README.md | 49 +++++++-- infra/deploy/secrets/verify-age-quorum.sh | 117 ++++++++++++++++++++++ 2 files changed, 155 insertions(+), 11 deletions(-) create mode 100755 infra/deploy/secrets/verify-age-quorum.sh diff --git a/infra/deploy/secrets/README.md b/infra/deploy/secrets/README.md index 4b85e59..7ec4bb3 100644 --- a/infra/deploy/secrets/README.md +++ b/infra/deploy/secrets/README.md @@ -251,17 +251,44 @@ files), so layering would only push the VAPID private key, both S3 keypairs and - `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time. - **Back it up** in the password manager. The public recipient is in `.sops.yaml`. -On **vps1** (dev) those two can end up being the same file. `render-secrets.sh` -falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes -the account driving a CI-triggered dev deploy has no passwordless sudo and no -tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) — -inherited from the old desktop setup, where that was true of the whole box. -Whether it's still true of vps1's CI-runner account specifically (as opposed -to the `agent` login, which does have passwordless sudo there per `ACCESS.md`) -wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY` -at the operator's keyring as a fallback either way, so the box keeps **one** -copy of the recovery root rather than two regardless of which account ends up -mattering. +### The two on-host copies are a deliberate quorum + +`/etc/thermograph/age.key` on **vps1** and **vps2** is not provisioning residue and +must not be tidied away. Together they are the recovery quorum for the entire +estate: five SOPS vaults, and every off-box backup, since `ops-cron.yml:142,197` +pipe those dumps through `age -r` to this same recipient. + +That is not theoretical. On **2026-07-30** the operator's copy was shredded from the +desktop *before* it had been copied to its replacement machine, leaving zero +operator-side copies. These two were the only surviving material, and the key was +restored from vps2. The ordering rule below exists because of that hour. + +- **Never remove the last operator-side copy** until its replacement has been + verified against all five vaults (`sops -d … | grep -c '='` → 16 / 12 / 8 / 16 / 9). + Deleting first and verifying second is unrecoverable if the copy is bad. +- **Both hosts must hold byte-identical copies.** Divergence is as bad as absence: + two different keys means one host renders secrets the other cannot read, and a + restore silently picks the wrong one. +- **Verify it, don't assume it** — `deploy/secrets/verify-age-quorum.sh` asserts + presence, permissions and hash equality across both hosts. It prints SHA-256 + digests and modes only, never the key, so it is safe in a CI log and suitable as + a cron gate. Deletion and divergence are both silent failures; nothing else in the + estate would notice either until a restore. + +Verified 2026-07-30: vps1 `0440 root:deploy`, vps2 `0400 root:root`, digests equal. +Note the asymmetry — on vps1 the group `deploy` can read the key that decrypts +`common.yaml` and `prod.yaml`, on the box that also runs Forgejo, its CI runner and +dev's unreviewed branches. The dev render does not need that group bit: it runs as +`agent`, which is not in group `deploy` and reaches the key through its passwordless +`sudo` instead (`/opt/thermograph-dev` is `agent:agent`). Tightening vps1 to `0400 +root:root` is therefore expected to be safe and is the obvious follow-up; it is +called out rather than done here because it changes a live host on the deploy path. + +This weakens — in practice, not in intent — the rule in `infra/CLAUDE.md` that "vps1 +must never hold prod credentials". Withholding `common.yaml` from dev's *render* does +not withhold the key that decrypts it from the *box*, because every vault is +encrypted to a single recipient. Fixing that properly means a second age identity for +dev, not a policy line. ## Prerequisites (once per machine) diff --git a/infra/deploy/secrets/verify-age-quorum.sh b/infra/deploy/secrets/verify-age-quorum.sh new file mode 100755 index 0000000..8bb36ba --- /dev/null +++ b/infra/deploy/secrets/verify-age-quorum.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Assert that the age recovery quorum is intact. +# +# infra/deploy/secrets/verify-age-quorum.sh +# +# The age private key is the single recovery root for all five SOPS vaults AND for +# every off-box backup (ops-cron.yml:142,197 pipe dumps through `age -r` to the same +# recipient). The copies at /etc/thermograph/age.key on vps1 and vps2 are a DELIBERATE +# two-host quorum, not provisioning residue — see the "age key" section of +# deploy/secrets/README.md. This script is what makes that policy checkable instead of +# merely stated. +# +# It fails if the key is missing on either host, if the two copies have diverged, or +# if either is more permissive than 0440. Divergence matters as much as absence: two +# hosts holding different keys means one of them renders secrets nobody else can +# decrypt, and a restore from backup silently picks the wrong one. +# +# OUTPUT DISCIPLINE: prints SHA-256 digests and file modes only. A digest of a +# 32-byte random key is not reversible, so it is safe in a CI log; the key itself is +# never read, echoed or transferred. Nothing here copies the key anywhere. +# +# Exit status: 0 = quorum intact; 1 = something needs a human. Suitable as a cron gate. +set -euo pipefail + +VPS1="${TG_VPS1_SSH:-vps1}" +VPS2="${TG_VPS2_SSH:-vps2}" +KEY_PATH="${TG_AGE_KEY_PATH:-/etc/thermograph/age.key}" +SSH_OPTS="${TG_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=10}" + +# Reads the digest and mode of the key on one host. Uses sudo when the key is not +# directly readable, mirroring how render-secrets.sh lifts it. +probe() { + local target="$1" out + # The remote script is a QUOTED heredoc and the key path is passed as $1 to + # `bash -s`, so nothing expands on this side. Interpolating the path into the + # command string would work too, but it is the shape that quietly breaks the day + # a path contains a space, and shellcheck is right to flag it (SC2029). + # shellcheck disable=SC2086 # SSH_OPTS is a deliberate word-split option list + out=$(ssh $SSH_OPTS "$target" bash -s -- "$KEY_PATH" 2>/dev/null <<'REMOTE' +key="$1" +if [ ! -e "$key" ]; then echo MISSING; exit 0; fi +mode=$(stat -c %a "$key") +owner=$(stat -c %U:%G "$key") +if [ -r "$key" ]; then + digest=$(sha256sum "$key" | cut -d' ' -f1) +else + digest=$(sudo sha256sum "$key" 2>/dev/null | cut -d' ' -f1) +fi +[ -n "$digest" ] || { echo UNREADABLE; exit 0; } +echo "$digest $mode $owner" +REMOTE + ) || { echo UNREACHABLE; return 0; } + printf '%s\n' "$out" +} + +rc=0 +declare -A DIGEST + +for pair in "vps1:$VPS1" "vps2:$VPS2"; do + name="${pair%%:*}" + target="${pair#*:}" + result="$(probe "$target")" + case "$result" in + MISSING) + echo "!! ${name}: no key at ${KEY_PATH} — the quorum is DOWN TO ONE COPY" >&2 + rc=1 + ;; + UNREADABLE) + echo "!! ${name}: key present but unreadable even via sudo" >&2 + rc=1 + ;; + UNREACHABLE) + echo "!! ${name}: host unreachable — quorum NOT verified (this is not a pass)" >&2 + rc=1 + ;; + *) + digest="${result%% *}" + rest="${result#* }" + DIGEST["$name"]="$digest" + printf ' %-5s %s mode=%s\n' "$name" "${digest:0:16}…" "$rest" + mode="${rest%% *}" + # 0400 or 0440 only. Anything wider means a non-root account on that box can + # read the key that decrypts prod — and vps1 also runs Forgejo and its CI runner. + case "$mode" in + 400|440) ;; + *) echo "!! ${name}: mode ${mode} is wider than 0440" >&2; rc=1 ;; + esac + # Advisory, not a failure: 0440 with a non-root group means that group can read + # the key that decrypts prod. On vps1 that is doubly worth knowing, because the + # same box runs Forgejo, its CI runner and dev's unreviewed branches. Left + # non-fatal because it is the current provisioned state — a check that fails on + # day one is a check that gets ignored by day three. See README.md. + group="${rest#* }"; group="${group#*:}" + if [ "$mode" = 440 ] && [ "$group" != root ]; then + printf ' note: group %s can read this key\n' "$group" + fi + ;; + esac +done + +if [ -n "${DIGEST[vps1]:-}" ] && [ -n "${DIGEST[vps2]:-}" ]; then + if [ "${DIGEST[vps1]}" = "${DIGEST[vps2]}" ]; then + echo " quorum: 2 copies, identical" + else + echo "!! the two copies have DIVERGED — they are different keys" >&2 + echo "!! do not 'fix' this by overwriting one. Establish which decrypts the" >&2 + echo "!! vaults (sops -d on any deploy/secrets/*.yaml) before touching either." >&2 + rc=1 + fi +fi + +if [ "$rc" -eq 0 ]; then + echo " OK — recovery quorum intact" +else + echo "!! recovery quorum degraded; see deploy/secrets/README.md" >&2 +fi +exit "$rc" -- 2.45.2