Merge pull request 'openbao: add a dormant OpenBao backend alongside SOPS' (#130) from openbao-backend into dev

infra/deploy/env-topology.sh

@@ -68,6 +68,14 @@ thermograph_topology() {
   TG_TAGS_FILE=""; TG_LOCK_FILE=""; TG_BIND_ADDR=""; TG_SKIP_COMMON=0
   TG_SVC_PREFIX=""; TG_DATA_NETWORK=""; TG_DB_SERVICE=""; TG_POST_DEPLOY=0
   TG_SSH_HOST=""; TG_SSH_TARGET=""
+  # Which store render-secrets.sh reads: sops | openbao. Defaults to sops for every
+  # environment and is overridden per environment below, so a cutover is a one-line
+  # reviewed change that follows the estate's own dev -> main -> release promotion.
+  #
+  # This lives here, and NOT in a host marker file like /etc/thermograph/deploy-mode,
+  # for the reason that marker was demoted to a fallback in the first place: vps2 runs
+  # prod and beta side by side, and one host-wide file cannot name two backends.
+  TG_SECRETS_BACKEND=sops
 
   case "$env_name" in
     prod)
@@ -220,6 +228,7 @@ thermograph_topology() {
   export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
   export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
   export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
+  export TG_SECRETS_BACKEND
   export TG_SSH_HOST TG_SSH_TARGET
 }
 

infra/deploy/render-secrets-openbao.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+# OpenBao source backend for render-secrets.sh. Sourced, never run directly.
+#
+# Two functions:
+#   thermograph_openbao_source <env>        -> merged dotenv on STDOUT
+#   thermograph_render_openbao  <env> <out> -> source, then write <out>
+#
+# render_thermograph_secrets in render-secrets.sh dispatches to the second when
+# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
+# keeps byte-identical behaviour and all new risk is confined to this file. See the
+# note above thermograph_render_openbao on why the write path is duplicated here
+# rather than shared, and when to consolidate.
+#
+# WHY OUTPUT SHAPE IS FROZEN: /etc/thermograph.env must stay raw, UNQUOTED KEY=value.
+# Centralis' secrets_inventory / secrets_render compare that file textually
+# (deploy/secrets/README.md:206-210), compose reads it via `env_file:`, and the Swarm
+# stack's env-entrypoint.sh parses it line by line. Quoting it "properly" would be an
+# improvement in isolation and a silent break in context.
+
+# thermograph_openbao_source <env_name>
+#
+# Reads the common set then the per-environment set and merges them with the
+# environment winning — the same last-wins semantic as the SOPS concatenation, except
+# performed explicitly here because OpenBao returns a map per path rather than a
+# stream that can simply be appended.
+thermograph_openbao_source() {
+  local env_name="${1:?env name required}"
+  local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
+  local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
+  local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
+  local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
+
+  command -v bao >/dev/null 2>&1 || {
+    echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
+    return 1
+  }
+  command -v python3 >/dev/null 2>&1 || {
+    echo "!! python3 required to render dotenv from OpenBao JSON" >&2
+    return 1
+  }
+
+  # AppRole credentials: role_id and secret_id, one per line, 0400 root — the same
+  # protection level as /etc/thermograph/age.key today. Read directly if we can, else
+  # via sudo, mirroring how render-secrets.sh lifts the age key.
+  local creds
+  if [ -r "$approle" ]; then
+    creds=$(cat "$approle")
+  else
+    creds=$(sudo cat "$approle" 2>/dev/null || true)
+  fi
+  [ -n "$creds" ] || {
+    echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
+    return 1
+  }
+
+  local role_id secret_id
+  role_id=$(printf '%s\n' "$creds" | sed -n '1p')
+  secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
+  [ -n "$role_id" ] && [ -n "$secret_id" ] || {
+    echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
+    return 1
+  }
+
+  export BAO_ADDR="$addr"
+  [ -f "$ca" ] && export BAO_CACERT="$ca"
+
+  # Exchange the AppRole for a short-lived token. The token TTL is set on the role
+  # (see bootstrap.sh) and is deliberately measured in minutes: it only has to
+  # outlive one render.
+  local token
+  token=$(bao write -field=token auth/approle/login \
+            role_id="$role_id" secret_id="$secret_id" 2>/dev/null) || {
+    echo "!! OpenBao AppRole login failed for env='${env_name}'" >&2
+    echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
+    return 1
+  }
+  export BAO_TOKEN="$token"
+
+  # Fetch. `common` is skipped for dev by the same flag the SOPS path uses — but note
+  # that on the OpenBao path the flag is belt, not braces: dev's AppRole is bound to
+  # the tg-host-dev policy, which DENIES thermograph/data/common outright. If this
+  # flag is ever forgotten at a call site, dev gets a 403 rather than a file full of
+  # production credentials. That inversion — from "the caller must remember" to "the
+  # store refuses" — is the main reason for this migration.
+  local common_json="" env_json
+  if [ "${THERMOGRAPH_SECRETS_SKIP_COMMON:-0}" != 1 ]; then
+    common_json=$(bao kv get -format=json -mount="$mount" common 2>/dev/null) || {
+      echo "!! cannot read ${mount}/common for env='${env_name}'" >&2
+      echo "!! if this is dev, THERMOGRAPH_SECRETS_SKIP_COMMON=1 was not set and the" >&2
+      echo "!! tg-host-dev policy correctly refused — that is the guard working." >&2
+      return 1
+    }
+  fi
+
+  env_json=$(bao kv get -format=json -mount="$mount" "env/${env_name}" 2>/dev/null) || {
+    echo "!! cannot read ${mount}/env/${env_name}" >&2
+    return 1
+  }
+
+  # Merge and emit. Python rather than jq because jq is not installed on these hosts
+  # and python3 is (the Centralis renderer already relies on it).
+  #
+  # This step also enforces the two invariants the dotenv format cannot express:
+  #   * a value containing a newline is REJECTED, because it would silently become
+  #     two lines and the second would parse as a bogus KEY=value or be dropped;
+  #   * a value containing a shell metacharacter is REJECTED, because
+  #     /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
+  #     deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
+  #     `$(...)` or a backtick in a secret is a live command-execution path on the
+  #     deploy host. The SOPS path has always had this hazard and has never tripped it
+  #     only because every current value happens to be alphanumeric. Refusing here
+  #     turns a latent code-execution bug into a loud render failure.
+  THERMOGRAPH_BAO_COMMON="$common_json" \
+  THERMOGRAPH_BAO_ENV="$env_json" \
+  python3 - <<'PY'
+import json, os, re, sys
+
+def load(raw):
+    if not raw:
+        return {}
+    doc = json.loads(raw)
+    return doc.get("data", {}).get("data", {}) or {}
+
+merged = {}
+# common first, environment second: last-wins, matching the SOPS concatenation order.
+for src in ("THERMOGRAPH_BAO_COMMON", "THERMOGRAPH_BAO_ENV"):
+    merged.update(load(os.environ.get(src, "")))
+
+KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+# Characters that change meaning when the file is sourced by bash.
+UNSAFE = set("`$\\\"'")
+
+bad = []
+lines = []
+# Validate EVERYTHING before emitting a single byte. Printing as we go would leave
+# partial output on stdout when a later key fails, and render-secrets.sh's whole
+# discipline is that a failed source never produces a partial env file.
+for key in sorted(merged):
+    if not KEY_RE.match(key):
+        bad.append(f"{key}: not a valid env var name")
+        continue
+    val = merged[key]
+    if val is None:
+        val = ""
+    if not isinstance(val, str):
+        val = json.dumps(val, separators=(",", ":"))
+    if "\n" in val or "\r" in val:
+        bad.append(f"{key}: value contains a newline (would corrupt the dotenv)")
+        continue
+    if UNSAFE & set(val):
+        bad.append(
+            f"{key}: value contains a shell metacharacter (one of ` $ \\ \" ') and "
+            f"/etc/thermograph.env is sourced by bash — refusing"
+        )
+        continue
+    lines.append(f"{key}={val}")
+
+if bad:
+    sys.stderr.write("!! refusing to render; unsafe values:\n")
+    for line in bad:
+        sys.stderr.write(f"!!   {line}\n")
+    sys.exit(1)
+
+if not lines:
+    sys.stderr.write("!! refusing to render: OpenBao returned zero keys\n")
+    sys.exit(1)
+
+sys.stdout.write("\n".join(lines) + "\n")
+PY
+}
+
+# thermograph_render_openbao <env_name> <out_file>
+#
+# The OpenBao equivalent of render_thermograph_secrets: source, then write.
+#
+# ON THE DUPLICATED WRITE PATH. The write logic below mirrors
+# render-secrets.sh:130-154 rather than being factored out and shared. That is a
+# deliberate, temporary choice, not an oversight:
+#
+#   render-secrets.sh is on the deploy path for all three environments and there is no
+#   test suite in front of it (infra/CLAUDE.md: "Shell here runs as root over SSH
+#   against live hosts with no test suite in front of it"). Extracting the write path
+#   would mean the SOPS render — which currently works — starts flowing through newly
+#   moved code that cannot be tested end-to-end from here. An early-return branch keeps
+#   the SOPS path byte-identical and confines all new risk to the new backend.
+#
+#   CONSOLIDATE THESE after prod has been on the OpenBao backend for a full release
+#   cycle and the SOPS path is being deleted anyway. Until then, a change to one write
+#   path must be made in both — which is exactly the cost being accepted here.
+thermograph_render_openbao() {
+  local env_name="${1:?env name required}"
+  local out="${2:?output path required}"
+
+  echo "==> Rendering $out from OpenBao (env=${env_name})"
+
+  local tmp; tmp=$(mktemp)
+  # Holds DECRYPTED secrets. Removed on every exit path. Deliberately not a
+  # `trap ... RETURN` — see render-secrets.sh:82-91 for why that is actively wrong in
+  # a sourced function (the trap persists into the caller and re-fires on its next
+  # `source`, where the local is unset and `set -u` makes it fatal and silent).
+  if ! thermograph_openbao_source "$env_name" > "$tmp"; then
+    rm -f "$tmp"
+    return 1
+  fi
+
+  # An empty render must never reach $out. The source function already refuses a
+  # zero-key result, so this is a second line of defence rather than the only one:
+  # a truncated /etc/thermograph.env is the single worst outcome available here,
+  # because thermograph.service uses `EnvironmentFile=-` (missing is NOT fatal) and
+  # the app self-generates AUTH_SECRET and the VAPID pair when they are absent. The
+  # result would be a green deploy that silently invalidated every session and
+  # deleted every push subscription.
+  if [ ! -s "$tmp" ]; then
+    echo "!! OpenBao render produced an empty file; refusing to write $out" >&2
+    rm -f "$tmp"
+    return 1
+  fi
+
+  local rc=0
+  if [ -f "$out" ] && [ -w "$out" ]; then
+    cat "$tmp" > "$out" || rc=1
+  elif install -m 0640 "$tmp" "$out" 2>/dev/null; then :
+  elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then :
+  else
+    echo "!! cannot write $out (need file write access or passwordless sudo)" >&2
+    rc=1
+  fi
+
+  if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then
+    if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \
+       || ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then
+      echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2
+      rc=1
+    fi
+  fi
+
+  rm -f "$tmp"
+  return "$rc"
+}

infra/deploy/render-secrets.sh

@@ -31,6 +31,38 @@ render_thermograph_secrets() {
   local out="${3:-/etc/thermograph.env}"
   [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true)
 
+  # Backend dispatch. TG_SECRETS_BACKEND comes from env-topology.sh (per environment,
+  # because vps2 runs two); THERMOGRAPH_SECRETS_BACKEND overrides it for a by-hand run.
+  # Default is sops, so an unmigrated host and an unset caller both behave exactly as
+  # before this branch existed.
+  #
+  # The OpenBao path returns early rather than threading a conditional through the
+  # rest of this function. That is on purpose: everything below — the age-key sudo
+  # lift, the two-file last-wins concatenation, the three-way write, the mirror — stays
+  # on precisely the code path it has always been on, so migrating cannot regress the
+  # backend that is still authoritative for prod.
+  local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
+  if [ "$backend" = openbao ]; then
+    if [ -z "$env_name" ]; then
+      echo "!! TG_SECRETS_BACKEND=openbao but no environment name was resolved" >&2
+      return 1
+    fi
+    local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
+    if [ ! -f "$bao_lib" ]; then
+      echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
+      echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
+      return 1
+    fi
+    # shellcheck source=/dev/null
+    . "$bao_lib"
+    thermograph_render_openbao "$env_name" "$out"
+    return $?
+  fi
+  if [ "$backend" != sops ]; then
+    echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
+    return 1
+  fi
+
   # The per-host file is required; common.yaml is optional so the initial cutover can
   # seed each host as an exact copy of its live env (byte-identical render, no value
   # changes) and factor out shared secrets into common.yaml later.

infra/openbao/README.md

@@ -0,0 +1,254 @@
+# OpenBao — the secret store, phase 1
+
+**Status: dormant.** Every file here is committed but nothing is cut over. SOPS is
+still authoritative for dev, beta and prod, because `TG_SECRETS_BACKEND` defaults to
+`sops` for all three in `infra/deploy/env-topology.sh`. Flipping an environment is a
+one-line reviewed change, and it should not happen until `verify-parity.sh` passes.
+
+Verified against **OpenBao v2.6.1** (2026-07-22). The binary is `bao`.
+
+---
+
+## Why do this at all
+
+Not for encryption, and not for audit — though the audit log is a real gain. The
+decisive reason is one specific failure class that a file-based vault cannot close.
+
+`infra/CLAUDE.md` states the rule: *"vps1 must never hold prod credentials. It's the
+box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of an
+isolation boundary."*
+
+Today that rule is enforced by a **shell flag** — `THERMOGRAPH_SECRETS_SKIP_COMMON=1`
+— set by the *caller*, at three separate call sites (`env-topology.sh:196` →
+`deploy.sh:67`, `deploy-dev.sh:86`, `infra-sync.yml:93-95`). The renderer itself does
+not know that `env=dev` means never-common. A fourth call site, or one by-hand
+`render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3 keypairs
+(one read-write on the bucket holding prod's database backups), the VAPID private key
+that signs Web Push to real subscribers, `REGISTRY_TOKEN`, and the IndexNow and metrics
+tokens onto the CI-runner box — **and exits 0**.
+
+Under `policies/tg-host-dev.hcl` that is not possible to get wrong. dev's identity
+cannot read `thermograph/data/common`. A misconfigured caller gets a 403 instead of a
+silent success. The failure class is *gone*, not guarded. That is what this migration
+buys, and it is worth the cost of running a stateful service.
+
+The estate has already hardened this once by hand (commit `7583258`, "infra-sync:
+refuse to render dev's vault onto a host that is not dev"). That is the shape of a
+problem that wants an ACL, not another guard.
+
+### What it does *not* buy — stated plainly
+
+- **Rotation of provider-issued credentials.** Roughly half the credential count is
+  Discord, Contabo S3, the Forgejo registry token, VAPID, IndexNow. OpenBao cannot
+  rotate any of them; they stay static KV entries. Contabo Object Storage is
+  S3-compatible but exposes no IAM API, so the AWS secrets engine cannot issue against
+  it either.
+- **Dynamic database credentials.** Inapplicable here for two independent reasons: the
+  apps read `THERMOGRAPH_DATABASE_URL` once at import and hold a pool, so a TTL'd
+  credential kills the pool at expiry; and there is no reload path anywhere in the
+  codebase — no SIGHUP handler, no config re-read.
+- **Better review than git.** This is a real regression. Today a secret change is a
+  PR with a diff, backstopped by `secrets-guard` CI. Under OpenBao it is an
+  out-of-band API call with no diff and no review. Mitigated by committing a key-name
+  manifest so CI can still assert no key vanished, plus the audit log and
+  `bao kv metadata` as the change record — but not fully.
+- **Fewer moving parts.** SOPS needs no server. This adds a stateful service to keep
+  alive, upgrade, back up and TLS-rotate, for ~40 secrets that change rarely.
+
+---
+
+## Shape of the deployment
+
+| Decision | Choice | Why |
+|---|---|---|
+| Host | **vps2 only** | vps1 must never hold prod credentials, and with a static seal the box also holds the key that decrypts everything. Cost: vps1 needs vps2 up to deploy dev — acceptable, since when vps2 is down prod is down anyway. |
+| Process | **native systemd**, not a container | A Swarm service is circular (`deploy-stack.sh` renders secrets to deploy the stack). A plain `docker run` makes the vault depend on the daemon deploys restart, and is one `prune -a` from gone. Native = one dependency, the disk. |
+| Storage | **raft**, single node | 2.6.0 **deprecated the `file` backend** for removal in 2.7.0. Raft also has the only backup primitive (`operator raft snapshot save`). A *two*-node raft would be worse than one: quorum of two means losing either node loses writes. |
+| Seal | **static auto-unseal** | See below. |
+| TLS | self-signed, 10y, on disk | Must **not** come from Caddy/ACME — that would put DNS and the public internet in the boot chain of the secret store. |
+| Auth | AppRole per environment, CIDR-bound | 5-minute tokens; secret-ids that only work from the right mesh address. |
+| Consumption | keep rendering a dotenv file | Preserves every existing seam and keeps OpenBao a *deploy-time* dependency, never a runtime one. |
+
+### The unseal decision, which is the crux
+
+**Static seal**, key at `/etc/openbao/unseal.key` (0400), plus an off-box copy.
+
+Shamir (`-key-shares=5 -key-threshold=3`) means the vault is sealed after every process
+restart — reboot, package upgrade, OOM kill — and a human must type three shares.
+`render-secrets.sh` is on the deploy path for all three environments, so a sealed vault
+blocks every deploy and every hotfix. With one operator the N-of-M threshold is
+theatre: one person holds all the shares, so it buys nothing against the real threat
+while costing an availability property the estate has today for free. A Contabo reboot
+currently needs no human at all.
+
+OpenBao's docs hedge static seal — *"carefully evaluate"* — but that caveat is aimed at
+multi-tenant enterprises. The argument here is that **static seal is not a downgrade
+from the status quo**: `/etc/thermograph/age.key` is already one 0400 file per host
+that decrypts everything forever with no audit trail. A static seal key is the
+identical trust model. What changes is everything layered above it.
+
+**The one new single point of failure:** lose that key and the raft snapshots are
+unrestorable. It must exist in ≥2 places, one not on vps2. Hard operator obligation.
+
+A `transit` seal against a second OpenBao on vps1 is genuinely better — vps2's disk
+would no longer contain the unseal key — but the vps1 unsealer needs unsealing too, so
+the circularity moves rather than dissolves, and it adds "vps1 must be up before vps2's
+vault unseals" as a new failure mode. Revisit once the primary migration is boring.
+
+---
+
+## Layout
+
+```
+thermograph/data/common          the 16 shared beta+prod values   (= common.yaml)
+thermograph/data/env/prod        prod's 16                        (= prod.yaml)
+thermograph/data/env/beta        beta's 8                         (= beta.yaml)
+thermograph/data/env/dev         dev's 12  — inherits NOTHING     (= dev.yaml)
+thermograph/data/centralis/prod  Centralis' 9              (= centralis.prod.yaml)
+thermograph/data/ops/backup      S3 creds for ops-cron + the backup age recipient
+thermograph/data/legacy/age      the age PRIVATE key — see "the age key survives"
+```
+
+Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
+isolation lives in the policy. Today both live in one shell variable.
+
+`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
+rule expressible as exactly one path, with no wildcard over `env/*` able to
+accidentally re-include it.
+
+**~20 of the 61 keys are not secrets** (`PORT`, `WORKERS`, `APP_CPUS`, `TIMESCALEDB_TAG`,
+`THERMOGRAPH_BASE_URL`, the Discord channel IDs, the VAPID *public* key, …). They stay
+in the vault **for the cutover**, because byte-identical parity is the entire safety
+property and splitting them would destroy it. Moving pure config back into the repo is
+a clean follow-on that shrinks the vault-outage blast radius.
+
+---
+
+## The age key survives this migration
+
+**This is the trap that would cause silent data loss.**
+
+`ops-cron.yml:142` and `:197` stream every off-box Postgres and Forgejo dump through
+`pg_dump | age -r <recipient> | rclone rcat`, with 30-day S3 retention. Restore uses
+`/etc/thermograph/age.key`. So the age keypair is not just the SOPS transport — **it is
+the backup encryption key.**
+
+Deleting it along with the SOPS vault files would destroy up to 30 days of database
+recoverability, and nothing would notice until someone attempted a restore.
+
+So: keep the age private key at `thermograph/data/legacy/age` *and* in the password
+manager, and keep it on disk until either the newest age-encrypted object in S3 has
+aged out, or the backup path is re-pointed at a dedicated backup recipient. Retiring
+SOPS and retiring age are two different projects.
+
+---
+
+## Runbook
+
+### Stand it up (once, by the operator, on vps2)
+
+```sh
+sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, unit, init
+# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
+export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
+export BAO_TOKEN=<root token>
+sudo -E bash infra/openbao/bootstrap-policies.sh   # mount, policies, approles
+bao token revoke -self
+```
+
+### Seed and prove (from your own machine — it needs your age key)
+
+```sh
+infra/openbao/seed-from-sops.sh --dry-run     # key names + counts, writes nothing
+infra/openbao/seed-from-sops.sh --all
+infra/openbao/verify-parity.sh --all          # MUST pass before any flip
+```
+
+`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
+the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
+inherits that rule.
+
+### Cut an environment over
+
+One PR per hop, following the estate's own promotion model.
+
+```diff
+   dev)
+-      TG_SECRETS_BACKEND=sops
++      TG_SECRETS_BACKEND=openbao
+```
+
+Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended
+gate before prod: **7 consecutive green parity runs on all three environments**, run
+nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting
+CI any vault access).
+
+`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
+SOPS path for a full release cycle after prod flips — it is the best mitigation
+available for anything unforeseen.
+
+### Rollback
+
+Revert the one-line PR and redeploy. The SOPS path is untouched by this work — verified
+by rendering dev (12 keys) and prod (32 keys) through it after the dispatch was added,
+matching the live hosts exactly.
+
+---
+
+## Failure modes
+
+| Failure | Effect | Mitigation |
+|---|---|---|
+| Vault down/sealed at deploy | Render returns 1, deploy aborts, **running stack unaffected** — temp-then-install means `/etc/thermograph.env` is never truncated | `Restart=always` + static seal so a reboot self-heals; alert on `/v1/sys/health`; `TG_SECRETS_BACKEND=sops` is a working revert |
+| **Audit device wedges** | ⚠️ OpenBao **stops answering requests entirely** when no enabled audit device can record them — a full `/var` on vps2 blocks every deploy | Two devices (file + syslog); logrotate signals **SIGHUP** or bao writes to an unlinked inode; disk alert on vps2 |
+| Static seal key lost | Raft snapshots unrestorable | ≥2 copies, one off-box. The one new SPOF |
+| Raft corruption / disk loss | Total vault loss | Nightly `operator raft snapshot save`, age-encrypted to S3 beside the DB dumps. **Verify a restore end-to-end before any consumer depends on it** |
+| Cold boot of vps2 | Nothing needs the vault | Swarm restarts from persisted specs; `stack.env` / `thermograph.env` / `centralis.env` all persist. **This property exists because we render a file, and would be destroyed by app-native reads** |
+| Secret-id leak | Useless off the mesh | `secret_id_bound_cidrs`; revoke by accessor without knowing the value |
+| beta credential reaches prod | Cross-env compromise | Separate AppRoles, separate policies, secret-ids owned by the separate deploy users (`agent` vs `deploy`) — a boundary that does **not** exist today, since both currently reach the same root-owned age key. Honest limit: root on vps2 defeats it, exactly as it defeats the single age key now |
+| Upgrade to 2.7.0 | Removes the `file` backend and built-in cloud KMS seals | Already avoided by choosing raft + static. Pin the version |
+
+---
+
+## Deliberately out of scope
+
+For ~40 credentials and one operator the real risk is over-engineering. Not adopted:
+HA/multi-node raft, namespaces, dynamic database credentials, `bao agent`, cert auth,
+PKI, identity groups, transit seal (revisit later), OIDC for the operator.
+
+**OIDC for CI is out of scope for a reason worth recording.** Forgejo Actions *does*
+support OIDC — shipped in **Forgejo v15.0**, needing Runner > v12.5.0, enabled via
+`enable-openid-connect: true` rather than GitHub's `permissions: id-token: write`. But
+this estate runs `codeberg.org/forgejo/forgejo:9-rootless`
+(`infra/deploy/forgejo/docker-stack.yml:59`). That is a six-major-version upgrade, and
+coupling it to this migration would mean two large independent migrations at once with
+no way to tell which one broke. Revisit after Forgejo reaches v15; then
+`role_type=jwt` with `bound_claims` removes the last CI-side bearer credential.
+
+CI keeps its SSH keys and `REGISTRY_TOKEN` as Forgejo secrets. **CI gets no vault
+access at all** — the SSH-in architecture already means the *host* renders, never the
+runner, which is the least-privilege topology and should be preserved rather than
+"upgraded".
+
+### Secret zero, honestly
+
+Nothing eliminates it; you choose where it sits and how little of it there is. After
+this phase the chain terminates at **9 Forgejo Actions secrets + 1 static seal key + 4
+AppRole secret-ids**, versus today's **13 CI secrets + 1 omnipotent age key per host**.
+
+---
+
+## Verification gaps to close on the box
+
+Flagged rather than guessed, because these could not be checked from a workstation:
+
+1. Whether a raft snapshot is restorable under a *different* seal key. Docs don't say.
+   **Test in phase 0, before anything depends on it.**
+2. Exact `-wrap-ttl` flag spelling on `bao write` — documented in the response-wrapping
+   concept page but absent from the commands index. Check `bao write -h`.
+3. Debian/Ubuntu package repo URL. The install docs claim `.deb` packages exist but
+   give no repository; `bootstrap.sh` uses the GitHub release tarball with checksum
+   verification, which is fine and also fixes a standing weakness — `provision-secrets.sh`
+   installs sops and age with a bare `sudo curl` and no verification at all.
+4. Vault-era naming persists inside OpenBao config (`vault { }` stanza,
+   `X-Vault-Wrap-TTL` header). Expect it; don't "fix" it.

infra/openbao/bootstrap-policies.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Second half of bootstrap: KV mount, policies, AppRoles, host credentials.
+# Run on vps2 with BAO_TOKEN set to a root token. Idempotent — safe to re-run.
+#
+#   export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
+#   export BAO_TOKEN=<root token>
+#   sudo -E bash infra/openbao/bootstrap-policies.sh
+#
+# Split from bootstrap.sh because everything here is safely automatable: none of it
+# produces a credential a human has to custody. The one thing that does — the recovery
+# material from `bao operator init` — is in bootstrap.sh and stops for the operator.
+set -euo pipefail
+
+MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
+SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+POLICY_DIR="$SELF_DIR/policies"
+
+: "${BAO_TOKEN:?BAO_TOKEN must be set to a root token}"
+: "${BAO_ADDR:=https://127.0.0.1:8200}"
+export BAO_ADDR
+command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; }
+
+echo "==> KV v2 mount at ${MOUNT}/"
+if bao secrets list -format=json | grep -q "\"${MOUNT}/\""; then
+  echo "    already mounted"
+else
+  bao secrets enable -path="$MOUNT" -version=2 kv
+fi
+
+echo "==> policies"
+for p in "$POLICY_DIR"/*.hcl; do
+  name="$(basename "$p" .hcl)"
+  bao policy write "$name" "$p" >/dev/null
+  echo "    wrote $name"
+done
+
+echo "==> approle auth"
+bao auth list -format=json | grep -q '"approle/"' || bao auth enable approle
+
+# One role per environment, CIDR-bound to the host that legitimately renders it.
+#
+# secret_id_ttl=0 (never expires) is deliberate: a secret-id that silently lapses
+# mid-quarter is a self-inflicted outage. The short lifetime lives on the TOKEN
+# instead — it only has to outlive one render.
+#
+# The CIDR binding is the part that makes a leaked secret-id close to useless: it is
+# only accepted from the correct WireGuard address.
+add_role() {
+  local role="$1" policy="$2" cidr="$3"
+  bao write "auth/approle/role/${role}" \
+    token_policies="$policy" \
+    secret_id_bound_cidrs="$cidr" \
+    token_bound_cidrs="$cidr" \
+    secret_id_ttl=0 secret_id_num_uses=0 \
+    token_ttl=5m token_max_ttl=15m token_num_uses=10 >/dev/null
+  echo "    role ${role} -> ${policy} (bound ${cidr})"
+}
+
+add_role tg-prod      tg-host-prod  10.10.0.1/32
+add_role tg-beta      tg-host-beta  10.10.0.1/32
+add_role tg-dev       tg-host-dev   10.10.0.2/32
+add_role tg-centralis tg-centralis  10.10.0.1/32
+
+# Install the local (vps2) credentials. prod and beta both render on this box.
+#
+# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
+# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
+# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
+# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
+# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
+# creates one.
+#
+# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
+# the one age key. This defends against MISCONFIGURATION, not against intrusion.
+install_creds() {
+  local role="$1" dest="$2" owner="$3"
+  local rid sid
+  rid=$(bao read -field=role_id "auth/approle/role/${role}/role-id")
+  sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
+  # umask before creation, so the file is never briefly world-readable.
+  ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
+  chown "$owner" "$dest"
+  chmod 0400 "$dest"
+  echo "    installed $dest (0400 $owner)"
+}
+
+if [ "$(id -u)" = 0 ]; then
+  install -d -o root -g root -m 0755 /etc/thermograph
+  install_creds tg-prod      /etc/thermograph/openbao-approle       "agent:agent"
+  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "deploy:deploy" \
+    || echo "    (beta creds skipped — no 'deploy' user on this box?)"
+  install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
+else
+  echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
+fi
+
+cat <<EOF
+
+==> done. Policies and AppRoles are in place; the vault is still EMPTY.
+
+dev's credentials must be installed on vps1, not here. From your own machine:
+
+  bao read -field=role_id  auth/approle/role/tg-dev/role-id
+  bao write -f -field=secret_id auth/approle/role/tg-dev/secret-id
+  # then, on vps1, write both lines to /etc/thermograph/openbao-approle (0400 agent)
+
+Prefer response wrapping so the secret-id never lands in shell history or a log:
+  bao write -f -wrap-ttl=5m auth/approle/role/tg-dev/secret-id   # gives a wrapping token
+  # on vps1:  bao unwrap <token>      (single-use: a failed unwrap means interception)
+
+Next: seed, then verify parity. Neither cuts anything over.
+  infra/openbao/seed-from-sops.sh --dry-run
+  infra/openbao/seed-from-sops.sh --all
+  infra/openbao/verify-parity.sh --all
+EOF

infra/openbao/bootstrap.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+# Bring up OpenBao on vps2 from nothing. Run ONCE, as root, ON vps2, BY THE OPERATOR.
+#
+#   sudo bash infra/openbao/bootstrap.sh
+#
+# ============================================================================
+# WHY THIS IS NOT AUTOMATED, AND MUST NOT BE
+# ============================================================================
+# `bao operator init` mints the credentials that become the new root of trust for the
+# entire estate: the recovery keys and the initial root token. Those have to be
+# CUSTODIED BY A HUMAN. Any place a script could put them — a file on the box, the
+# repo, a CI log, an agent transcript — is worse custody than the age key this
+# migration is meant to improve on.
+#
+# So this script does everything up to and including init, prints the recovery
+# material ONCE to the operator's terminal, and then stops and tells you what to do
+# with it. It deliberately does not store it, mail it, or push it anywhere.
+#
+# Everything AFTER custody (policies, AppRoles, seeding, parity checks) is automated,
+# because none of it produces a credential a human needs to hold.
+set -euo pipefail
+
+BAO_VERSION="${BAO_VERSION:-2.6.1}"
+MESH_ADDR="${MESH_ADDR:-10.10.0.1}"
+SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+
+[ "$(id -u)" = 0 ] || { echo "!! run as root (sudo bash $0)" >&2; exit 1; }
+
+hostname_now="$(hostname)"
+case "$hostname_now" in
+  vmi3453260) : ;;   # vps2
+  *)
+    # Refuse to bootstrap the secret store on the wrong box. Putting it on vps1 would
+    # invert the estate's central security decision: infra/CLAUDE.md states "vps1 must
+    # never hold prod credentials. It's the box that runs Forgejo, its CI runner, and
+    # dev's unreviewed branch — the opposite of an isolation boundary." With a static
+    # seal, the box also holds the key that decrypts everything.
+    #
+    # This mirrors the guard infra-sync.yml already has (commit 7583258, "refuse to
+    # render dev's vault onto a host that is not dev").
+    echo "!! this host is '$hostname_now', not vps2 (vmi3453260)." >&2
+    echo "!! OpenBao belongs on vps2. Refusing." >&2
+    exit 1
+    ;;
+esac
+
+echo "==> 1/7 install the bao binary (v${BAO_VERSION})"
+if ! command -v bao >/dev/null 2>&1; then
+  # NOTE: checksum verification. provision-secrets.sh installs sops and age with a bare
+  # `sudo curl` and no verification at all, which is a standing weakness (a compromised
+  # release URL yields a root binary that sees every plaintext). Do not copy that here.
+  # The checksums file is signed by the release; verify before installing.
+  tmpd="$(mktemp -d)"
+  base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
+  curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
+  curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
+  ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
+    echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
+    rm -rf "$tmpd"; exit 1
+  }
+  tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
+  install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
+  rm -rf "$tmpd"
+fi
+bao version
+
+echo "==> 2/7 user, directories, TLS"
+id -u openbao >/dev/null 2>&1 || useradd --system --home /var/lib/openbao --shell /usr/sbin/nologin openbao
+install -d -o openbao -g openbao -m 0700 /var/lib/openbao
+install -d -o openbao -g openbao -m 0750 /var/log/openbao
+install -d -o root    -g root    -m 0755 /etc/openbao
+install -d -o openbao -g openbao -m 0700 /etc/openbao/tls
+
+if [ ! -f /etc/openbao/tls/bao.crt ]; then
+  # Self-signed, 10 years, SAN on the mesh address. Deliberately NOT from Caddy/ACME:
+  # that would put DNS and the public internet in the boot chain of the secret store.
+  openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
+    -keyout /etc/openbao/tls/bao.key -out /etc/openbao/tls/bao.crt \
+    -subj "/CN=openbao.thermograph.internal" \
+    -addext "subjectAltName=IP:${MESH_ADDR},IP:127.0.0.1" >/dev/null 2>&1
+  chown openbao:openbao /etc/openbao/tls/bao.key /etc/openbao/tls/bao.crt
+  chmod 0400 /etc/openbao/tls/bao.key
+  chmod 0444 /etc/openbao/tls/bao.crt
+fi
+# Consumers need the cert to trust the listener.
+install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
+
+echo "==> 3/7 static seal key"
+if [ ! -f /etc/openbao/unseal.key ]; then
+  # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
+  ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
+  chown openbao:openbao /etc/openbao/unseal.key
+  chmod 0400 /etc/openbao/unseal.key
+  echo "    generated /etc/openbao/unseal.key (0400 openbao)"
+  echo "    ⚠️  COPY THIS OFF THE BOX into your password manager before proceeding."
+  echo "        Without an off-box copy, a raft snapshot is UNRESTORABLE."
+else
+  echo "    /etc/openbao/unseal.key already present, leaving alone"
+fi
+
+echo "==> 4/7 config + systemd unit"
+install -m 0640 -o root -g openbao "$SELF_DIR/config/openbao.hcl" /etc/openbao/config.hcl
+install -m 0644 -o root -g root "$SELF_DIR/openbao.service" /etc/systemd/system/openbao.service
+# logrotate MUST use SIGHUP: without it bao keeps writing to an unlinked inode and the
+# audit log silently stops growing — and a wedged audit device makes OpenBao stop
+# answering requests entirely, which would block every deploy on the estate.
+cat > /etc/logrotate.d/openbao <<'ROTATE'
+/var/log/openbao/audit.log {
+  daily
+  rotate 30
+  compress
+  missingok
+  notifempty
+  create 0600 openbao openbao
+  postrotate
+    systemctl kill --signal=SIGHUP openbao.service 2>/dev/null || true
+  endscript
+}
+ROTATE
+systemctl daemon-reload
+systemctl enable --now openbao.service
+sleep 3
+systemctl is-active --quiet openbao.service || {
+  echo "!! openbao.service failed to start; check: journalctl -u openbao -n 50" >&2
+  exit 1
+}
+
+export BAO_ADDR="https://127.0.0.1:8200"
+export BAO_CACERT=/etc/openbao/tls/bao.crt
+
+echo "==> 5/7 initialise"
+if bao status -format=json 2>/dev/null | grep -q '"initialized": *true'; then
+  echo "    already initialised, skipping"
+else
+  echo
+  echo "    ############################################################"
+  echo "    #  RECOVERY MATERIAL FOLLOWS. IT IS SHOWN EXACTLY ONCE.    #"
+  echo "    #  Store the 3 recovery keys in 3 DIFFERENT places.        #"
+  echo "    #  2 of 3 are needed to rekey or mint a new root token.    #"
+  echo "    ############################################################"
+  echo
+  # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
+  # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
+  # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
+  # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
+  bao operator init -recovery-shares=3 -recovery-threshold=2
+  echo
+  echo "    Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
+  echo "    stored off this machine."
+  read -r _
+fi
+
+bao status || true
+
+cat <<EOF
+
+==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
+
+Paste the initial root token (or one minted from recovery keys) and run:
+
+  export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
+  export BAO_TOKEN=<root token>
+  bash ${SELF_DIR}/bootstrap-policies.sh
+
+That script enables the KV mount, writes the policies, creates the AppRoles and
+installs each host's credentials. Then, from YOUR OWN machine (it needs your age
+key, and per infra/CLAUDE.md a script that reads production secrets is not for an
+agent to run):
+
+  infra/openbao/seed-from-sops.sh --dry-run     # check first
+  infra/openbao/seed-from-sops.sh --all
+  infra/openbao/verify-parity.sh --all          # must PASS before any flip
+
+Then revoke the root token:  bao token revoke -self
+
+NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
+every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
+until you flip an environment in a reviewed PR — dev first.
+EOF

infra/openbao/config/openbao.hcl

@@ -0,0 +1,118 @@
+// OpenBao server config for the Thermograph estate. Lives at /etc/openbao/config.hcl
+// on vps2. Verified against OpenBao v2.6.1 (2026-07-22).
+//
+// ============================================================================
+// WHY THIS RUNS AS A NATIVE SYSTEMD SERVICE AND NOT A CONTAINER
+// ============================================================================
+// The estate is Docker-native and this is the one thing that must not be. The rule
+// is that OpenBao must not depend on anything it secures.
+//
+//   * A Swarm service is fatal: deploy-stack.sh renders secrets in order to deploy
+//     the stack, so the stack cannot contain the thing that holds them.
+//   * A standalone `docker run --restart=always` is merely bad: it makes the vault's
+//     availability a function of the same Docker daemon that deploys restart, and
+//     leaves it one careless `docker system prune -a` from gone. The root CLAUDE.md
+//     already lists `prune -a` as a thing that eats the rollback image; it would
+//     eat this too.
+//
+// A native binary with Restart=always has exactly one dependency: the local disk.
+// See infra/openbao/openbao.service.
+//
+// Corollary that is easy to get wrong: the listener cert must NOT come from the
+// Caddy/ACME path. That would put DNS and the public internet in the boot chain of
+// the estate's secret store. It is self-signed, long-lived, and on disk.
+// ============================================================================
+
+ui           = false
+cluster_name = "thermograph"
+disable_mlock = true
+log_level    = "info"
+
+// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
+// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
+// a migration within a release. Raft also has the only first-class backup primitive
+// (`bao operator raft snapshot save`), which is the disaster-recovery story.
+//
+// A single-node raft is the honest shape for a one-operator estate. Note that a
+// TWO-node raft would be strictly worse than one: quorum of two means losing either
+// node loses write availability. Three voters would be needed, and the desktop
+// (10.10.0.3) is intermittent, so it cannot be a reliable third.
+storage "raft" {
+  path    = "/var/lib/openbao"
+  node_id = "vps2"
+}
+
+// Mesh-only. 10.10.0.1 is vps2's WireGuard address. There is deliberately no
+// 0.0.0.0 listener and no public DNS record — vps2 is a public VPS, and reaching
+// OpenBao should require being on the mesh, exactly like the Swarm control ports
+// and the Loki endpoint.
+listener "tcp" {
+  address       = "10.10.0.1:8200"
+  tls_cert_file = "/etc/openbao/tls/bao.crt"
+  tls_key_file  = "/etc/openbao/tls/bao.key"
+}
+
+// Loopback, so a by-hand `bao` on the box works without the mesh address.
+listener "tcp" {
+  address       = "127.0.0.1:8200"
+  tls_cert_file = "/etc/openbao/tls/bao.crt"
+  tls_key_file  = "/etc/openbao/tls/bao.key"
+}
+
+api_addr     = "https://10.10.0.1:8200"
+cluster_addr = "https://10.10.0.1:8201"
+
+// ============================================================================
+// SEAL: static auto-unseal. This is the crux of the whole design.
+// ============================================================================
+// The `static` seal (OpenBao v2.4.0+) auto-unseals from a local key with no cloud
+// KMS and no HSM. OpenBao's own docs hedge it — "carefully evaluate use of Static
+// Key Auto Unseal" — but that caveat is written for multi-tenant enterprises.
+//
+// The decisive argument here is that static seal is NOT A DOWNGRADE FROM THE STATUS
+// QUO. Today /etc/thermograph/age.key is one file, 0400, root-owned, on each
+// rendering host, which decrypts every secret in the estate forever with no audit
+// trail. A static seal key at 0400 on vps2 is the IDENTICAL trust model: one file on
+// one box whose compromise is total compromise. What changes is everything layered
+// above it — per-consumer ACLs, an audit log, five-minute tokens, versioned rollback.
+//
+// WHY NOT SHAMIR: `bao operator init -key-shares=5 -key-threshold=3` means the vault
+// is sealed after every process restart — reboot, package upgrade, OOM kill — and a
+// human must type three shares. render-secrets.sh is on the deploy path for all three
+// environments, so a sealed vault blocks every deploy and every hotfix. With one
+// operator the N-of-M threshold is theatre: one person holds all the shares, so it
+// buys nothing against the real threat while costing an availability property the
+// estate currently has for free. Today a Contabo reboot needs no human at all.
+// Introducing a human-in-the-loop requirement that does not exist today is a
+// regression this estate cannot absorb.
+//
+// THE ONE NEW SINGLE POINT OF FAILURE this migration introduces: lose this key and
+// the raft snapshots become unrestorable. It MUST exist in at least two places, one
+// of them not on vps2. That is a hard operator obligation, not a suggestion.
+//
+// Rotation is supported n-1 via previous_key / previous_key_id.
+seal "static" {
+  current_key_id = "20260801-1"
+  current_key    = "file:///etc/openbao/unseal.key"
+}
+
+// ============================================================================
+// AUDIT: two devices, deliberately.
+// ============================================================================
+// ⚠️ OpenBao STOPS ANSWERING REQUESTS ENTIRELY when no enabled audit device can
+// record them. That is correct behaviour for an audit log and a catastrophic way to
+// discover a full disk: a full /var on vps2 would block every deploy on the estate.
+// Two independent devices mean one wedging is survivable.
+//
+// Values are HMAC-SHA256'd in the log, so it is safe to ship to Loki through the
+// existing Alloy pipeline — which finally makes "who read which secret, when" a
+// queryable question. Today reading age.key leaves no trace whatsoever.
+//
+// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
+// unlinked inode and the log silently stops growing.
+audit "file" {
+  file_path = "/var/log/openbao/audit.log"
+  mode      = "0600"
+}
+
+audit "syslog" {}

infra/openbao/openbao.service

@@ -0,0 +1,50 @@
+# OpenBao, as a native systemd service on vps2. Install to
+# /etc/systemd/system/openbao.service.
+#
+# Native, not containerised, on purpose — see the header of config/openbao.hcl. The
+# whole point is that this unit's only dependency is the local disk: not the Docker
+# daemon that deploys restart, not the Swarm it would otherwise be deployed by, not
+# Caddy/ACME for its certificate, and not TimescaleDB.
+#
+# Note there is currently NO systemd unit of any kind loaded on either host for
+# Thermograph (infra/deploy/thermograph.service exists in the repo but is not
+# installed), so this is the first. That is deliberate: it is the one component that
+# must survive the container layer being broken.
+
+[Unit]
+Description=OpenBao secret store
+Documentation=https://openbao.org/docs/
+# network-online rather than plain network: the mesh listener binds 10.10.0.1, and
+# binding a WireGuard address before the interface is up fails the unit outright.
+After=network-online.target
+Wants=network-online.target
+# Explicitly NOT After=docker.service. Adding it would recreate the dependency this
+# design exists to avoid.
+ConditionFileNotEmpty=/etc/openbao/config.hcl
+
+[Service]
+User=openbao
+Group=openbao
+ExecStart=/usr/local/bin/bao server -config=/etc/openbao/config.hcl
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillSignal=SIGINT
+
+# Restart always, which together with the static seal is what makes a host reboot
+# self-healing. This pair is the reason deploys are not blocked by a reboot.
+Restart=always
+RestartSec=5
+
+# mlock is disabled in the config (disable_mlock = true), so no IPC_LOCK capability
+# is needed. If mlock is ever enabled, add: AmbientCapabilities=CAP_IPC_LOCK
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=full
+ProtectHome=yes
+# The three paths the service legitimately writes.
+ReadWritePaths=/var/lib/openbao /var/log/openbao
+# Raft + audit both do a lot of small writes; the default is too low for a store.
+LimitNOFILE=65536
+LimitCORE=0
+
+[Install]
+WantedBy=multi-user.target

infra/openbao/policies/tg-centralis.hcl

@@ -0,0 +1,67 @@
+// Policy for the Centralis control-plane container on vps2.
+//
+// Centralis is the tool an operator drives a rotation THROUGH, so it needs more than
+// a host render does: it must enumerate key names across every environment to build
+// `secrets_inventory`, and write values to perform `secrets_rotate`.
+//
+// The split below is the important part. Centralis gets:
+//   * METADATA read/list on every path  -> it can build a names-and-versions
+//     inventory across all three environments WITHOUT being able to read a value.
+//   * DATA write on every path          -> it can rotate.
+//   * DATA read on NOTHING.
+//
+// That is a genuine improvement on the SOPS design, not a port of it. Today
+// `secrets_inventory` can list key names without a decryption key only because SOPS
+// happens to leave key names as plaintext in the YAML — a property of the file
+// format, relied on deliberately (tools/secrets.ts:199 `extractKeyNames`). Under
+// OpenBao the same "names, never values" guarantee becomes an ENFORCED capability
+// boundary instead of a fortunate accident. Centralis cannot print a secret it is
+// not able to fetch.
+//
+// The `read` omission is load-bearing: Centralis holds the Docker socket (root
+// -equivalent on prod) and an SSH private key to beta and dev. Granting it data read
+// on prod's secrets would make a Centralis compromise equivalent to full estate
+// credential disclosure. It already effectively is via the Docker socket — but there
+// is no reason to hand it a second, easier path.
+
+// Names and version history across the whole tree — no values.
+path "thermograph/metadata/*" {
+  capabilities = ["read", "list"]
+}
+
+// Rotation: write a new version. Note "create" + "update" but NOT "read":
+// OpenBao permits a blind write, which is exactly the shape secrets_rotate wants —
+// it mints or receives a new value and stores it, and never needs the old one.
+path "thermograph/data/*" {
+  capabilities = ["create", "update"]
+}
+
+// Its OWN configuration is the one place Centralis may read, because it must render
+// /etc/centralis.env for itself. This is the direct analogue of centralis.prod.yaml.
+path "thermograph/data/centralis/prod" {
+  capabilities = ["read", "create", "update"]
+}
+
+// Version rollback, so a bad rotation is recoverable through the control plane
+// rather than requiring a shell on the box. This is the capability that replaces
+// "revert the PR" in the SOPS model.
+path "thermograph/undelete/*" {
+  capabilities = ["update"]
+}
+
+path "thermograph/destroy/*" {
+  capabilities = ["deny"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/renew-self" {
+  capabilities = ["update"]
+}
+
+// Read its own AppRole role-id for self-diagnosis, but never the secret-id.
+path "auth/approle/role/tg-centralis/role-id" {
+  capabilities = ["read"]
+}

infra/openbao/policies/tg-host-beta.hcl

@@ -0,0 +1,37 @@
+// Policy for BETA's render identity on vps2. Mirror of tg-host-prod.hcl.
+//
+// Beta legitimately reads the shared set: unlike dev, beta is on vps2 and already
+// co-resident with prod, so withholding `shared` from it would buy nothing while
+// breaking the render (beta's 24 live keys = beta.yaml's 8 + shared's 16).
+
+path "thermograph/data/common" {
+  capabilities = ["read"]
+}
+
+path "thermograph/metadata/common" {
+  capabilities = ["read", "list"]
+}
+
+path "thermograph/data/env/beta" {
+  capabilities = ["read"]
+}
+
+path "thermograph/metadata/env/beta" {
+  capabilities = ["read", "list"]
+}
+
+path "thermograph/data/env/prod" {
+  capabilities = ["deny"]
+}
+
+path "thermograph/data/env/dev" {
+  capabilities = ["deny"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/renew-self" {
+  capabilities = ["update"]
+}

infra/openbao/policies/tg-host-dev.hcl

@@ -0,0 +1,72 @@
+// Policy for the DEV host's render identity (vps1).
+//
+// THIS FILE IS THE POINT OF THE MIGRATION.
+//
+// infra/CLAUDE.md states the rule: "vps1 must never hold prod credentials. It's the
+// box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of
+// an isolation boundary."
+//
+// Today that rule is enforced by a SHELL FLAG (THERMOGRAPH_SECRETS_SKIP_COMMON=1) set
+// by the *caller*, at three separate call sites: env-topology.sh:196 -> deploy.sh:67,
+// deploy-dev.sh:86, and infra-sync.yml:93-95. The renderer itself does not know that
+// env=dev means never-common. A fourth call site, or one by-hand
+// `render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3
+// keypairs (one read-write on the bucket holding prod's database backups), the VAPID
+// private key that signs Web Push to real subscribers, REGISTRY_TOKEN, and the
+// IndexNow and metrics tokens onto the Forgejo CI-runner box — and exits 0.
+//
+// Under this policy that is no longer possible to get wrong. dev's identity cannot
+// read the common path. A misconfigured caller gets a 403 from OpenBao, not a silent
+// success with production credentials on disk. The failure class is gone, not guarded.
+
+// dev's own values. This is the ONLY secret path dev can read.
+path "thermograph/data/env/dev" {
+  capabilities = ["read"]
+}
+
+path "thermograph/metadata/env/dev" {
+  capabilities = ["read", "list"]
+}
+
+// EXPLICIT DENY on the shared production credential set.
+//
+// A deny rule is redundant with OpenBao's default-deny — anything not granted is
+// already refused. It is here anyway, and must stay, for two reasons:
+//   1. It is documentation that survives refactoring. Someone widening dev's grants
+//      has to delete an explicit deny to break the rule, which is a much louder edit
+//      than adding a path to an allow list.
+//   2. In OpenBao, a deny on a path beats any grant at that path regardless of rule
+//      order or specificity. So if dev's identity is ever accidentally given a
+//      broader policy alongside this one, this still wins.
+path "thermograph/data/common" {
+  capabilities = ["deny"]
+}
+
+path "thermograph/metadata/common" {
+  capabilities = ["deny"]
+}
+
+// Deny the other environments outright. Same argument: dev has no business reading
+// beta or prod, and being explicit means a widened grant elsewhere cannot silently
+// re-open it.
+path "thermograph/data/env/prod" {
+  capabilities = ["deny"]
+}
+
+path "thermograph/data/env/beta" {
+  capabilities = ["deny"]
+}
+
+path "thermograph/data/centralis/*" {
+  capabilities = ["deny"]
+}
+
+// Token self-management, so the render can look up and renew its own lease without
+// needing a broader grant.
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/renew-self" {
+  capabilities = ["update"]
+}

infra/openbao/policies/tg-host-prod.hcl

@@ -0,0 +1,44 @@
+// Policy for PROD's render identity on vps2.
+//
+// Reads the shared set plus prod's own overrides — the OpenBao equivalent of
+// "common.yaml then prod.yaml, last-wins".
+//
+// Note what this policy does NOT grant: beta. vps2 runs prod and beta side by side
+// with one filesystem and one SSH credential, so the host is emphatically not an
+// isolation boundary between them (infra/CLAUDE.md is explicit about this). Separate
+// identities with separate policies do not fix that — a foothold on the box can read
+// both AppRole credentials — but they do mean a *mistake* cannot cross the line:
+// prod's render cannot accidentally pull beta's database password, and the audit log
+// attributes every read to one environment or the other.
+
+path "thermograph/data/common" {
+  capabilities = ["read"]
+}
+
+path "thermograph/metadata/common" {
+  capabilities = ["read", "list"]
+}
+
+path "thermograph/data/env/prod" {
+  capabilities = ["read"]
+}
+
+path "thermograph/metadata/env/prod" {
+  capabilities = ["read", "list"]
+}
+
+path "thermograph/data/env/beta" {
+  capabilities = ["deny"]
+}
+
+path "thermograph/data/env/dev" {
+  capabilities = ["deny"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}
+
+path "auth/token/renew-self" {
+  capabilities = ["update"]
+}

infra/openbao/policies/tg-ops-backup.hcl

@@ -0,0 +1,50 @@
+// Policy for the ops-cron backup job (.forgejo/workflows/ops-cron.yml).
+//
+// Exists to DELETE FOUR CI SECRETS. Today S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY and
+// S3_SECRET_KEY are Forgejo repo secrets, even though ops-cron only ever uses them
+// INSIDE the script it SSHes onto the host — they are passed through `envs:` and
+// consumed as RCLONE_CONFIG_ARCHIVE_*. Nothing needs them on the runner. Moving them
+// here means the host reads them directly and CI holds four fewer credentials.
+//
+// It also collapses a genuine duplication: the same values already live in the SOPS
+// vault as THERMOGRAPH_S3_* in common.yaml, so there are currently two sources of
+// truth for one credential pair, free to drift.
+//
+// The backup recipient belongs here too — see thermograph/data/ops/backup below.
+// ops-cron.yml:142 and :197 currently HARDCODE the age recipient as a literal, in two
+// places, and that same public key is duplicated in eleven places across the repo.
+
+path "thermograph/data/ops/backup" {
+  capabilities = ["read"]
+}
+
+path "thermograph/metadata/ops/backup" {
+  capabilities = ["read", "list"]
+}
+
+// ============================================================================
+// THE AGE KEY IS NOT RETIRED BY THIS MIGRATION. This path is why.
+// ============================================================================
+// ops-cron.yml streams every off-box database dump through
+// `pg_dump | age -r <recipient> | rclone rcat`, with 30-day S3 retention, and restore
+// depends on the private half at /etc/thermograph/age.key. The Forgejo backup path is
+// the same shape.
+//
+// So the age keypair is not merely the SOPS transport — it is the BACKUP ENCRYPTION
+// KEY. Deleting it alongside the SOPS vault files would silently destroy up to 30 days
+// of database recoverability, and nothing would notice until a restore was attempted.
+//
+// Therefore: the age private key is stored at thermograph/data/legacy/age (operator
+// access only, deliberately NOT granted by this policy) and kept on disk until either
+// the newest age-encrypted object in S3 has aged out, or the backup path is re-pointed
+// at a dedicated backup recipient. Only then may /etc/thermograph/age.key be removed.
+//
+// The `deny` is explicit so that widening this policy cannot hand the backup job the
+// key to read everything else in the estate.
+path "thermograph/data/legacy/age" {
+  capabilities = ["deny"]
+}
+
+path "auth/token/lookup-self" {
+  capabilities = ["read"]
+}

infra/openbao/seed-from-sops.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+# Seed OpenBao from the SOPS vault. One-shot, idempotent, run BY THE OPERATOR.
+#
+#   infra/openbao/seed-from-sops.sh --dry-run          # key names only, writes nothing
+#   infra/openbao/seed-from-sops.sh --env dev          # seed one environment
+#   infra/openbao/seed-from-sops.sh --all              # seed everything
+#
+# ============================================================================
+# THIS SCRIPT READS EVERY PRODUCTION SECRET IN THE ESTATE IN PLAINTEXT.
+# ============================================================================
+# infra/CLAUDE.md and deploy/secrets/README.md both say that
+# deploy/secrets/seed-from-live.sh "reads production secrets and is explicitly NOT for
+# an agent to run". This script is in the same category and inherits the same rule:
+# run it yourself, from your own terminal, with your own age key.
+#
+# It never prints a value. Every diagnostic is key NAMES and counts, so the output is
+# safe to paste into an issue. --dry-run makes that guarantee mechanical: it resolves
+# and validates everything, then reports what it WOULD write without contacting
+# OpenBao at all.
+#
+# Why seed from SOPS rather than re-entering values by hand: the safety property of
+# this entire migration is that the OpenBao render can be diffed BYTE-FOR-BYTE against
+# the SOPS render (see verify-parity.sh). That only works if the values are identical.
+# Re-typing 61 values would silently rotate whichever ones were mistyped, and the
+# self-generating ones (AUTH_SECRET, the VAPID pair) fail SILENTLY when wrong — the app
+# mints a replacement, comes up green, and invalidates every session and push
+# subscription. So: copy exactly now, rotate deliberately later.
+set -euo pipefail
+
+MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
+AGE_KEY="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
+DRY_RUN=0
+TARGETS=()
+
+# Resolve the repo's infra/ directory from this script's own location, so the script
+# works from any cwd. This is the same mistake terraform's remote-exec makes
+# (main.tf:256 passes app_dir where app_dir/infra is wanted), so it is worth being
+# explicit rather than clever.
+SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)"
+VAULT_DIR="$INFRA_DIR/deploy/secrets"
+
+usage() {
+  cat >&2 <<'EOF'
+usage: seed-from-sops.sh [--dry-run] (--all | --env <name> ...)
+
+  --dry-run        validate and report key names/counts; contact OpenBao not at all
+  --all            seed common, dev, beta, prod and centralis.prod
+  --env <name>     seed one of: common | dev | beta | prod | centralis.prod
+
+Environment:
+  SOPS_AGE_KEY_FILE   age private key (default ~/.config/sops/age/keys.txt)
+  BAO_ADDR            OpenBao address (default https://10.10.0.1:8200)
+  BAO_TOKEN           an operator token with write on <mount>/data/*
+  THERMOGRAPH_BAO_MOUNT   KV v2 mount name (default thermograph)
+EOF
+  exit 2
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --dry-run) DRY_RUN=1; shift ;;
+    --all) TARGETS=(common dev beta prod centralis.prod); shift ;;
+    --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;;
+    -h|--help) usage ;;
+    *) echo "!! unknown argument: $1" >&2; usage ;;
+  esac
+done
+
+[ ${#TARGETS[@]} -gt 0 ] || usage
+
+# The KV v2 path for a given vault file. common.yaml and centralis.prod.yaml are
+# special-cased; everything else is an environment overlay.
+#
+# Keeping `common` at its own top-level path rather than under env/ is what makes the
+# dev policy expressible: tg-host-dev.hcl denies exactly one path, and no wildcard
+# over env/* can accidentally re-include it.
+bao_path_for() {
+  case "$1" in
+    common)         printf 'common\n' ;;
+    centralis.prod) printf 'centralis/prod\n' ;;
+    dev|beta|prod)  printf 'env/%s\n' "$1" ;;
+    *) echo "!! unknown vault target: $1" >&2; return 1 ;;
+  esac
+}
+
+[ -f "$AGE_KEY" ] || { echo "!! age key not found at $AGE_KEY" >&2; exit 1; }
+command -v sops >/dev/null 2>&1 || { echo "!! sops not on PATH" >&2; exit 1; }
+command -v python3 >/dev/null 2>&1 || { echo "!! python3 not on PATH" >&2; exit 1; }
+
+if [ "$DRY_RUN" = 0 ]; then
+  command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; }
+  [ -n "${BAO_TOKEN:-}" ] || { echo "!! BAO_TOKEN is not set" >&2; exit 1; }
+  export BAO_ADDR="${BAO_ADDR:-https://10.10.0.1:8200}"
+fi
+
+rc=0
+for target in "${TARGETS[@]}"; do
+  src="$VAULT_DIR/${target}.yaml"
+  if [ ! -f "$src" ]; then
+    echo "!! no vault file at $src" >&2
+    rc=1
+    continue
+  fi
+
+  path="$(bao_path_for "$target")" || { rc=1; continue; }
+
+  # Decrypt to JSON, not dotenv. JSON is lossless: the dotenv writer flattens a value
+  # containing a newline into a literal \n that is indistinguishable from a backslash
+  # followed by n (render-secrets.sh:198 documents this). Seeding through dotenv would
+  # bake that ambiguity into OpenBao permanently.
+  plain_json="$(SOPS_AGE_KEY_FILE="$AGE_KEY" \
+      sops -d --input-type yaml --output-type json "$src" 2>/dev/null)" || {
+    echo "!! failed to decrypt $src" >&2
+    rc=1
+    continue
+  }
+
+  # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
+  # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
+  # SEED time — before anything depends on it — rather than at deploy time.
+  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
+import json, os, re, sys
+
+doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
+if not isinstance(doc, dict) or not doc:
+    sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
+    sys.exit(1)
+
+KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+UNSAFE = set("`$\\\"'")
+
+out, bad = {}, []
+for key, val in doc.items():
+    if key == "sops":
+        continue
+    if not KEY_RE.match(key):
+        bad.append(f"{key}: not a valid env var name")
+        continue
+    if val is None:
+        val = ""
+    if not isinstance(val, str):
+        val = json.dumps(val, separators=(",", ":"))
+    if "\n" in val or "\r" in val:
+        bad.append(f"{key}: contains a newline")
+        continue
+    if UNSAFE & set(val):
+        bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
+        continue
+    out[key] = val
+
+if bad:
+    sys.stderr.write("!! unsafe values, refusing:\n")
+    for line in bad:
+        sys.stderr.write(f"!!   {line}\n")
+    sys.exit(1)
+
+# stdout line 1: space-separated key NAMES, for the operator-visible log.
+# stdout line 2+: the JSON payload, fed to `bao kv put` on stdin.
+sys.stdout.write(" ".join(sorted(out)) + "\n")
+sys.stdout.write(json.dumps(out) + "\n")
+PY
+)" || { echo "!! validation failed for $target" >&2; rc=1; continue; }
+
+  names="$(printf '%s\n' "$reshaped" | sed -n '1p')"
+  payload="$(printf '%s\n' "$reshaped" | sed -n '2p')"
+  count="$(printf '%s\n' "$names" | wc -w | tr -d ' ')"
+
+  if [ "$DRY_RUN" = 1 ]; then
+    echo "== ${target}.yaml -> ${MOUNT}/${path}  (${count} keys, DRY RUN, nothing written)"
+    printf '   %s\n' "$names"
+    continue
+  fi
+
+  # `bao kv put -` reads a JSON object from stdin, so no value ever appears in argv
+  # (where it would be visible in /proc and in the shell's history).
+  if printf '%s' "$payload" | bao kv put -mount="$MOUNT" "$path" - >/dev/null; then
+    echo "== ${target}.yaml -> ${MOUNT}/${path}  (${count} keys written)"
+  else
+    echo "!! failed writing ${MOUNT}/${path}" >&2
+    rc=1
+  fi
+done
+
+if [ "$rc" = 0 ] && [ "$DRY_RUN" = 0 ]; then
+  cat <<EOF
+
+Seeded. NOTHING is cut over yet — SOPS is still authoritative for every
+environment, because TG_SECRETS_BACKEND defaults to sops in env-topology.sh.
+
+Next: prove equivalence before flipping anything.
+  infra/openbao/verify-parity.sh --env dev
+EOF
+fi
+
+exit "$rc"

infra/openbao/verify-parity.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+# Prove that the OpenBao render is EQUIVALENT to the SOPS render, without printing a
+# single secret value.
+#
+#   infra/openbao/verify-parity.sh --env dev
+#   infra/openbao/verify-parity.sh --all
+#
+# This is the gate for every cutover. The migration's entire safety argument is that
+# the two backends produce the same KEY=value set, so flipping TG_SECRETS_BACKEND
+# cannot change what any service sees. That argument has to be TESTED, not asserted —
+# especially because the failure modes on the other side are silent:
+#
+#   * a missing THERMOGRAPH_AUTH_SECRET makes the app mint a random one per worker
+#     (accounts/users.py:33), so logins start working intermittently and nothing logs;
+#   * a half-rendered VAPID pair makes push.py generate a NEW keypair and then delete
+#     every existing subscription row as "gone" (push.py:180-184);
+#   * a wrong POSTGRES_PASSWORD makes the data layer swallow the error and refetch from
+#     Open-Meteo, spending third-party quota, while /healthz stays green.
+#
+# None of those announce themselves. A byte-level diff beforehand is the only thing
+# standing between a flag flip and one of them.
+#
+# OUTPUT DISCIPLINE: only key NAMES and counts are ever printed. Where values differ,
+# the key name is reported and the values are not — the same rule
+# deploy/secrets/dry-run-render.sh already follows.
+set -euo pipefail
+
+SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)"
+TARGETS=()
+
+usage() {
+  cat >&2 <<'EOF'
+usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)
+
+Renders each environment through BOTH backends and compares the resulting
+KEY=value sets. Prints key names and counts only, never values.
+
+Exit status: 0 = every requested environment is at parity; 1 = a mismatch.
+Suitable as a CI / cron gate.
+EOF
+  exit 2
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --all) TARGETS=(dev beta prod); shift ;;
+    --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;;
+    -h|--help) usage ;;
+    *) echo "!! unknown argument: $1" >&2; usage ;;
+  esac
+done
+[ ${#TARGETS[@]} -gt 0 ] || usage
+
+# shellcheck source=/dev/null
+. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
+
+overall=0
+for env_name in "${TARGETS[@]}"; do
+  echo "== parity check: $env_name"
+
+  # dev renders WITHOUT common on both backends. On the SOPS side that is a caller
+  # flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps
+  # the comparison apples-to-apples — and if the flag were wrong, the OpenBao side
+  # would fail closed with a 403 rather than quietly diverge, which is exactly the
+  # improvement being verified.
+  skip_common=0
+  [ "$env_name" = dev ] && skip_common=1
+
+  sops_out=$(mktemp); bao_out=$(mktemp)
+  # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
+  # trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a
+  # sourced context persists into the caller's shell and re-fires on the next source).
+  cleanup() { rm -f "$sops_out" "$bao_out"; }
+
+  # --- SOPS side: common (unless dev) then <env>, appended, last-wins ---
+  key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}"
+  [ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
+  if [ ! -r "$key" ]; then
+    echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2
+    cleanup; overall=1; continue
+  fi
+
+  if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then
+    SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
+      "$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || {
+        echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; }
+  fi
+  SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
+    "$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || {
+      echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; }
+
+  # --- OpenBao side ---
+  if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \
+       thermograph_openbao_source "$env_name" > "$bao_out"; then
+    echo "!! OpenBao render failed for $env_name" >&2
+    cleanup; overall=1; continue
+  fi
+
+  # --- Compare ---
+  # The SOPS output can legitimately contain the SAME key twice (common then env), and
+  # every consumer takes the LAST occurrence — `env_file:` and `source` both do. So
+  # collapsing to last-wins is not a normalisation for convenience, it is what the
+  # consumers actually see. Comparing raw would report a false mismatch the moment a
+  # key is overridden.
+  norm() {
+    python3 -c '
+import sys
+seen = {}
+order = []
+for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
+    line = line.rstrip("\n")
+    if not line or line.startswith("#") or "=" not in line:
+        continue
+    k, _, v = line.partition("=")
+    if k not in seen:
+        order.append(k)
+    seen[k] = v          # last occurrence wins
+for k in sorted(order):
+    print(f"{k}={seen[k]}")
+' "$1"
+  }
+
+  a=$(mktemp); b=$(mktemp)
+  norm "$sops_out" > "$a"; norm "$bao_out" > "$b"
+
+  n_sops=$(wc -l < "$a" | tr -d ' ')
+  n_bao=$(wc -l < "$b" | tr -d ' ')
+
+  if cmp -s "$a" "$b"; then
+    echo "   PASS — ${n_sops} keys, byte-identical after last-wins collapse"
+  else
+    overall=1
+    echo "   FAIL — sops=${n_sops} keys, openbao=${n_bao} keys"
+    # Report only NAMES. Three buckets, because the fix differs per bucket:
+    #   only-in-sops  -> the seed missed a key (re-run seed-from-sops.sh)
+    #   only-in-bao   -> OpenBao has something the vault does not (stale/extra write)
+    #   value-differs -> the seed captured a different value; DO NOT flip the backend
+    comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/   only in SOPS:    /'
+    comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/   only in OpenBao: /'
+    join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \
+      | awk -F= '$2 != $3 { print "   value differs:   " $1 }' | sort -u
+  fi
+
+  rm -f "$a" "$b"
+  cleanup
+done
+
+if [ "$overall" = 0 ]; then
+  echo
+  echo "PARITY OK for: ${TARGETS[*]}"
+  echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
+  echo "Recommended gate before prod: 7 consecutive green runs on all three."
+else
+  echo
+  echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
+fi
+exit "$overall"