From 702bcd920cb4bce62ca654c40830bc8e5cd6d69b Mon Sep 17 00:00:00 2001
From: Emi Griffith <me@emigriffith.dev>
Date: Thu, 30 Jul 2026 19:47:22 -0700
Subject: [PATCH 1/5] openbao: fix bootstrap so it completes on 2.6.x
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

bootstrap.sh could not run to completion on OpenBao 2.6.1:

- Release asset names were wrong. `bao_<ver>_linux_amd64.tar.gz` and
  `bao_<ver>_SHA256SUMS` both 404; the assets are `openbao_<ver>_...` and
  `checksums.txt`. The tarball is now saved under its real name and the checksum
  grep anchored to end-of-line, because checksums.txt also lists a .sbom.json and
  `sha256sum -c` resolves each line by the filename inside it.
- `openssl rand -base64 32` appends a newline and the static seal reads the key
  file raw, so the service failed with `Error configuring seal "static": unknown
  encoding for AES-256 key`.
- The audit stanza relied on the block label being the device type. 2.6.1 requires
  explicit `type` and `path` with device settings under `options`. Worth noting the
  intermediate state: with type and path but a bare `file_path`, the server starts
  and silently ignores the log location, which is the worse failure given a wedged
  audit device stops OpenBao answering at all.
- `disable_mlock` is unsupported in 2.6.1 and warned on every start.
- Nothing opened port 8200 and ufw defaults to deny(incoming), so vps1 could not
  reach the vault. dev renders on vps1, so this would have surfaced as a timeout
  during cutover rather than here.

bootstrap-policies.sh installed beta's credentials as `deploy:deploy`, but vps2 has
no `deploy` user and both environments deploy as `agent` (env-topology.sh sets
TG_SSH_TARGET=agent@ for both; deploy.yml uses one VPS2_SSH_USER). install_creds
also printed a success line after a failed chown: the call site wrapped it in
`|| echo`, which suppresses `set -e` for everything inside the function, so it fell
through to chmod and reported an ownership it never applied. It now removes the
half-written file and returns non-zero.

Corrects the isolation rationale accordingly — separate credentials buy audit
attribution and a policy boundary against mistakes, not deploy-user isolation.

Documents the 2.6.0 root-token change (HCSEC-2026-08): `generate-root` now targets
an authenticated endpoint, so recovery keys cannot mint a replacement token and
revoking the last root token before a second admin identity exists is a one-way
door. Also corrects the runbook's `verify-parity.sh --all`, which cannot work from
a single host given the AppRole CIDR bindings.
---
 infra/openbao/README.md             | 41 +++++++++++++--
 infra/openbao/bootstrap-policies.sh | 37 +++++++++----
 infra/openbao/bootstrap.sh          | 80 +++++++++++++++++++++++------
 infra/openbao/config/openbao.hcl    | 24 +++++++--
 4 files changed, 147 insertions(+), 35 deletions(-)

diff --git a/infra/openbao/README.md b/infra/openbao/README.md
index 196784a..037bfcb 100644
--- a/infra/openbao/README.md
+++ b/infra/openbao/README.md
@@ -148,22 +148,53 @@ SOPS and retiring age are two different projects.
 ### Stand it up (once, by the operator, on vps2)
 
 ```sh
-sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, unit, init
+sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, ufw, unit, init
 # custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
-export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
+export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
 export BAO_TOKEN=<root token>
 sudo -E bash infra/openbao/bootstrap-policies.sh   # mount, policies, approles
-bao token revoke -self
 ```
 
-### Seed and prove (from your own machine — it needs your age key)
+> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
+> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
+> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
+> minting a root token requires a token you already have. **The recovery keys are not
+> a way back in**; they rekey, they do not authenticate. Unless
+> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
+> the last root token locks you out of an otherwise healthy, unsealed vault, and the
+> only remedy is re-initialising from scratch. Revoke only once a second admin
+> identity exists.
+
+Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
+`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
+openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
+with `certificate signed by unknown authority`.
+
+### Seed and prove (on vps2 — the age key is already there)
 
 ```sh
+export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
 infra/openbao/seed-from-sops.sh --dry-run     # key names + counts, writes nothing
 infra/openbao/seed-from-sops.sh --all
-infra/openbao/verify-parity.sh --all          # MUST pass before any flip
 ```
 
+Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
+to the host that legitimately renders it, so an environment can only be verified from
+its own host:
+
+```sh
+# on vps2
+infra/openbao/verify-parity.sh --env prod                          # expect 32 keys
+THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \
+  infra/openbao/verify-parity.sh --env beta                        # expect 24 keys
+# on vps1
+cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev  # expect 12 keys
+```
+
+The explicit `THERMOGRAPH_BAO_APPROLE` for beta is a stopgap: `env-topology.sh` does
+not yet derive the AppRole path per environment, so the renderer falls back to prod's
+credentials and `tg-host-prod.hcl` correctly denies `env/beta`.
+
 `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
 the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
 inherits that rule.
diff --git a/infra/openbao/bootstrap-policies.sh b/infra/openbao/bootstrap-policies.sh
index 81d07bd..2104e51 100755
--- a/infra/openbao/bootstrap-policies.sh
+++ b/infra/openbao/bootstrap-policies.sh
@@ -63,15 +63,21 @@ add_role tg-centralis tg-centralis  10.10.0.1/32
 
 # Install the local (vps2) credentials. prod and beta both render on this box.
 #
-# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
-# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
-# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
-# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
-# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
-# creates one.
+# Both files are owned by `agent`, because that is the single identity both
+# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
+# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
+# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
+# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
+# they do not, and the chown simply failed.
 #
-# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
-# the one age key. This defends against MISCONFIGURATION, not against intrusion.
+# So be accurate about what separate credentials buy on this box: NOT deploy-user
+# isolation — there is one deploy user, and it can read both files. What they buy is
+# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
+# *mistake* crossing between environments: a beta render authenticating with beta's
+# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
+# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
+# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
+# a separate SSH credential for beta in deploy.yml — a larger change than this script.
 install_creds() {
   local role="$1" dest="$2" owner="$3"
   local rid sid
@@ -79,7 +85,17 @@ install_creds() {
   sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
   # umask before creation, so the file is never briefly world-readable.
   ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
-  chown "$owner" "$dest"
+  # Abort the function if chown fails, and remove the half-installed file. The call
+  # site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
+  # so without this check a failed chown falls straight through to chmod and prints a
+  # line asserting an ownership that was never applied. A root-owned credential the
+  # renderer cannot read, reported as installed, is worse than no credential at all:
+  # it fails at deploy time instead of here.
+  if ! chown "$owner" "$dest"; then
+    rm -f "$dest"
+    echo "    !! chown $owner failed for $dest — removed, nothing installed" >&2
+    return 1
+  fi
   chmod 0400 "$dest"
   echo "    installed $dest (0400 $owner)"
 }
@@ -87,8 +103,7 @@ install_creds() {
 if [ "$(id -u)" = 0 ]; then
   install -d -o root -g root -m 0755 /etc/thermograph
   install_creds tg-prod      /etc/thermograph/openbao-approle       "agent:agent"
-  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "deploy:deploy" \
-    || echo "    (beta creds skipped — no 'deploy' user on this box?)"
+  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "agent:agent"
   install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
 else
   echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh
index 39fc5e6..f3d0f69 100755
--- a/infra/openbao/bootstrap.sh
+++ b/infra/openbao/bootstrap.sh
@@ -52,13 +52,19 @@ if ! command -v bao >/dev/null 2>&1; then
   # The checksums file is signed by the release; verify before installing.
   tmpd="$(mktemp -d)"
   base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
-  curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
-  curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
-  ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
+  # Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
+  # The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
+  tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
+  curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
+  curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
+  # Anchor to end-of-line and save under the real asset name: checksums.txt also lists
+  # <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
+  # so an unanchored match fails on a file that was never fetched.
+  ( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
     echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
     rm -rf "$tmpd"; exit 1
   }
-  tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
+  tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
   install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
   rm -rf "$tmpd"
 fi
@@ -88,7 +94,11 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
 echo "==> 3/7 static seal key"
 if [ ! -f /etc/openbao/unseal.key ]; then
   # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
-  ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
+  # tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
+  # static seal reads this file RAW. That one trailing byte makes the key neither
+  # 32 raw bytes nor clean base64, so bao refuses to start with
+  # `Error configuring seal "static": unknown encoding for AES-256 key`.
+  ( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
   chown openbao:openbao /etc/openbao/unseal.key
   chmod 0400 /etc/openbao/unseal.key
   echo "    generated /etc/openbao/unseal.key (0400 openbao)"
@@ -125,6 +135,24 @@ systemctl is-active --quiet openbao.service || {
   exit 1
 }
 
+# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
+# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
+# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
+# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
+# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
+# desktop, the phone — has any business reaching the secret store.
+if command -v ufw >/dev/null 2>&1; then
+  if ufw status | grep -q '8200.*10\.10\.0\.2'; then
+    echo "    ufw: 8200/tcp from 10.10.0.2 already allowed"
+  else
+    ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
+      comment 'vps1/dev -> openbao' >/dev/null
+    echo "    ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
+  fi
+else
+  echo "    !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
+fi
+
 export BAO_ADDR="https://127.0.0.1:8200"
 export BAO_CACERT=/etc/openbao/tls/bao.crt
 
@@ -140,9 +168,16 @@ else
   echo "    ############################################################"
   echo
   # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
-  # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
-  # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
-  # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
+  # yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
+  # is redundancy against loss, not separation of duty — there is one operator.
+  #
+  # These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
+  # the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
+  # (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
+  # minting a root token requires a token you already have. The deprecated endpoint is
+  # off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
+  # Consequence: revoking the last root token before another admin identity exists is
+  # a one-way door. See the ordering note in the handoff below.
   bao operator init -recovery-shares=3 -recovery-threshold=2
   echo
   echo "    Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
@@ -156,22 +191,37 @@ cat <<EOF
 
 ==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
 
-Paste the initial root token (or one minted from recovery keys) and run:
+Paste the initial root token printed above and run:
 
   export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
   export BAO_TOKEN=<root token>
   bash ${SELF_DIR}/bootstrap-policies.sh
 
+KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
+cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
+\`bao token revoke -self\` locks you out of an otherwise healthy vault.
+
 That script enables the KV mount, writes the policies, creates the AppRoles and
-installs each host's credentials. Then, from YOUR OWN machine (it needs your age
-key, and per infra/CLAUDE.md a script that reads production secrets is not for an
-agent to run):
+installs each host's credentials. Then seed — on THIS box, where the age key already
+lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
+secrets is not for an agent to run:
 
-  infra/openbao/seed-from-sops.sh --dry-run     # check first
+  export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
+  infra/openbao/seed-from-sops.sh --dry-run     # check first: 16/12/8/16 keys
   infra/openbao/seed-from-sops.sh --all
-  infra/openbao/verify-parity.sh --all          # must PASS before any flip
 
-Then revoke the root token:  bao token revoke -self
+Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
+that legitimately renders it, so an environment can only be verified from its own host.
+
+  # here (vps2) -- beta needs its own AppRole, prod uses the default
+  infra/openbao/verify-parity.sh --env prod                        # expect 32 keys
+  THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \\
+    infra/openbao/verify-parity.sh --env beta                      # expect 24 keys
+  # on vps1
+  cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev   # expect 12 keys
+
+Only once all three PASS, and only after you have a second admin identity that is not
+the root token, revoke it:  bao token revoke -self
 
 NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
 every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
diff --git a/infra/openbao/config/openbao.hcl b/infra/openbao/config/openbao.hcl
index 07f4d87..b9175de 100644
--- a/infra/openbao/config/openbao.hcl
+++ b/infra/openbao/config/openbao.hcl
@@ -25,8 +25,12 @@
 
 ui           = false
 cluster_name = "thermograph"
-disable_mlock = true
 log_level    = "info"
+// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
+// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
+// setting the server ignores only trains the operator to skim startup warnings,
+// which is where a real one will eventually hide. The corresponding note about
+// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
 
 // STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
 // DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
@@ -110,9 +114,21 @@ seal "static" {
 //
 // logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
 // unlinked inode and the log silently stops growing.
+// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
+// device NAME, not its type — and device settings go in `options`. Written as
+// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
+// be specified"); with type+path but a bare file_path it starts but silently
+// IGNORES the path and mode, which is the worse failure of the two.
 audit "file" {
-  file_path = "/var/log/openbao/audit.log"
-  mode      = "0600"
+  type = "file"
+  path = "file/"
+  options = {
+    file_path = "/var/log/openbao/audit.log"
+    mode      = "0600"
+  }
 }
 
-audit "syslog" {}
+audit "syslog" {
+  type = "syslog"
+  path = "syslog/"
+}

From 5001269b374c35cab5cd2b6984680dd9c29bbad7 Mon Sep 17 00:00:00 2001
From: Emi Griffith <me@emigriffith.dev>
Date: Thu, 30 Jul 2026 19:49:28 -0700
Subject: [PATCH 2/5] openbao: derive the AppRole credential path per
 environment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

render-secrets-openbao.sh resolved the AppRole file to a single host-wide default,
/etc/thermograph/openbao-approle — which is prod's. On vps2 that made beta's render
authenticate as tg-prod, and tg-host-prod.hcl denies thermograph/data/env/beta, so
beta could never render from OpenBao. It surfaces as `cannot read
thermograph/env/beta`: the policy working correctly against the wrong identity.

env-topology.sh exists precisely because vps2 runs two environments and one
host-wide marker cannot answer "which environment is this". The AppRole path is the
same question and had been given the same host-wide answer. It is now derived per
environment alongside host, checkout, branch, ports and DB role, and the renderer
resolves THERMOGRAPH_BAO_APPROLE, then TG_BAO_APPROLE, then the conventional path —
the same precedence render-secrets.sh:44 already uses for the backend selector.

Only beta differs. prod and dev keep the conventional path; dev is alone on vps1 so
there is no collision there.

Beta parity passes when the path is supplied by hand (24 keys = 8 + 16 shared);
this makes it automatic. Derivation confirmed by sourcing env-topology.sh for all
three environments.
---
 infra/deploy/env-topology.sh           | 16 +++++++++++++++-
 infra/deploy/render-secrets-openbao.sh |  8 +++++++-
 infra/openbao/README.md                |  9 ++++-----
 infra/openbao/bootstrap.sh             |  5 ++---
 4 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh
index 9e2b7c9..d57d148 100644
--- a/infra/deploy/env-topology.sh
+++ b/infra/deploy/env-topology.sh
@@ -76,6 +76,16 @@ thermograph_topology() {
   # for the reason that marker was demoted to a fallback in the first place: vps2 runs
   # prod and beta side by side, and one host-wide file cannot name two backends.
   TG_SECRETS_BACKEND=sops
+  # Which AppRole credential file render-secrets-openbao.sh authenticates with. Same
+  # reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO
+  # environments, so a single host-wide default cannot serve both. prod and dev each
+  # get the conventional path (dev is alone on vps1, so there is no collision there);
+  # beta is the one that must differ, because it shares a filesystem with prod.
+  #
+  # Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl
+  # correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that
+  # runs prod.
+  TG_BAO_APPROLE=/etc/thermograph/openbao-approle
 
   case "$env_name" in
     prod)
@@ -116,6 +126,10 @@ thermograph_topology() {
       # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh).
       TG_SSH_HOST=169.58.46.181
       TG_SSH_TARGET=agent@169.58.46.181
+      # The one environment that cannot use the default AppRole path: it shares a
+      # filesystem with prod, so it needs its own credential file to authenticate as
+      # tg-beta rather than tg-prod. bootstrap-policies.sh installs it here.
+      TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta
       # A SECOND checkout on the same box. Separate from prod's so the two can
       # sit on different commits of this repo, and so `git reset --hard` in one
       # deploy can never yank the tree out from under the other's running
@@ -228,7 +242,7 @@ thermograph_topology() {
   export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
   export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
   export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
-  export TG_SECRETS_BACKEND
+  export TG_SECRETS_BACKEND TG_BAO_APPROLE
   export TG_SSH_HOST TG_SSH_TARGET
 }
 
diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh
index 82c449c..aaf606d 100644
--- a/infra/deploy/render-secrets-openbao.sh
+++ b/infra/deploy/render-secrets-openbao.sh
@@ -27,7 +27,13 @@ thermograph_openbao_source() {
   local env_name="${1:?env name required}"
   local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
   local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
-  local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
+  # Explicit override first, then the per-environment value env-topology.sh derives,
+  # then the conventional path. Same precedence shape as render-secrets.sh:44's
+  # THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the
+  # THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the
+  # deploy path sets. Falling straight through to the bare default is what made beta
+  # authenticate as tg-prod and get denied its own path.
+  local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}"
   local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
 
   command -v bao >/dev/null 2>&1 || {
diff --git a/infra/openbao/README.md b/infra/openbao/README.md
index 037bfcb..855e4d7 100644
--- a/infra/openbao/README.md
+++ b/infra/openbao/README.md
@@ -185,15 +185,14 @@ its own host:
 ```sh
 # on vps2
 infra/openbao/verify-parity.sh --env prod                          # expect 32 keys
-THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \
-  infra/openbao/verify-parity.sh --env beta                        # expect 24 keys
+infra/openbao/verify-parity.sh --env beta                          # expect 24 keys
 # on vps1
 cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev  # expect 12 keys
 ```
 
-The explicit `THERMOGRAPH_BAO_APPROLE` for beta is a stopgap: `env-topology.sh` does
-not yet derive the AppRole path per environment, so the renderer falls back to prod's
-credentials and `tg-host-prod.hcl` correctly denies `env/beta`.
+Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per
+environment, so beta authenticates as `tg-beta` rather than falling back to prod's
+credentials and being denied its own path by `tg-host-prod.hcl`.
 
 `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
 the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh
index f3d0f69..6fa199d 100755
--- a/infra/openbao/bootstrap.sh
+++ b/infra/openbao/bootstrap.sh
@@ -213,10 +213,9 @@ secrets is not for an agent to run:
 Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
 that legitimately renders it, so an environment can only be verified from its own host.
 
-  # here (vps2) -- beta needs its own AppRole, prod uses the default
+  # here (vps2)
   infra/openbao/verify-parity.sh --env prod                        # expect 32 keys
-  THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \\
-    infra/openbao/verify-parity.sh --env beta                      # expect 24 keys
+  infra/openbao/verify-parity.sh --env beta                        # expect 24 keys
   # on vps1
   cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev   # expect 12 keys
 

From 657e27ba9f19b9c8e18f3a85677995b56df5d307 Mon Sep 17 00:00:00 2001
From: Emi Griffith <me@emigriffith.dev>
Date: Thu, 30 Jul 2026 19:55:17 -0700
Subject: [PATCH 3/5] openbao: render Centralis from the vault
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Centralis could not migrate at all. `seed-from-sops.sh --all` refused
centralis.prod with `CENTRALIS_TOKENS: contains a shell metacharacter`, and
render-secrets-openbao.sh had no Centralis path whatsoever — so the seeded data
would have had nothing to read it.

Both halves come from applying the app stack's rules to a file that does not share
its shape. /etc/thermograph.env is raw unquoted KEY=value. /etc/centralis.env is
consumed by exactly one thing and that thing is a shell:

    sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'

CENTRALIS_TOKENS is JSON, so it can never satisfy a no-metacharacters rule, and
rendering it unquoted is precisely the 2026-07-24 failure: bash strips the quotes,
Centralis fails closed on the malformed registry and serves a single `shared`
identity, indistinguishable from the file never being written.

So:

- seed-from-sops.sh validates against the renderer that will consume the path. The
  dotenv rules still apply to common and env/*; centralis/* is exempt because it is
  single-quoted downstream.
- render-secrets-openbao.sh gains thermograph_render_openbao_centralis, mirroring
  render_centralis_secrets: fetch as JSON, emit POSIX single-quoted assignments,
  then PROVE the file by sourcing it under `env -i` with `set -a` and comparing
  every value back before anything touches the destination. Mirrored rather than
  shared, per the existing note above thermograph_render_openbao — consolidate when
  the SOPS path is deleted.
- render_centralis_secrets gains the same early-return backend dispatch the app
  path already uses, so the new function is reachable. Without it the SOPS path was
  the only path regardless of TG_SECRETS_BACKEND.

Verified against a synthetic registry containing a JSON token map, an apostrophe,
and `$(...)`/backtick/backslash/double-quote payloads: 5 keys round-trip
byte-for-byte through `set -a; . file`, CENTRALIS_TOKENS parses to the right
subjects, and the command substitution does not execute. The negative case is
covered too — fed the unquoted 2026-07-24 shape, the round-trip check reports the
value as changed (by length, never content) and refuses to write.

Also corrects the claim in render-secrets-openbao.sh that the SOPS path never
tripped the metacharacter hazard "because every current value happens to be
alphanumeric". CENTRALIS_TOKENS is the counter-example; it never tripped those
paths because it was never in them.
---
 infra/deploy/render-secrets-openbao.sh | 260 ++++++++++++++++++++++++-
 infra/deploy/render-secrets.sh         |  24 +++
 infra/openbao/README.md                |  16 ++
 infra/openbao/seed-from-sops.sh        |  28 ++-
 4 files changed, 315 insertions(+), 13 deletions(-)

diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh
index aaf606d..518f999 100644
--- a/infra/deploy/render-secrets-openbao.sh
+++ b/infra/deploy/render-secrets-openbao.sh
@@ -1,9 +1,14 @@
 #!/usr/bin/env bash
 # OpenBao source backend for render-secrets.sh. Sourced, never run directly.
 #
-# Two functions:
-#   thermograph_openbao_source <env>        -> merged dotenv on STDOUT
-#   thermograph_render_openbao  <env> <out> -> source, then write <out>
+# Functions:
+#   thermograph_openbao_source <env>                 -> merged dotenv on STDOUT
+#   thermograph_render_openbao  <env> <out>          -> source, then write <out>
+#   thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
+#
+# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
+# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
+# a different shape for a different consumer, not an inconsistency.
 #
 # render_thermograph_secrets in render-secrets.sh dispatches to the second when
 # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
@@ -113,9 +118,16 @@ thermograph_openbao_source() {
   #     /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
   #     deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
   #     `$(...)` or a backtick in a secret is a live command-execution path on the
-  #     deploy host. The SOPS path has always had this hazard and has never tripped it
-  #     only because every current value happens to be alphanumeric. Refusing here
-  #     turns a latent code-execution bug into a loud render failure.
+  #     deploy host. Refusing here turns a latent code-execution bug into a loud
+  #     render failure.
+  #
+  #     An earlier version of this comment claimed the SOPS path had never tripped
+  #     this "only because every current value happens to be alphanumeric". That is
+  #     false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
+  #     tripped THESE paths because it is not in them — it lives at
+  #     centralis/<env> and renders to /etc/centralis.env, which is single-quoted
+  #     precisely because raw dotenv cannot carry it. See
+  #     thermograph_render_openbao_centralis at the end of this file.
   THERMOGRAPH_BAO_COMMON="$common_json" \
   THERMOGRAPH_BAO_ENV="$env_json" \
   python3 - <<'PY'
@@ -243,3 +255,239 @@ thermograph_render_openbao() {
   rm -f "$tmp"
   return "$rc"
 }
+
+# ---------------------------------------------------------------------------
+# CENTRALIS
+# ---------------------------------------------------------------------------
+# /etc/centralis.env is a different file, with a different consumer, and therefore a
+# different output shape. render-secrets.sh:206-233 carries the full argument; the
+# short version is that it is consumed by exactly one thing and that thing is a shell:
+#
+#     sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
+#
+# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
+# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
+# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
+# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
+# by serving a single `shared` identity — indistinguishable from the file never having
+# been written. That is the 2026-07-24 incident. So this path emits POSIX
+# single-quoted assignments and PROVES them by sourcing the result in a clean shell
+# before anything touches $dest.
+#
+# This mirrors render_centralis_secrets rather than sharing with it, for the same
+# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
+# and all new risk stays in this file. Consolidate the two when the SOPS path is being
+# deleted anyway, not before.
+
+# _thermograph_openbao_login <approle_file> <addr> <ca>  -> token on STDOUT
+#
+# Separate from the login inline in thermograph_openbao_source deliberately: that
+# function is now exercised by prod and beta parity and is left untouched.
+_thermograph_openbao_login() {
+  local approle="$1" addr="$2" ca="$3"
+  local creds role_id secret_id
+  if [ -r "$approle" ]; then
+    creds=$(cat "$approle")
+  else
+    creds=$(sudo cat "$approle" 2>/dev/null || true)
+  fi
+  [ -n "$creds" ] || {
+    echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
+    return 1
+  }
+  role_id=$(printf '%s\n' "$creds" | sed -n '1p')
+  secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
+  [ -n "$role_id" ] && [ -n "$secret_id" ] || {
+    echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
+    return 1
+  }
+  export BAO_ADDR="$addr"
+  [ -f "$ca" ] && export BAO_CACERT="$ca"
+  bao write -field=token auth/approle/login \
+    role_id="$role_id" secret_id="$secret_id" 2>/dev/null
+}
+
+# thermograph_render_openbao_centralis <env_name> [dest]
+thermograph_render_openbao_centralis() {
+  local env_name="${1:?env name required}"
+  local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
+  local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
+  local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
+  local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
+  local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
+
+  command -v bao >/dev/null 2>&1 || {
+    echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
+    return 1
+  }
+  command -v python3 >/dev/null 2>&1 || {
+    echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
+    return 1
+  }
+
+  echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
+
+  local token
+  token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
+  [ -n "$token" ] || {
+    echo "!! OpenBao AppRole login failed for centralis" >&2
+    echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
+    return 1
+  }
+  export BAO_TOKEN="$token"
+
+  # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
+  # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
+  # persists into the caller's shell and re-fires on its next `source`.
+  local work rc=0
+  work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
+  chmod 700 "$work"
+
+  # Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
+  # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
+  # command that is the left operand of `||`, which this subshell is.
+  (
+    umask 077
+
+    bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
+      > "$work/kv.json" 2>/dev/null || {
+      echo "!! cannot read ${mount}/centralis/${env_name}" >&2
+      exit 1
+    }
+
+    python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
+import json, sys
+doc = json.load(open(sys.argv[1], encoding="utf-8"))
+data = doc.get("data", {}).get("data", {}) or {}
+if not data:
+    sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
+    sys.exit(1)
+json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
+UNWRAP
+
+    python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
+import json, re, sys
+
+src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
+NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
+
+
+def shell_quote(v):
+    """POSIX single-quoting. Inside '...' every byte is literal except ' itself,
+    which is closed, backslash-escaped and reopened. There is no escape sequence
+    to get wrong and no character class to keep up to date: it is total, and
+    total is the only property worth having for a file bash will expand."""
+    return "'" + v.replace("'", "'\\''") + "'"
+
+
+def scalar(k, v):
+    if isinstance(v, str):
+        return v
+    if isinstance(v, bool):          # before int: bool is an int in Python
+        return "true" if v else "false"
+    if isinstance(v, (int, float)):
+        return json.dumps(v)
+    if v is None:
+        return ""
+    sys.stderr.write(
+        "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
+    sys.exit(1)
+
+
+data = json.load(open(src, encoding="utf-8"))
+want, lines = {}, []
+for k, v in data.items():
+    if not NAME.match(k):
+        sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
+        sys.exit(1)
+    want[k] = scalar(k, v)
+    lines.append("%s=%s\n" % (k, shell_quote(want[k])))
+
+with open(out_env, "w", encoding="utf-8") as fh:
+    fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
+    fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
+    fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
+    fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
+    fh.writelines(lines)
+with open(want_json, "w", encoding="utf-8") as fh:
+    json.dump(want, fh)
+sys.stderr.write("    %d keys quoted\n" % len(want))
+EMIT
+
+    # PROVE IT. Source the rendered file exactly the way the deploy does — clean
+    # environment, `set -a`, nothing inherited that could mask a dropped key — and
+    # read every value back. This is the check the 2026-07-24 file could not pass.
+    # shellcheck disable=SC2016  # single quotes are the point: "$1" must be expanded
+    # by the inner `bash -c` against the argument after `_`, not by this shell.
+    env -i PATH="$PATH" bash -c \
+      'set -a; . "$1"; set +a; exec python3 -c "
+import json, os, sys
+sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
+      > "$work/got.json" || exit 1
+
+    python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
+import json, sys
+
+want = json.load(open(sys.argv[1], encoding="utf-8"))
+got = json.load(open(sys.argv[2], encoding="utf-8"))
+
+bad = []
+for k, v in want.items():
+    if k not in got:
+        bad.append("%s: not set at all after sourcing" % k)
+    elif got[k] != v:
+        # NEVER print either value. The difference is the finding; the content is
+        # not, and this runs inside a deploy log.
+        bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
+                   % (k, len(v), len(got[k])))
+if bad:
+    sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
+    for b in bad:
+        sys.stderr.write("!!   %s\n" % b)
+    sys.exit(1)
+
+# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
+# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
+# single `shared` identity, which looks exactly like the registry never having been
+# configured. Refuse to write a file that would do that.
+raw = got.get("CENTRALIS_TOKENS", "")
+if raw:
+    try:
+        reg = json.loads(raw)
+    except ValueError as exc:
+        sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
+        sys.exit(1)
+    if not isinstance(reg, dict) or not reg:
+        sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
+        sys.exit(1)
+    if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
+        sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
+        sys.exit(1)
+    # Subjects are names on audit lines, not credentials. Printing them is the whole
+    # point: it is how an operator sees at a glance that nobody was lost.
+    sys.stderr.write("    CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
+                     % (len(reg), ", ".join(sorted(reg))))
+
+sys.stderr.write("    %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
+VERIFY
+  ) || rc=1
+
+  # Only now, with a file read back through bash and matched value for value, does
+  # anything touch $dest. 0600: four bearer credentials for the estate's control
+  # plane, with no group that needs them.
+  if [ "$rc" -eq 0 ]; then
+    if [ -f "$dest" ] && [ -w "$dest" ]; then
+      cat "$work/out.env" > "$dest" || rc=1
+    elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
+    elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
+    else
+      echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
+      rc=1
+    fi
+  else
+    echo "!! render aborted; ${dest} left untouched" >&2
+  fi
+
+  rm -rf "$work"
+  return "$rc"
+}
diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh
index 32093d9..ea7de00 100755
--- a/infra/deploy/render-secrets.sh
+++ b/infra/deploy/render-secrets.sh
@@ -272,6 +272,30 @@ render_centralis_secrets() {
     echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
     return 1
   fi
+
+  # Same early-return dispatch as render_thermograph_secrets, for the same reason
+  # given there: everything below — the age-key sudo lift, the JSON decrypt, the
+  # quote-and-prove pipeline, the three-way write — stays on exactly the code path it
+  # has always been on, so this cannot regress the backend still authoritative for
+  # Centralis. The OpenBao path needs no age key, so it returns before that check.
+  local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
+  if [ "$backend" = openbao ]; then
+    local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
+    if [ ! -f "$bao_lib" ]; then
+      echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
+      echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
+      return 1
+    fi
+    # shellcheck source=/dev/null
+    . "$bao_lib"
+    thermograph_render_openbao_centralis "$env_name" "$dest"
+    return
+  fi
+  if [ "$backend" != sops ]; then
+    echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
+    return 1
+  fi
+
   if [ ! -f "$key" ]; then
     echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
     return 1
diff --git a/infra/openbao/README.md b/infra/openbao/README.md
index 855e4d7..14c6ac8 100644
--- a/infra/openbao/README.md
+++ b/infra/openbao/README.md
@@ -112,6 +112,22 @@ thermograph/data/legacy/age      the age PRIVATE key — see "the age key surviv
 Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
 isolation lives in the policy. Today both live in one shell variable.
 
+**`centralis/<env>` renders differently from everything above it, and must.** The
+app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
+is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
+pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
+sourcing it in a clean shell and comparing every value back before the file is
+written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
+strips its quotes, Centralis fails closed on the malformed result and serves a
+single `shared` identity, which is indistinguishable from the file never having
+been written. That is the 2026-07-24 incident, and the round-trip check exists so a
+render carrying it cannot reach `/etc`.
+
+Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
+metacharacters, no newlines) only to `common` and `env/*`. Applying them to
+`centralis/*` would reject data that renders perfectly well — the validator matches
+the renderer that will consume the path, not a single global rule.
+
 `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
 rule expressible as exactly one path, with no wildcard over `env/*` able to
 accidentally re-include it.
diff --git a/infra/openbao/seed-from-sops.sh b/infra/openbao/seed-from-sops.sh
index 09f32cc..3988ca7 100755
--- a/infra/openbao/seed-from-sops.sh
+++ b/infra/openbao/seed-from-sops.sh
@@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do
   # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
   # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
   # SEED time — before anything depends on it — rather than at deploy time.
-  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
+  # Which renderer will consume this path decides which values are legal, so the
+  # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
+  # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
+  # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
+  # render_centralis_secrets and its OpenBao counterpart, where every byte inside
+  # '...' is literal — so applying the dotenv rules to it rejects data that renders
+  # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
+  quoted=0
+  case "$target" in centralis.*) quoted=1 ;; esac
+
+  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
 import json, os, re, sys
 
 doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
@@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc:
     sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
     sys.exit(1)
 
+# 1 => destined for a single-quoted env file; 0 => raw dotenv.
+QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
+
 KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 UNSAFE = set("`$\\\"'")
 
@@ -141,12 +154,13 @@ for key, val in doc.items():
         val = ""
     if not isinstance(val, str):
         val = json.dumps(val, separators=(",", ":"))
-    if "\n" in val or "\r" in val:
-        bad.append(f"{key}: contains a newline")
-        continue
-    if UNSAFE & set(val):
-        bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
-        continue
+    if not QUOTED:
+        if "\n" in val or "\r" in val:
+            bad.append(f"{key}: contains a newline")
+            continue
+        if UNSAFE & set(val):
+            bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
+            continue
     out[key] = val
 
 if bad:

From 11f1275998254d445ba2935cf4be269461f2e24a Mon Sep 17 00:00:00 2001
From: Emi Griffith <me@emigriffith.dev>
Date: Thu, 30 Jul 2026 21:24:18 -0700
Subject: [PATCH 4/5] rotate Contabo S3 credentials

---
 infra/deploy/secrets/common.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/infra/deploy/secrets/common.yaml b/infra/deploy/secrets/common.yaml
index 83ec094..247789d 100644
--- a/infra/deploy/secrets/common.yaml
+++ b/infra/deploy/secrets/common.yaml
@@ -4,13 +4,13 @@ THERMOGRAPH_BASE: ENC[AES256_GCM,data:sQ==,iv:Pop6S2qU0o2+2xMKWQD2P3Vjdh4rUafWF7
 THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:CA==,iv:TM6XiygrDQn2qps4lO6hY2ldjW8LWlO3D/R/DBPqvGQ=,tag:+thBT7fMzH+DigecgxF4CQ==,type:str]
 THERMOGRAPH_INDEXNOW_KEY: ENC[AES256_GCM,data:XZEMMJfmrSjc/sKEePaigTqEkh3ob/E25R25DvOAWwY=,iv:fqdaegzG1Na0zW+UgOh92RS4FP22pxStdD1/Jq5DRjs=,tag:/8ORtVkGUWjqgGXG/xZHmg==,type:str]
 THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:5pL+8tZbs8TZ5ezJpkPUmMss0gLcte73lU+au/476Ws=,iv:ujU3G+rbPZQx/igg50GeIPQxqgd+szIsjc3AqGhGLBU=,tag:hrkgn2DPkmBGThlsdMplqg==,type:str]
-THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:XG+6wizXMVVFixhqLhg72HoKw9kEmH+Tty8jYMycvVk=,iv:tNz/WrkMYip2QyAr6bRy6PaIee56ZZ64BELBgvidVT0=,tag:HiQ55DGbeDAOSF0vpNWpWw==,type:str]
+THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:Ui5dcmhzz84nTp1gYhJ89VOJ2eOBFyLjf0WkZ7Y3sI0=,iv:WI0QqbUW7BHo7Zz0082zxpYNIYpVIhq1NeBGbQeDc/I=,tag:urNsE8DPaldJcw9aqX1PYA==,type:str]
 THERMOGRAPH_METRICS_TOKEN: ENC[AES256_GCM,data:DmOZU3HWQTuoSvQMb7iPkClJbDH48NB3OcRIYRkqNz8=,iv:acgGh0w2mc5ZD7rB7m/GW3Awsx3RWK+02L6YkAv5Rw0=,tag:x3U9F0Dekw/r9euRMjVQAA==,type:str]
 THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:hTSSUIfOlY0GYI8gZ7FcOi7c89iye0Ym73l4M22zkNk=,iv:aPdbXKb4pOqigo9J3a7oM3qTip6HQcHUQDlAwg6XfoE=,tag:C4p+Ob/iSElNdF4m9E+M7A==,type:str]
 THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:vGJDaElN6JRUawlD55Vr5Q==,iv:4ZAQ5NXb9xuMN+qPspM17W6zm9NIDXLX9hTJ/etSL0E=,tag:Ss7xaUl+kOiVF7oDs9FiYw==,type:str]
 THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:PtZg52Uu70Ndx7L/dRTCZTmqv80NXiohOg+gfrzG,iv:hKQTy3Twd5uWwnS4UIs9oqT4NzpYUWQxWO1lUqrzpAc=,tag:kYwYp58fxkzRe5yQPSXa3g==,type:str]
-THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:1xuZRaC4p+ZM/oFAUuGDZb6AAu2PbjFrHQY3zmczRpI=,iv:xzI24qlUOvYiThBLOue0odvopuxdHCwDlwI4JwiP9EU=,tag:yRS9GjGLBEYk/kZ6miFudg==,type:str]
-THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:N9mPLx6Mcgqm5TlgqIMFhNuZsBZxVPXf3LplO9k+VA==,iv:y5+MLI0zC5Kb6pRdORTMVJ1iMMoFrVRfv99mKWMRjp8=,tag:7VvKCLU+1TJtQcHWlDwybg==,type:str]
+THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:4u5d3RHHBeIDfRIwQVDldWUhH7PyTcNP9s3xTrL4N74=,iv:RXWu+4ZdTmWx3PGw6nWhN2587byy4wqjY74Ldu4xvuk=,tag:dwS6htP2Dv2OQyNzmjZZIA==,type:str]
+THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:hg5HqBua81dG3d4KaxLxljRkeGnt4h4=,iv:q+4tQGrlsQaRGUTJuqByqss+twR6v6NQrI/qSPjCbsQ=,tag:2D1+hv/KSDJu+4+J9S0hww==,type:str]
 THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:hTxKlgPWeW4HGBf08SxdAdK+ce6T5Fl9f+5tSGV6WpS+za5EI3zsK8BgRw==,iv:l+omFaCyL2+PHWdchadkmED2gIAoj5T0u/Ltzaw8mok=,tag:AQ3wT/wD5JAouX1M+Tjjmw==,type:str]
 THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:bgtqZFfTzYz5llh/YhAyGwGtRedK3/ze8SAWj8YdldY6s4nt2F3XeTANVO2Y9+NzvXc16PB3WWPnYEzCxnrG2KciIeYpOYZMoPpGldENPBLJtLa76Ej9,iv:HuZ7XtlXIt4wyJ79k3IYjHfWrQp7lnak4uCspyzS4BE=,tag:uPZvba8LVugDy76pKat6TA==,type:str]
 TIMESCALEDB_TAG: ENC[AES256_GCM,data:3mDU/k5fx0894+E=,iv:/iw3BZPF3mZ9M3bFNJSTzP9t15YZWCJwzoVXYL9Vj9o=,tag:Y4ez3+XfLJN1PtJyZlJj+g==,type:str]
@@ -25,7 +25,7 @@ sops:
             bncE6yn10QK5kVcF9dDvpQ6RbaG+ESpNZTnuhSL5PDqrKtjnsV0Iqw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
-    lastmodified: "2026-07-25T00:38:24Z"
-    mac: ENC[AES256_GCM,data:C12PnFE7MO0Ge5dCAHCcyXh650hFtRuexl5d3DL0IqTf8nLyNtRJzrPyl9whIs+S3HulsXh3xJOwwPdg6aTqdCfGSQ4Kp5qV+OHxH9lfpHs2yH+exa/eX+7Khpjk0OfvX5beC0GSPpYf9c01buaUTWcIh3pJXpDMdqr+aLC+Zd0=,iv:6VeaaEEkWU5rdTHy75rc2emb2u/HmOmLzO11D38ZXHc=,tag:iGRevzdtqZF9kyH0wxqG3w==,type:str]
+    lastmodified: "2026-07-31T04:23:12Z"
+    mac: ENC[AES256_GCM,data:KkItyA0iw2j4w9pf/efFcpyOP8RmO8XzfMoQhJcRXLZbrUC3/xltbxkfTIe2hRgAqKyz5d7EuAHBPwwSCubue0O2gzlG0nCnRQH7YyO4cUrP24XYb+aI+tBM2jdZIuZrGDNcq04cwKiecC6MyOOp8uQPZ41AOPH3sONGWj1VH2s=,iv:+j5uv4QbuWpXAyElvbYGKWhe9vmGTfs3Hpy5I5FetJo=,tag:wcfPuugefeQAyxdb2HV1Fg==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

From b2077b52946a58163895d90e00d77f2e27597cdd Mon Sep 17 00:00:00 2001
From: Emi Griffith <me@emigriffith.dev>
Date: Thu, 30 Jul 2026 20:26:37 -0700
Subject: [PATCH 5/5] secrets: make the two-host age key quorum a checked
 policy
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The copies of /etc/thermograph/age.key on vps1 and vps2 have always read as
provisioning residue — a side effect of provision-secrets.sh rather than a
decision. They are in fact the estate's recovery quorum: that key is the single
recovery root for all five SOPS vaults AND for every off-box backup, since
ops-cron.yml:142,197 pipe those dumps through `age -r` to the same recipient.

On 2026-07-30 the operator's copy was shredded from the desktop before it had been
copied to its replacement machine, leaving zero operator-side copies. These two were
the only surviving material and the key was restored from vps2. Documenting the
ordering rule that would have prevented it, and adding the check that makes the
quorum verifiable rather than assumed.

verify-age-quorum.sh asserts presence, permissions and hash equality across both
hosts. Divergence is treated as seriously as absence: two hosts holding different
keys means one renders secrets the other cannot read, and a restore silently picks
the wrong one. Prints SHA-256 digests and modes only — a digest of a 32-byte random
key is not reversible, so it is safe in a CI log, and the key is never read or
copied anywhere. Exit status makes it usable as a cron gate.

Also replaces the "wasn't re-verified for this pass" hedge in the README with what
was actually measured: vps1 0440 root:deploy, vps2 0400 root:root, digests equal.
The dev render does not need vps1's group bit — it runs as `agent`, which is not in
group `deploy` and reaches the key through passwordless sudo, with
/opt/thermograph-dev owned agent:agent. Tightening vps1 to 0400 root:root is
therefore expected to be safe, and is called out as the follow-up rather than done
here because it changes a live host on the deploy path.

The check reports that group read as an advisory rather than a failure. It is the
current provisioned state, and a check that fails on day one is a check that is
ignored by day three.
---
 infra/deploy/secrets/README.md            |  49 +++++++--
 infra/deploy/secrets/verify-age-quorum.sh | 117 ++++++++++++++++++++++
 2 files changed, 155 insertions(+), 11 deletions(-)
 create mode 100755 infra/deploy/secrets/verify-age-quorum.sh

diff --git a/infra/deploy/secrets/README.md b/infra/deploy/secrets/README.md
index 4b85e59..7ec4bb3 100644
--- a/infra/deploy/secrets/README.md
+++ b/infra/deploy/secrets/README.md
@@ -251,17 +251,44 @@ files), so layering would only push the VAPID private key, both S3 keypairs and
   - `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time.
 - **Back it up** in the password manager. The public recipient is in `.sops.yaml`.
 
-On **vps1** (dev) those two can end up being the same file. `render-secrets.sh`
-falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes
-the account driving a CI-triggered dev deploy has no passwordless sudo and no
-tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) —
-inherited from the old desktop setup, where that was true of the whole box.
-Whether it's still true of vps1's CI-runner account specifically (as opposed
-to the `agent` login, which does have passwordless sudo there per `ACCESS.md`)
-wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY`
-at the operator's keyring as a fallback either way, so the box keeps **one**
-copy of the recovery root rather than two regardless of which account ends up
-mattering.
+### The two on-host copies are a deliberate quorum
+
+`/etc/thermograph/age.key` on **vps1** and **vps2** is not provisioning residue and
+must not be tidied away. Together they are the recovery quorum for the entire
+estate: five SOPS vaults, and every off-box backup, since `ops-cron.yml:142,197`
+pipe those dumps through `age -r` to this same recipient.
+
+That is not theoretical. On **2026-07-30** the operator's copy was shredded from the
+desktop *before* it had been copied to its replacement machine, leaving zero
+operator-side copies. These two were the only surviving material, and the key was
+restored from vps2. The ordering rule below exists because of that hour.
+
+- **Never remove the last operator-side copy** until its replacement has been
+  verified against all five vaults (`sops -d … | grep -c '='` → 16 / 12 / 8 / 16 / 9).
+  Deleting first and verifying second is unrecoverable if the copy is bad.
+- **Both hosts must hold byte-identical copies.** Divergence is as bad as absence:
+  two different keys means one host renders secrets the other cannot read, and a
+  restore silently picks the wrong one.
+- **Verify it, don't assume it** — `deploy/secrets/verify-age-quorum.sh` asserts
+  presence, permissions and hash equality across both hosts. It prints SHA-256
+  digests and modes only, never the key, so it is safe in a CI log and suitable as
+  a cron gate. Deletion and divergence are both silent failures; nothing else in the
+  estate would notice either until a restore.
+
+Verified 2026-07-30: vps1 `0440 root:deploy`, vps2 `0400 root:root`, digests equal.
+Note the asymmetry — on vps1 the group `deploy` can read the key that decrypts
+`common.yaml` and `prod.yaml`, on the box that also runs Forgejo, its CI runner and
+dev's unreviewed branches. The dev render does not need that group bit: it runs as
+`agent`, which is not in group `deploy` and reaches the key through its passwordless
+`sudo` instead (`/opt/thermograph-dev` is `agent:agent`). Tightening vps1 to `0400
+root:root` is therefore expected to be safe and is the obvious follow-up; it is
+called out rather than done here because it changes a live host on the deploy path.
+
+This weakens — in practice, not in intent — the rule in `infra/CLAUDE.md` that "vps1
+must never hold prod credentials". Withholding `common.yaml` from dev's *render* does
+not withhold the key that decrypts it from the *box*, because every vault is
+encrypted to a single recipient. Fixing that properly means a second age identity for
+dev, not a policy line.
 
 ## Prerequisites (once per machine)
 
diff --git a/infra/deploy/secrets/verify-age-quorum.sh b/infra/deploy/secrets/verify-age-quorum.sh
new file mode 100755
index 0000000..8bb36ba
--- /dev/null
+++ b/infra/deploy/secrets/verify-age-quorum.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Assert that the age recovery quorum is intact.
+#
+#   infra/deploy/secrets/verify-age-quorum.sh
+#
+# The age private key is the single recovery root for all five SOPS vaults AND for
+# every off-box backup (ops-cron.yml:142,197 pipe dumps through `age -r` to the same
+# recipient). The copies at /etc/thermograph/age.key on vps1 and vps2 are a DELIBERATE
+# two-host quorum, not provisioning residue — see the "age key" section of
+# deploy/secrets/README.md. This script is what makes that policy checkable instead of
+# merely stated.
+#
+# It fails if the key is missing on either host, if the two copies have diverged, or
+# if either is more permissive than 0440. Divergence matters as much as absence: two
+# hosts holding different keys means one of them renders secrets nobody else can
+# decrypt, and a restore from backup silently picks the wrong one.
+#
+# OUTPUT DISCIPLINE: prints SHA-256 digests and file modes only. A digest of a
+# 32-byte random key is not reversible, so it is safe in a CI log; the key itself is
+# never read, echoed or transferred. Nothing here copies the key anywhere.
+#
+# Exit status: 0 = quorum intact; 1 = something needs a human. Suitable as a cron gate.
+set -euo pipefail
+
+VPS1="${TG_VPS1_SSH:-vps1}"
+VPS2="${TG_VPS2_SSH:-vps2}"
+KEY_PATH="${TG_AGE_KEY_PATH:-/etc/thermograph/age.key}"
+SSH_OPTS="${TG_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=10}"
+
+# Reads the digest and mode of the key on one host. Uses sudo when the key is not
+# directly readable, mirroring how render-secrets.sh lifts it.
+probe() {
+  local target="$1" out
+  # The remote script is a QUOTED heredoc and the key path is passed as $1 to
+  # `bash -s`, so nothing expands on this side. Interpolating the path into the
+  # command string would work too, but it is the shape that quietly breaks the day
+  # a path contains a space, and shellcheck is right to flag it (SC2029).
+  # shellcheck disable=SC2086  # SSH_OPTS is a deliberate word-split option list
+  out=$(ssh $SSH_OPTS "$target" bash -s -- "$KEY_PATH" 2>/dev/null <<'REMOTE'
+key="$1"
+if [ ! -e "$key" ]; then echo MISSING; exit 0; fi
+mode=$(stat -c %a "$key")
+owner=$(stat -c %U:%G "$key")
+if [ -r "$key" ]; then
+  digest=$(sha256sum "$key" | cut -d' ' -f1)
+else
+  digest=$(sudo sha256sum "$key" 2>/dev/null | cut -d' ' -f1)
+fi
+[ -n "$digest" ] || { echo UNREADABLE; exit 0; }
+echo "$digest $mode $owner"
+REMOTE
+  ) || { echo UNREACHABLE; return 0; }
+  printf '%s\n' "$out"
+}
+
+rc=0
+declare -A DIGEST
+
+for pair in "vps1:$VPS1" "vps2:$VPS2"; do
+  name="${pair%%:*}"
+  target="${pair#*:}"
+  result="$(probe "$target")"
+  case "$result" in
+    MISSING)
+      echo "!! ${name}: no key at ${KEY_PATH} — the quorum is DOWN TO ONE COPY" >&2
+      rc=1
+      ;;
+    UNREADABLE)
+      echo "!! ${name}: key present but unreadable even via sudo" >&2
+      rc=1
+      ;;
+    UNREACHABLE)
+      echo "!! ${name}: host unreachable — quorum NOT verified (this is not a pass)" >&2
+      rc=1
+      ;;
+    *)
+      digest="${result%% *}"
+      rest="${result#* }"
+      DIGEST["$name"]="$digest"
+      printf '   %-5s %s  mode=%s\n' "$name" "${digest:0:16}…" "$rest"
+      mode="${rest%% *}"
+      # 0400 or 0440 only. Anything wider means a non-root account on that box can
+      # read the key that decrypts prod — and vps1 also runs Forgejo and its CI runner.
+      case "$mode" in
+        400|440) ;;
+        *) echo "!! ${name}: mode ${mode} is wider than 0440" >&2; rc=1 ;;
+      esac
+      # Advisory, not a failure: 0440 with a non-root group means that group can read
+      # the key that decrypts prod. On vps1 that is doubly worth knowing, because the
+      # same box runs Forgejo, its CI runner and dev's unreviewed branches. Left
+      # non-fatal because it is the current provisioned state — a check that fails on
+      # day one is a check that gets ignored by day three. See README.md.
+      group="${rest#* }"; group="${group#*:}"
+      if [ "$mode" = 440 ] && [ "$group" != root ]; then
+        printf '         note: group %s can read this key\n' "$group"
+      fi
+      ;;
+  esac
+done
+
+if [ -n "${DIGEST[vps1]:-}" ] && [ -n "${DIGEST[vps2]:-}" ]; then
+  if [ "${DIGEST[vps1]}" = "${DIGEST[vps2]}" ]; then
+    echo "   quorum: 2 copies, identical"
+  else
+    echo "!! the two copies have DIVERGED — they are different keys" >&2
+    echo "!! do not 'fix' this by overwriting one. Establish which decrypts the" >&2
+    echo "!! vaults (sops -d on any deploy/secrets/*.yaml) before touching either." >&2
+    rc=1
+  fi
+fi
+
+if [ "$rc" -eq 0 ]; then
+  echo "   OK — recovery quorum intact"
+else
+  echo "!! recovery quorum degraded; see deploy/secrets/README.md" >&2
+fi
+exit "$rc"