Merge pull request 'promote: dev → main' (#139) from dev into main

infra/deploy/env-topology.sh

@@ -76,6 +76,16 @@ thermograph_topology() {
   # for the reason that marker was demoted to a fallback in the first place: vps2 runs
   # prod and beta side by side, and one host-wide file cannot name two backends.
   TG_SECRETS_BACKEND=sops
+  # Which AppRole credential file render-secrets-openbao.sh authenticates with. Same
+  # reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO
+  # environments, so a single host-wide default cannot serve both. prod and dev each
+  # get the conventional path (dev is alone on vps1, so there is no collision there);
+  # beta is the one that must differ, because it shares a filesystem with prod.
+  #
+  # Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl
+  # correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that
+  # runs prod.
+  TG_BAO_APPROLE=/etc/thermograph/openbao-approle
 
   case "$env_name" in
     prod)
@@ -116,6 +126,10 @@ thermograph_topology() {
       # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh).
       TG_SSH_HOST=169.58.46.181
       TG_SSH_TARGET=agent@169.58.46.181
+      # The one environment that cannot use the default AppRole path: it shares a
+      # filesystem with prod, so it needs its own credential file to authenticate as
+      # tg-beta rather than tg-prod. bootstrap-policies.sh installs it here.
+      TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta
       # A SECOND checkout on the same box. Separate from prod's so the two can
       # sit on different commits of this repo, and so `git reset --hard` in one
       # deploy can never yank the tree out from under the other's running
@@ -228,7 +242,7 @@ thermograph_topology() {
   export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
   export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
   export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
-  export TG_SECRETS_BACKEND
+  export TG_SECRETS_BACKEND TG_BAO_APPROLE
   export TG_SSH_HOST TG_SSH_TARGET
 }
 

infra/deploy/render-secrets-openbao.sh

@@ -1,9 +1,14 @@
 #!/usr/bin/env bash
 # OpenBao source backend for render-secrets.sh. Sourced, never run directly.
 #
-# Two functions:
+# Functions:
-#   thermograph_openbao_source <env>        -> merged dotenv on STDOUT
+#   thermograph_openbao_source <env>                 -> merged dotenv on STDOUT
-#   thermograph_render_openbao  <env> <out> -> source, then write <out>
+#   thermograph_render_openbao  <env> <out>          -> source, then write <out>
+#   thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
+#
+# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
+# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
+# a different shape for a different consumer, not an inconsistency.
 #
 # render_thermograph_secrets in render-secrets.sh dispatches to the second when
 # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
@@ -27,7 +32,13 @@ thermograph_openbao_source() {
   local env_name="${1:?env name required}"
   local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
   local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
-  local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
+  # Explicit override first, then the per-environment value env-topology.sh derives,
+  # then the conventional path. Same precedence shape as render-secrets.sh:44's
+  # THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the
+  # THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the
+  # deploy path sets. Falling straight through to the bare default is what made beta
+  # authenticate as tg-prod and get denied its own path.
+  local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}"
   local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
 
   command -v bao >/dev/null 2>&1 || {
@@ -107,9 +118,16 @@ thermograph_openbao_source() {
   #     /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
   #     deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
   #     `$(...)` or a backtick in a secret is a live command-execution path on the
-  #     deploy host. The SOPS path has always had this hazard and has never tripped it
+  #     deploy host. Refusing here turns a latent code-execution bug into a loud
-  #     only because every current value happens to be alphanumeric. Refusing here
+  #     render failure.
-  #     turns a latent code-execution bug into a loud render failure.
+  #
+  #     An earlier version of this comment claimed the SOPS path had never tripped
+  #     this "only because every current value happens to be alphanumeric". That is
+  #     false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
+  #     tripped THESE paths because it is not in them — it lives at
+  #     centralis/<env> and renders to /etc/centralis.env, which is single-quoted
+  #     precisely because raw dotenv cannot carry it. See
+  #     thermograph_render_openbao_centralis at the end of this file.
   THERMOGRAPH_BAO_COMMON="$common_json" \
   THERMOGRAPH_BAO_ENV="$env_json" \
   python3 - <<'PY'
@@ -237,3 +255,239 @@ thermograph_render_openbao() {
   rm -f "$tmp"
   return "$rc"
 }
+
+# ---------------------------------------------------------------------------
+# CENTRALIS
+# ---------------------------------------------------------------------------
+# /etc/centralis.env is a different file, with a different consumer, and therefore a
+# different output shape. render-secrets.sh:206-233 carries the full argument; the
+# short version is that it is consumed by exactly one thing and that thing is a shell:
+#
+#     sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
+#
+# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
+# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
+# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
+# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
+# by serving a single `shared` identity — indistinguishable from the file never having
+# been written. That is the 2026-07-24 incident. So this path emits POSIX
+# single-quoted assignments and PROVES them by sourcing the result in a clean shell
+# before anything touches $dest.
+#
+# This mirrors render_centralis_secrets rather than sharing with it, for the same
+# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
+# and all new risk stays in this file. Consolidate the two when the SOPS path is being
+# deleted anyway, not before.
+
+# _thermograph_openbao_login <approle_file> <addr> <ca>  -> token on STDOUT
+#
+# Separate from the login inline in thermograph_openbao_source deliberately: that
+# function is now exercised by prod and beta parity and is left untouched.
+_thermograph_openbao_login() {
+  local approle="$1" addr="$2" ca="$3"
+  local creds role_id secret_id
+  if [ -r "$approle" ]; then
+    creds=$(cat "$approle")
+  else
+    creds=$(sudo cat "$approle" 2>/dev/null || true)
+  fi
+  [ -n "$creds" ] || {
+    echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
+    return 1
+  }
+  role_id=$(printf '%s\n' "$creds" | sed -n '1p')
+  secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
+  [ -n "$role_id" ] && [ -n "$secret_id" ] || {
+    echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
+    return 1
+  }
+  export BAO_ADDR="$addr"
+  [ -f "$ca" ] && export BAO_CACERT="$ca"
+  bao write -field=token auth/approle/login \
+    role_id="$role_id" secret_id="$secret_id" 2>/dev/null
+}
+
+# thermograph_render_openbao_centralis <env_name> [dest]
+thermograph_render_openbao_centralis() {
+  local env_name="${1:?env name required}"
+  local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
+  local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
+  local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
+  local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
+  local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
+
+  command -v bao >/dev/null 2>&1 || {
+    echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
+    return 1
+  }
+  command -v python3 >/dev/null 2>&1 || {
+    echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
+    return 1
+  }
+
+  echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
+
+  local token
+  token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
+  [ -n "$token" ] || {
+    echo "!! OpenBao AppRole login failed for centralis" >&2
+    echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
+    return 1
+  }
+  export BAO_TOKEN="$token"
+
+  # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
+  # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
+  # persists into the caller's shell and re-fires on its next `source`.
+  local work rc=0
+  work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
+  chmod 700 "$work"
+
+  # Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
+  # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
+  # command that is the left operand of `||`, which this subshell is.
+  (
+    umask 077
+
+    bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
+      > "$work/kv.json" 2>/dev/null || {
+      echo "!! cannot read ${mount}/centralis/${env_name}" >&2
+      exit 1
+    }
+
+    python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
+import json, sys
+doc = json.load(open(sys.argv[1], encoding="utf-8"))
+data = doc.get("data", {}).get("data", {}) or {}
+if not data:
+    sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
+    sys.exit(1)
+json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
+UNWRAP
+
+    python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
+import json, re, sys
+
+src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
+NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
+
+
+def shell_quote(v):
+    """POSIX single-quoting. Inside '...' every byte is literal except ' itself,
+    which is closed, backslash-escaped and reopened. There is no escape sequence
+    to get wrong and no character class to keep up to date: it is total, and
+    total is the only property worth having for a file bash will expand."""
+    return "'" + v.replace("'", "'\\''") + "'"
+
+
+def scalar(k, v):
+    if isinstance(v, str):
+        return v
+    if isinstance(v, bool):          # before int: bool is an int in Python
+        return "true" if v else "false"
+    if isinstance(v, (int, float)):
+        return json.dumps(v)
+    if v is None:
+        return ""
+    sys.stderr.write(
+        "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
+    sys.exit(1)
+
+
+data = json.load(open(src, encoding="utf-8"))
+want, lines = {}, []
+for k, v in data.items():
+    if not NAME.match(k):
+        sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
+        sys.exit(1)
+    want[k] = scalar(k, v)
+    lines.append("%s=%s\n" % (k, shell_quote(want[k])))
+
+with open(out_env, "w", encoding="utf-8") as fh:
+    fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
+    fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
+    fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
+    fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
+    fh.writelines(lines)
+with open(want_json, "w", encoding="utf-8") as fh:
+    json.dump(want, fh)
+sys.stderr.write("    %d keys quoted\n" % len(want))
+EMIT
+
+    # PROVE IT. Source the rendered file exactly the way the deploy does — clean
+    # environment, `set -a`, nothing inherited that could mask a dropped key — and
+    # read every value back. This is the check the 2026-07-24 file could not pass.
+    # shellcheck disable=SC2016  # single quotes are the point: "$1" must be expanded
+    # by the inner `bash -c` against the argument after `_`, not by this shell.
+    env -i PATH="$PATH" bash -c \
+      'set -a; . "$1"; set +a; exec python3 -c "
+import json, os, sys
+sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
+      > "$work/got.json" || exit 1
+
+    python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
+import json, sys
+
+want = json.load(open(sys.argv[1], encoding="utf-8"))
+got = json.load(open(sys.argv[2], encoding="utf-8"))
+
+bad = []
+for k, v in want.items():
+    if k not in got:
+        bad.append("%s: not set at all after sourcing" % k)
+    elif got[k] != v:
+        # NEVER print either value. The difference is the finding; the content is
+        # not, and this runs inside a deploy log.
+        bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
+                   % (k, len(v), len(got[k])))
+if bad:
+    sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
+    for b in bad:
+        sys.stderr.write("!!   %s\n" % b)
+    sys.exit(1)
+
+# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
+# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
+# single `shared` identity, which looks exactly like the registry never having been
+# configured. Refuse to write a file that would do that.
+raw = got.get("CENTRALIS_TOKENS", "")
+if raw:
+    try:
+        reg = json.loads(raw)
+    except ValueError as exc:
+        sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
+        sys.exit(1)
+    if not isinstance(reg, dict) or not reg:
+        sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
+        sys.exit(1)
+    if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
+        sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
+        sys.exit(1)
+    # Subjects are names on audit lines, not credentials. Printing them is the whole
+    # point: it is how an operator sees at a glance that nobody was lost.
+    sys.stderr.write("    CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
+                     % (len(reg), ", ".join(sorted(reg))))
+
+sys.stderr.write("    %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
+VERIFY
+  ) || rc=1
+
+  # Only now, with a file read back through bash and matched value for value, does
+  # anything touch $dest. 0600: four bearer credentials for the estate's control
+  # plane, with no group that needs them.
+  if [ "$rc" -eq 0 ]; then
+    if [ -f "$dest" ] && [ -w "$dest" ]; then
+      cat "$work/out.env" > "$dest" || rc=1
+    elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
+    elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
+    else
+      echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
+      rc=1
+    fi
+  else
+    echo "!! render aborted; ${dest} left untouched" >&2
+  fi
+
+  rm -rf "$work"
+  return "$rc"
+}

infra/deploy/render-secrets.sh

@@ -272,6 +272,30 @@ render_centralis_secrets() {
     echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
     return 1
   fi
+
+  # Same early-return dispatch as render_thermograph_secrets, for the same reason
+  # given there: everything below — the age-key sudo lift, the JSON decrypt, the
+  # quote-and-prove pipeline, the three-way write — stays on exactly the code path it
+  # has always been on, so this cannot regress the backend still authoritative for
+  # Centralis. The OpenBao path needs no age key, so it returns before that check.
+  local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
+  if [ "$backend" = openbao ]; then
+    local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
+    if [ ! -f "$bao_lib" ]; then
+      echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
+      echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
+      return 1
+    fi
+    # shellcheck source=/dev/null
+    . "$bao_lib"
+    thermograph_render_openbao_centralis "$env_name" "$dest"
+    return
+  fi
+  if [ "$backend" != sops ]; then
+    echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
+    return 1
+  fi
+
   if [ ! -f "$key" ]; then
     echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
     return 1

infra/deploy/secrets/README.md

@@ -251,17 +251,44 @@ files), so layering would only push the VAPID private key, both S3 keypairs and
   - `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time.
 - **Back it up** in the password manager. The public recipient is in `.sops.yaml`.
 
-On **vps1** (dev) those two can end up being the same file. `render-secrets.sh`
+### The two on-host copies are a deliberate quorum
-falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes
+
-the account driving a CI-triggered dev deploy has no passwordless sudo and no
+`/etc/thermograph/age.key` on **vps1** and **vps2** is not provisioning residue and
-tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) —
+must not be tidied away. Together they are the recovery quorum for the entire
-inherited from the old desktop setup, where that was true of the whole box.
+estate: five SOPS vaults, and every off-box backup, since `ops-cron.yml:142,197`
-Whether it's still true of vps1's CI-runner account specifically (as opposed
+pipe those dumps through `age -r` to this same recipient.
-to the `agent` login, which does have passwordless sudo there per `ACCESS.md`)
+
-wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY`
+That is not theoretical. On **2026-07-30** the operator's copy was shredded from the
-at the operator's keyring as a fallback either way, so the box keeps **one**
+desktop *before* it had been copied to its replacement machine, leaving zero
-copy of the recovery root rather than two regardless of which account ends up
+operator-side copies. These two were the only surviving material, and the key was
-mattering.
+restored from vps2. The ordering rule below exists because of that hour.
+
+- **Never remove the last operator-side copy** until its replacement has been
+  verified against all five vaults (`sops -d … | grep -c '='` → 16 / 12 / 8 / 16 / 9).
+  Deleting first and verifying second is unrecoverable if the copy is bad.
+- **Both hosts must hold byte-identical copies.** Divergence is as bad as absence:
+  two different keys means one host renders secrets the other cannot read, and a
+  restore silently picks the wrong one.
+- **Verify it, don't assume it** — `deploy/secrets/verify-age-quorum.sh` asserts
+  presence, permissions and hash equality across both hosts. It prints SHA-256
+  digests and modes only, never the key, so it is safe in a CI log and suitable as
+  a cron gate. Deletion and divergence are both silent failures; nothing else in the
+  estate would notice either until a restore.
+
+Verified 2026-07-30: vps1 `0440 root:deploy`, vps2 `0400 root:root`, digests equal.
+Note the asymmetry — on vps1 the group `deploy` can read the key that decrypts
+`common.yaml` and `prod.yaml`, on the box that also runs Forgejo, its CI runner and
+dev's unreviewed branches. The dev render does not need that group bit: it runs as
+`agent`, which is not in group `deploy` and reaches the key through its passwordless
+`sudo` instead (`/opt/thermograph-dev` is `agent:agent`). Tightening vps1 to `0400
+root:root` is therefore expected to be safe and is the obvious follow-up; it is
+called out rather than done here because it changes a live host on the deploy path.
+
+This weakens — in practice, not in intent — the rule in `infra/CLAUDE.md` that "vps1
+must never hold prod credentials". Withholding `common.yaml` from dev's *render* does
+not withhold the key that decrypts it from the *box*, because every vault is
+encrypted to a single recipient. Fixing that properly means a second age identity for
+dev, not a policy line.
 
 ## Prerequisites (once per machine)
 

infra/deploy/secrets/common.yaml

@@ -4,13 +4,13 @@ THERMOGRAPH_BASE: ENC[AES256_GCM,data:sQ==,iv:Pop6S2qU0o2+2xMKWQD2P3Vjdh4rUafWF7
 THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:CA==,iv:TM6XiygrDQn2qps4lO6hY2ldjW8LWlO3D/R/DBPqvGQ=,tag:+thBT7fMzH+DigecgxF4CQ==,type:str]
 THERMOGRAPH_INDEXNOW_KEY: ENC[AES256_GCM,data:XZEMMJfmrSjc/sKEePaigTqEkh3ob/E25R25DvOAWwY=,iv:fqdaegzG1Na0zW+UgOh92RS4FP22pxStdD1/Jq5DRjs=,tag:/8ORtVkGUWjqgGXG/xZHmg==,type:str]
 THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:5pL+8tZbs8TZ5ezJpkPUmMss0gLcte73lU+au/476Ws=,iv:ujU3G+rbPZQx/igg50GeIPQxqgd+szIsjc3AqGhGLBU=,tag:hrkgn2DPkmBGThlsdMplqg==,type:str]
-THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:XG+6wizXMVVFixhqLhg72HoKw9kEmH+Tty8jYMycvVk=,iv:tNz/WrkMYip2QyAr6bRy6PaIee56ZZ64BELBgvidVT0=,tag:HiQ55DGbeDAOSF0vpNWpWw==,type:str]
+THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:Ui5dcmhzz84nTp1gYhJ89VOJ2eOBFyLjf0WkZ7Y3sI0=,iv:WI0QqbUW7BHo7Zz0082zxpYNIYpVIhq1NeBGbQeDc/I=,tag:urNsE8DPaldJcw9aqX1PYA==,type:str]
 THERMOGRAPH_METRICS_TOKEN: ENC[AES256_GCM,data:DmOZU3HWQTuoSvQMb7iPkClJbDH48NB3OcRIYRkqNz8=,iv:acgGh0w2mc5ZD7rB7m/GW3Awsx3RWK+02L6YkAv5Rw0=,tag:x3U9F0Dekw/r9euRMjVQAA==,type:str]
 THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:hTSSUIfOlY0GYI8gZ7FcOi7c89iye0Ym73l4M22zkNk=,iv:aPdbXKb4pOqigo9J3a7oM3qTip6HQcHUQDlAwg6XfoE=,tag:C4p+Ob/iSElNdF4m9E+M7A==,type:str]
 THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:vGJDaElN6JRUawlD55Vr5Q==,iv:4ZAQ5NXb9xuMN+qPspM17W6zm9NIDXLX9hTJ/etSL0E=,tag:Ss7xaUl+kOiVF7oDs9FiYw==,type:str]
 THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:PtZg52Uu70Ndx7L/dRTCZTmqv80NXiohOg+gfrzG,iv:hKQTy3Twd5uWwnS4UIs9oqT4NzpYUWQxWO1lUqrzpAc=,tag:kYwYp58fxkzRe5yQPSXa3g==,type:str]
-THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:1xuZRaC4p+ZM/oFAUuGDZb6AAu2PbjFrHQY3zmczRpI=,iv:xzI24qlUOvYiThBLOue0odvopuxdHCwDlwI4JwiP9EU=,tag:yRS9GjGLBEYk/kZ6miFudg==,type:str]
+THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:4u5d3RHHBeIDfRIwQVDldWUhH7PyTcNP9s3xTrL4N74=,iv:RXWu+4ZdTmWx3PGw6nWhN2587byy4wqjY74Ldu4xvuk=,tag:dwS6htP2Dv2OQyNzmjZZIA==,type:str]
-THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:N9mPLx6Mcgqm5TlgqIMFhNuZsBZxVPXf3LplO9k+VA==,iv:y5+MLI0zC5Kb6pRdORTMVJ1iMMoFrVRfv99mKWMRjp8=,tag:7VvKCLU+1TJtQcHWlDwybg==,type:str]
+THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:hg5HqBua81dG3d4KaxLxljRkeGnt4h4=,iv:q+4tQGrlsQaRGUTJuqByqss+twR6v6NQrI/qSPjCbsQ=,tag:2D1+hv/KSDJu+4+J9S0hww==,type:str]
 THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:hTxKlgPWeW4HGBf08SxdAdK+ce6T5Fl9f+5tSGV6WpS+za5EI3zsK8BgRw==,iv:l+omFaCyL2+PHWdchadkmED2gIAoj5T0u/Ltzaw8mok=,tag:AQ3wT/wD5JAouX1M+Tjjmw==,type:str]
 THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:bgtqZFfTzYz5llh/YhAyGwGtRedK3/ze8SAWj8YdldY6s4nt2F3XeTANVO2Y9+NzvXc16PB3WWPnYEzCxnrG2KciIeYpOYZMoPpGldENPBLJtLa76Ej9,iv:HuZ7XtlXIt4wyJ79k3IYjHfWrQp7lnak4uCspyzS4BE=,tag:uPZvba8LVugDy76pKat6TA==,type:str]
 TIMESCALEDB_TAG: ENC[AES256_GCM,data:3mDU/k5fx0894+E=,iv:/iw3BZPF3mZ9M3bFNJSTzP9t15YZWCJwzoVXYL9Vj9o=,tag:Y4ez3+XfLJN1PtJyZlJj+g==,type:str]
@@ -25,7 +25,7 @@ sops:
             bncE6yn10QK5kVcF9dDvpQ6RbaG+ESpNZTnuhSL5PDqrKtjnsV0Iqw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
-    lastmodified: "2026-07-25T00:38:24Z"
+    lastmodified: "2026-07-31T04:23:12Z"
-    mac: ENC[AES256_GCM,data:C12PnFE7MO0Ge5dCAHCcyXh650hFtRuexl5d3DL0IqTf8nLyNtRJzrPyl9whIs+S3HulsXh3xJOwwPdg6aTqdCfGSQ4Kp5qV+OHxH9lfpHs2yH+exa/eX+7Khpjk0OfvX5beC0GSPpYf9c01buaUTWcIh3pJXpDMdqr+aLC+Zd0=,iv:6VeaaEEkWU5rdTHy75rc2emb2u/HmOmLzO11D38ZXHc=,tag:iGRevzdtqZF9kyH0wxqG3w==,type:str]
+    mac: ENC[AES256_GCM,data:KkItyA0iw2j4w9pf/efFcpyOP8RmO8XzfMoQhJcRXLZbrUC3/xltbxkfTIe2hRgAqKyz5d7EuAHBPwwSCubue0O2gzlG0nCnRQH7YyO4cUrP24XYb+aI+tBM2jdZIuZrGDNcq04cwKiecC6MyOOp8uQPZ41AOPH3sONGWj1VH2s=,iv:+j5uv4QbuWpXAyElvbYGKWhe9vmGTfs3Hpy5I5FetJo=,tag:wcfPuugefeQAyxdb2HV1Fg==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

infra/deploy/secrets/verify-age-quorum.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Assert that the age recovery quorum is intact.
+#
+#   infra/deploy/secrets/verify-age-quorum.sh
+#
+# The age private key is the single recovery root for all five SOPS vaults AND for
+# every off-box backup (ops-cron.yml:142,197 pipe dumps through `age -r` to the same
+# recipient). The copies at /etc/thermograph/age.key on vps1 and vps2 are a DELIBERATE
+# two-host quorum, not provisioning residue — see the "age key" section of
+# deploy/secrets/README.md. This script is what makes that policy checkable instead of
+# merely stated.
+#
+# It fails if the key is missing on either host, if the two copies have diverged, or
+# if either is more permissive than 0440. Divergence matters as much as absence: two
+# hosts holding different keys means one of them renders secrets nobody else can
+# decrypt, and a restore from backup silently picks the wrong one.
+#
+# OUTPUT DISCIPLINE: prints SHA-256 digests and file modes only. A digest of a
+# 32-byte random key is not reversible, so it is safe in a CI log; the key itself is
+# never read, echoed or transferred. Nothing here copies the key anywhere.
+#
+# Exit status: 0 = quorum intact; 1 = something needs a human. Suitable as a cron gate.
+set -euo pipefail
+
+VPS1="${TG_VPS1_SSH:-vps1}"
+VPS2="${TG_VPS2_SSH:-vps2}"
+KEY_PATH="${TG_AGE_KEY_PATH:-/etc/thermograph/age.key}"
+SSH_OPTS="${TG_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=10}"
+
+# Reads the digest and mode of the key on one host. Uses sudo when the key is not
+# directly readable, mirroring how render-secrets.sh lifts it.
+probe() {
+  local target="$1" out
+  # The remote script is a QUOTED heredoc and the key path is passed as $1 to
+  # `bash -s`, so nothing expands on this side. Interpolating the path into the
+  # command string would work too, but it is the shape that quietly breaks the day
+  # a path contains a space, and shellcheck is right to flag it (SC2029).
+  # shellcheck disable=SC2086  # SSH_OPTS is a deliberate word-split option list
+  out=$(ssh $SSH_OPTS "$target" bash -s -- "$KEY_PATH" 2>/dev/null <<'REMOTE'
+key="$1"
+if [ ! -e "$key" ]; then echo MISSING; exit 0; fi
+mode=$(stat -c %a "$key")
+owner=$(stat -c %U:%G "$key")
+if [ -r "$key" ]; then
+  digest=$(sha256sum "$key" | cut -d' ' -f1)
+else
+  digest=$(sudo sha256sum "$key" 2>/dev/null | cut -d' ' -f1)
+fi
+[ -n "$digest" ] || { echo UNREADABLE; exit 0; }
+echo "$digest $mode $owner"
+REMOTE
+  ) || { echo UNREACHABLE; return 0; }
+  printf '%s\n' "$out"
+}
+
+rc=0
+declare -A DIGEST
+
+for pair in "vps1:$VPS1" "vps2:$VPS2"; do
+  name="${pair%%:*}"
+  target="${pair#*:}"
+  result="$(probe "$target")"
+  case "$result" in
+    MISSING)
+      echo "!! ${name}: no key at ${KEY_PATH} — the quorum is DOWN TO ONE COPY" >&2
+      rc=1
+      ;;
+    UNREADABLE)
+      echo "!! ${name}: key present but unreadable even via sudo" >&2
+      rc=1
+      ;;
+    UNREACHABLE)
+      echo "!! ${name}: host unreachable — quorum NOT verified (this is not a pass)" >&2
+      rc=1
+      ;;
+    *)
+      digest="${result%% *}"
+      rest="${result#* }"
+      DIGEST["$name"]="$digest"
+      printf '   %-5s %s  mode=%s\n' "$name" "${digest:0:16}…" "$rest"
+      mode="${rest%% *}"
+      # 0400 or 0440 only. Anything wider means a non-root account on that box can
+      # read the key that decrypts prod — and vps1 also runs Forgejo and its CI runner.
+      case "$mode" in
+        400|440) ;;
+        *) echo "!! ${name}: mode ${mode} is wider than 0440" >&2; rc=1 ;;
+      esac
+      # Advisory, not a failure: 0440 with a non-root group means that group can read
+      # the key that decrypts prod. On vps1 that is doubly worth knowing, because the
+      # same box runs Forgejo, its CI runner and dev's unreviewed branches. Left
+      # non-fatal because it is the current provisioned state — a check that fails on
+      # day one is a check that gets ignored by day three. See README.md.
+      group="${rest#* }"; group="${group#*:}"
+      if [ "$mode" = 440 ] && [ "$group" != root ]; then
+        printf '         note: group %s can read this key\n' "$group"
+      fi
+      ;;
+  esac
+done
+
+if [ -n "${DIGEST[vps1]:-}" ] && [ -n "${DIGEST[vps2]:-}" ]; then
+  if [ "${DIGEST[vps1]}" = "${DIGEST[vps2]}" ]; then
+    echo "   quorum: 2 copies, identical"
+  else
+    echo "!! the two copies have DIVERGED — they are different keys" >&2
+    echo "!! do not 'fix' this by overwriting one. Establish which decrypts the" >&2
+    echo "!! vaults (sops -d on any deploy/secrets/*.yaml) before touching either." >&2
+    rc=1
+  fi
+fi
+
+if [ "$rc" -eq 0 ]; then
+  echo "   OK — recovery quorum intact"
+else
+  echo "!! recovery quorum degraded; see deploy/secrets/README.md" >&2
+fi
+exit "$rc"

infra/openbao/README.md

@@ -112,6 +112,22 @@ thermograph/data/legacy/age      the age PRIVATE key — see "the age key surviv
 Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
 isolation lives in the policy. Today both live in one shell variable.
 
+**`centralis/<env>` renders differently from everything above it, and must.** The
+app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
+is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
+pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
+sourcing it in a clean shell and comparing every value back before the file is
+written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
+strips its quotes, Centralis fails closed on the malformed result and serves a
+single `shared` identity, which is indistinguishable from the file never having
+been written. That is the 2026-07-24 incident, and the round-trip check exists so a
+render carrying it cannot reach `/etc`.
+
+Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
+metacharacters, no newlines) only to `common` and `env/*`. Applying them to
+`centralis/*` would reject data that renders perfectly well — the validator matches
+the renderer that will consume the path, not a single global rule.
+
 `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
 rule expressible as exactly one path, with no wildcard over `env/*` able to
 accidentally re-include it.
@@ -148,22 +164,52 @@ SOPS and retiring age are two different projects.
 ### Stand it up (once, by the operator, on vps2)
 
 ```sh
-sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, unit, init
+sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, ufw, unit, init
 # custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
-export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
+export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
 export BAO_TOKEN=<root token>
 sudo -E bash infra/openbao/bootstrap-policies.sh   # mount, policies, approles
-bao token revoke -self
 ```
 
-### Seed and prove (from your own machine — it needs your age key)
+> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
+> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
+> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
+> minting a root token requires a token you already have. **The recovery keys are not
+> a way back in**; they rekey, they do not authenticate. Unless
+> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
+> the last root token locks you out of an otherwise healthy, unsealed vault, and the
+> only remedy is re-initialising from scratch. Revoke only once a second admin
+> identity exists.
+
+Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
+`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
+openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
+with `certificate signed by unknown authority`.
+
+### Seed and prove (on vps2 — the age key is already there)
 
 ```sh
+export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
 infra/openbao/seed-from-sops.sh --dry-run     # key names + counts, writes nothing
 infra/openbao/seed-from-sops.sh --all
-infra/openbao/verify-parity.sh --all          # MUST pass before any flip
 ```
 
+Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
+to the host that legitimately renders it, so an environment can only be verified from
+its own host:
+
+```sh
+# on vps2
+infra/openbao/verify-parity.sh --env prod                          # expect 32 keys
+infra/openbao/verify-parity.sh --env beta                          # expect 24 keys
+# on vps1
+cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev  # expect 12 keys
+```
+
+Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per
+environment, so beta authenticates as `tg-beta` rather than falling back to prod's
+credentials and being denied its own path by `tg-host-prod.hcl`.
+
 `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
 the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
 inherits that rule.

infra/openbao/bootstrap-policies.sh

@@ -63,15 +63,21 @@ add_role tg-centralis tg-centralis  10.10.0.1/32
 
 # Install the local (vps2) credentials. prod and beta both render on this box.
 #
-# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
+# Both files are owned by `agent`, because that is the single identity both
-# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
+# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
-# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
+# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
-# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
+# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
-# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
+# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
-# creates one.
+# they do not, and the chown simply failed.
 #
-# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
+# So be accurate about what separate credentials buy on this box: NOT deploy-user
-# the one age key. This defends against MISCONFIGURATION, not against intrusion.
+# isolation — there is one deploy user, and it can read both files. What they buy is
+# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
+# *mistake* crossing between environments: a beta render authenticating with beta's
+# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
+# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
+# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
+# a separate SSH credential for beta in deploy.yml — a larger change than this script.
 install_creds() {
   local role="$1" dest="$2" owner="$3"
   local rid sid
@@ -79,7 +85,17 @@ install_creds() {
   sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
   # umask before creation, so the file is never briefly world-readable.
   ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
-  chown "$owner" "$dest"
+  # Abort the function if chown fails, and remove the half-installed file. The call
+  # site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
+  # so without this check a failed chown falls straight through to chmod and prints a
+  # line asserting an ownership that was never applied. A root-owned credential the
+  # renderer cannot read, reported as installed, is worse than no credential at all:
+  # it fails at deploy time instead of here.
+  if ! chown "$owner" "$dest"; then
+    rm -f "$dest"
+    echo "    !! chown $owner failed for $dest — removed, nothing installed" >&2
+    return 1
+  fi
   chmod 0400 "$dest"
   echo "    installed $dest (0400 $owner)"
 }
@@ -87,8 +103,7 @@ install_creds() {
 if [ "$(id -u)" = 0 ]; then
   install -d -o root -g root -m 0755 /etc/thermograph
   install_creds tg-prod      /etc/thermograph/openbao-approle       "agent:agent"
-  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "deploy:deploy" \
+  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "agent:agent"
-    || echo "    (beta creds skipped — no 'deploy' user on this box?)"
   install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
 else
   echo "!! not root; skipping credential install. Re-run with sudo -E." >&2

infra/openbao/bootstrap.sh

@@ -52,13 +52,19 @@ if ! command -v bao >/dev/null 2>&1; then
   # The checksums file is signed by the release; verify before installing.
   tmpd="$(mktemp -d)"
   base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
-  curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
+  # Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
-  curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
+  # The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
-  ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
+  tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
+  curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
+  curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
+  # Anchor to end-of-line and save under the real asset name: checksums.txt also lists
+  # <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
+  # so an unanchored match fails on a file that was never fetched.
+  ( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
     echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
     rm -rf "$tmpd"; exit 1
   }
-  tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
+  tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
   install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
   rm -rf "$tmpd"
 fi
@@ -88,7 +94,11 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
 echo "==> 3/7 static seal key"
 if [ ! -f /etc/openbao/unseal.key ]; then
   # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
-  ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
+  # tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
+  # static seal reads this file RAW. That one trailing byte makes the key neither
+  # 32 raw bytes nor clean base64, so bao refuses to start with
+  # `Error configuring seal "static": unknown encoding for AES-256 key`.
+  ( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
   chown openbao:openbao /etc/openbao/unseal.key
   chmod 0400 /etc/openbao/unseal.key
   echo "    generated /etc/openbao/unseal.key (0400 openbao)"
@@ -125,6 +135,24 @@ systemctl is-active --quiet openbao.service || {
   exit 1
 }
 
+# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
+# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
+# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
+# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
+# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
+# desktop, the phone — has any business reaching the secret store.
+if command -v ufw >/dev/null 2>&1; then
+  if ufw status | grep -q '8200.*10\.10\.0\.2'; then
+    echo "    ufw: 8200/tcp from 10.10.0.2 already allowed"
+  else
+    ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
+      comment 'vps1/dev -> openbao' >/dev/null
+    echo "    ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
+  fi
+else
+  echo "    !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
+fi
+
 export BAO_ADDR="https://127.0.0.1:8200"
 export BAO_CACERT=/etc/openbao/tls/bao.crt
 
@@ -140,9 +168,16 @@ else
   echo "    ############################################################"
   echo
   # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
-  # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
+  # yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
-  # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
+  # is redundancy against loss, not separation of duty — there is one operator.
-  # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
+  #
+  # These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
+  # the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
+  # (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
+  # minting a root token requires a token you already have. The deprecated endpoint is
+  # off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
+  # Consequence: revoking the last root token before another admin identity exists is
+  # a one-way door. See the ordering note in the handoff below.
   bao operator init -recovery-shares=3 -recovery-threshold=2
   echo
   echo "    Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
@@ -156,22 +191,36 @@ cat <<EOF
 
 ==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
 
-Paste the initial root token (or one minted from recovery keys) and run:
+Paste the initial root token printed above and run:
 
   export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
   export BAO_TOKEN=<root token>
   bash ${SELF_DIR}/bootstrap-policies.sh
 
+KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
+cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
+\`bao token revoke -self\` locks you out of an otherwise healthy vault.
+
 That script enables the KV mount, writes the policies, creates the AppRoles and
-installs each host's credentials. Then, from YOUR OWN machine (it needs your age
+installs each host's credentials. Then seed — on THIS box, where the age key already
-key, and per infra/CLAUDE.md a script that reads production secrets is not for an
+lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
-agent to run):
+secrets is not for an agent to run:
 
-  infra/openbao/seed-from-sops.sh --dry-run     # check first
+  export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
+  infra/openbao/seed-from-sops.sh --dry-run     # check first: 16/12/8/16 keys
   infra/openbao/seed-from-sops.sh --all
-  infra/openbao/verify-parity.sh --all          # must PASS before any flip
 
-Then revoke the root token:  bao token revoke -self
+Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
+that legitimately renders it, so an environment can only be verified from its own host.
+
+  # here (vps2)
+  infra/openbao/verify-parity.sh --env prod                        # expect 32 keys
+  infra/openbao/verify-parity.sh --env beta                        # expect 24 keys
+  # on vps1
+  cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev   # expect 12 keys
+
+Only once all three PASS, and only after you have a second admin identity that is not
+the root token, revoke it:  bao token revoke -self
 
 NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
 every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative

infra/openbao/config/openbao.hcl

@@ -25,8 +25,12 @@
 
 ui           = false
 cluster_name = "thermograph"
-disable_mlock = true
 log_level    = "info"
+// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
+// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
+// setting the server ignores only trains the operator to skim startup warnings,
+// which is where a real one will eventually hide. The corresponding note about
+// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
 
 // STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
 // DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
@@ -110,9 +114,21 @@ seal "static" {
 //
 // logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
 // unlinked inode and the log silently stops growing.
+// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
+// device NAME, not its type — and device settings go in `options`. Written as
+// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
+// be specified"); with type+path but a bare file_path it starts but silently
+// IGNORES the path and mode, which is the worse failure of the two.
 audit "file" {
-  file_path = "/var/log/openbao/audit.log"
+  type = "file"
-  mode      = "0600"
+  path = "file/"
+  options = {
+    file_path = "/var/log/openbao/audit.log"
+    mode      = "0600"
+  }
 }
 
-audit "syslog" {}
+audit "syslog" {
+  type = "syslog"
+  path = "syslog/"
+}

infra/openbao/seed-from-sops.sh

@@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do
   # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
   # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
   # SEED time — before anything depends on it — rather than at deploy time.
-  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
+  # Which renderer will consume this path decides which values are legal, so the
+  # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
+  # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
+  # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
+  # render_centralis_secrets and its OpenBao counterpart, where every byte inside
+  # '...' is literal — so applying the dotenv rules to it rejects data that renders
+  # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
+  quoted=0
+  case "$target" in centralis.*) quoted=1 ;; esac
+
+  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
 import json, os, re, sys
 
 doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
@@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc:
     sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
     sys.exit(1)
 
+# 1 => destined for a single-quoted env file; 0 => raw dotenv.
+QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
+
 KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 UNSAFE = set("`$\\\"'")
 
@@ -141,12 +154,13 @@ for key, val in doc.items():
         val = ""
     if not isinstance(val, str):
         val = json.dumps(val, separators=(",", ":"))
-    if "\n" in val or "\r" in val:
+    if not QUOTED:
-        bad.append(f"{key}: contains a newline")
+        if "\n" in val or "\r" in val:
-        continue
+            bad.append(f"{key}: contains a newline")
-    if UNSAFE & set(val):
+            continue
-        bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
+        if UNSAFE & set(val):
-        continue
+            bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
+            continue
     out[key] = val
 
 if bad: