rotate Contabo S3 credentials

infra/deploy/env-topology.sh

@@ -76,16 +76,6 @@ thermograph_topology() {
   # for the reason that marker was demoted to a fallback in the first place: vps2 runs
   # prod and beta side by side, and one host-wide file cannot name two backends.
   TG_SECRETS_BACKEND=sops
-  # Which AppRole credential file render-secrets-openbao.sh authenticates with. Same
-  # reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO
-  # environments, so a single host-wide default cannot serve both. prod and dev each
-  # get the conventional path (dev is alone on vps1, so there is no collision there);
-  # beta is the one that must differ, because it shares a filesystem with prod.
-  #
-  # Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl
-  # correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that
-  # runs prod.
-  TG_BAO_APPROLE=/etc/thermograph/openbao-approle
 
   case "$env_name" in
     prod)
@@ -126,10 +116,6 @@ thermograph_topology() {
       # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh).
       TG_SSH_HOST=169.58.46.181
       TG_SSH_TARGET=agent@169.58.46.181
-      # The one environment that cannot use the default AppRole path: it shares a
-      # filesystem with prod, so it needs its own credential file to authenticate as
-      # tg-beta rather than tg-prod. bootstrap-policies.sh installs it here.
-      TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta
       # A SECOND checkout on the same box. Separate from prod's so the two can
       # sit on different commits of this repo, and so `git reset --hard` in one
       # deploy can never yank the tree out from under the other's running
@@ -242,7 +228,7 @@ thermograph_topology() {
   export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
   export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
   export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
-  export TG_SECRETS_BACKEND TG_BAO_APPROLE
+  export TG_SECRETS_BACKEND
   export TG_SSH_HOST TG_SSH_TARGET
 }
 

infra/deploy/render-secrets-openbao.sh

@@ -1,14 +1,9 @@
 #!/usr/bin/env bash
 # OpenBao source backend for render-secrets.sh. Sourced, never run directly.
 #
-# Functions:
+# Two functions:
-#   thermograph_openbao_source <env>                 -> merged dotenv on STDOUT
+#   thermograph_openbao_source <env>        -> merged dotenv on STDOUT
-#   thermograph_render_openbao  <env> <out>          -> source, then write <out>
+#   thermograph_render_openbao  <env> <out> -> source, then write <out>
-#   thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
-#
-# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
-# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
-# a different shape for a different consumer, not an inconsistency.
 #
 # render_thermograph_secrets in render-secrets.sh dispatches to the second when
 # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
@@ -32,13 +27,7 @@ thermograph_openbao_source() {
   local env_name="${1:?env name required}"
   local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
   local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
-  # Explicit override first, then the per-environment value env-topology.sh derives,
+  local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
-  # then the conventional path. Same precedence shape as render-secrets.sh:44's
-  # THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the
-  # THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the
-  # deploy path sets. Falling straight through to the bare default is what made beta
-  # authenticate as tg-prod and get denied its own path.
-  local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}"
   local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
 
   command -v bao >/dev/null 2>&1 || {
@@ -118,16 +107,9 @@ thermograph_openbao_source() {
   #     /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
   #     deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
   #     `$(...)` or a backtick in a secret is a live command-execution path on the
-  #     deploy host. Refusing here turns a latent code-execution bug into a loud
+  #     deploy host. The SOPS path has always had this hazard and has never tripped it
-  #     render failure.
+  #     only because every current value happens to be alphanumeric. Refusing here
-  #
+  #     turns a latent code-execution bug into a loud render failure.
-  #     An earlier version of this comment claimed the SOPS path had never tripped
-  #     this "only because every current value happens to be alphanumeric". That is
-  #     false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
-  #     tripped THESE paths because it is not in them — it lives at
-  #     centralis/<env> and renders to /etc/centralis.env, which is single-quoted
-  #     precisely because raw dotenv cannot carry it. See
-  #     thermograph_render_openbao_centralis at the end of this file.
   THERMOGRAPH_BAO_COMMON="$common_json" \
   THERMOGRAPH_BAO_ENV="$env_json" \
   python3 - <<'PY'
@@ -255,239 +237,3 @@ thermograph_render_openbao() {
   rm -f "$tmp"
   return "$rc"
 }
-
-# ---------------------------------------------------------------------------
-# CENTRALIS
-# ---------------------------------------------------------------------------
-# /etc/centralis.env is a different file, with a different consumer, and therefore a
-# different output shape. render-secrets.sh:206-233 carries the full argument; the
-# short version is that it is consumed by exactly one thing and that thing is a shell:
-#
-#     sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
-#
-# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
-# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
-# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
-# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
-# by serving a single `shared` identity — indistinguishable from the file never having
-# been written. That is the 2026-07-24 incident. So this path emits POSIX
-# single-quoted assignments and PROVES them by sourcing the result in a clean shell
-# before anything touches $dest.
-#
-# This mirrors render_centralis_secrets rather than sharing with it, for the same
-# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
-# and all new risk stays in this file. Consolidate the two when the SOPS path is being
-# deleted anyway, not before.
-
-# _thermograph_openbao_login <approle_file> <addr> <ca>  -> token on STDOUT
-#
-# Separate from the login inline in thermograph_openbao_source deliberately: that
-# function is now exercised by prod and beta parity and is left untouched.
-_thermograph_openbao_login() {
-  local approle="$1" addr="$2" ca="$3"
-  local creds role_id secret_id
-  if [ -r "$approle" ]; then
-    creds=$(cat "$approle")
-  else
-    creds=$(sudo cat "$approle" 2>/dev/null || true)
-  fi
-  [ -n "$creds" ] || {
-    echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
-    return 1
-  }
-  role_id=$(printf '%s\n' "$creds" | sed -n '1p')
-  secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
-  [ -n "$role_id" ] && [ -n "$secret_id" ] || {
-    echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
-    return 1
-  }
-  export BAO_ADDR="$addr"
-  [ -f "$ca" ] && export BAO_CACERT="$ca"
-  bao write -field=token auth/approle/login \
-    role_id="$role_id" secret_id="$secret_id" 2>/dev/null
-}
-
-# thermograph_render_openbao_centralis <env_name> [dest]
-thermograph_render_openbao_centralis() {
-  local env_name="${1:?env name required}"
-  local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
-  local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
-  local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
-  local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
-  local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
-
-  command -v bao >/dev/null 2>&1 || {
-    echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
-    return 1
-  }
-  command -v python3 >/dev/null 2>&1 || {
-    echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
-    return 1
-  }
-
-  echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
-
-  local token
-  token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
-  [ -n "$token" ] || {
-    echo "!! OpenBao AppRole login failed for centralis" >&2
-    echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
-    return 1
-  }
-  export BAO_TOKEN="$token"
-
-  # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
-  # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
-  # persists into the caller's shell and re-fires on its next `source`.
-  local work rc=0
-  work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
-  chmod 700 "$work"
-
-  # Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
-  # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
-  # command that is the left operand of `||`, which this subshell is.
-  (
-    umask 077
-
-    bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
-      > "$work/kv.json" 2>/dev/null || {
-      echo "!! cannot read ${mount}/centralis/${env_name}" >&2
-      exit 1
-    }
-
-    python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
-import json, sys
-doc = json.load(open(sys.argv[1], encoding="utf-8"))
-data = doc.get("data", {}).get("data", {}) or {}
-if not data:
-    sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
-    sys.exit(1)
-json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
-UNWRAP
-
-    python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
-import json, re, sys
-
-src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
-NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
-
-
-def shell_quote(v):
-    """POSIX single-quoting. Inside '...' every byte is literal except ' itself,
-    which is closed, backslash-escaped and reopened. There is no escape sequence
-    to get wrong and no character class to keep up to date: it is total, and
-    total is the only property worth having for a file bash will expand."""
-    return "'" + v.replace("'", "'\\''") + "'"
-
-
-def scalar(k, v):
-    if isinstance(v, str):
-        return v
-    if isinstance(v, bool):          # before int: bool is an int in Python
-        return "true" if v else "false"
-    if isinstance(v, (int, float)):
-        return json.dumps(v)
-    if v is None:
-        return ""
-    sys.stderr.write(
-        "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
-    sys.exit(1)
-
-
-data = json.load(open(src, encoding="utf-8"))
-want, lines = {}, []
-for k, v in data.items():
-    if not NAME.match(k):
-        sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
-        sys.exit(1)
-    want[k] = scalar(k, v)
-    lines.append("%s=%s\n" % (k, shell_quote(want[k])))
-
-with open(out_env, "w", encoding="utf-8") as fh:
-    fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
-    fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
-    fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
-    fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
-    fh.writelines(lines)
-with open(want_json, "w", encoding="utf-8") as fh:
-    json.dump(want, fh)
-sys.stderr.write("    %d keys quoted\n" % len(want))
-EMIT
-
-    # PROVE IT. Source the rendered file exactly the way the deploy does — clean
-    # environment, `set -a`, nothing inherited that could mask a dropped key — and
-    # read every value back. This is the check the 2026-07-24 file could not pass.
-    # shellcheck disable=SC2016  # single quotes are the point: "$1" must be expanded
-    # by the inner `bash -c` against the argument after `_`, not by this shell.
-    env -i PATH="$PATH" bash -c \
-      'set -a; . "$1"; set +a; exec python3 -c "
-import json, os, sys
-sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
-      > "$work/got.json" || exit 1
-
-    python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
-import json, sys
-
-want = json.load(open(sys.argv[1], encoding="utf-8"))
-got = json.load(open(sys.argv[2], encoding="utf-8"))
-
-bad = []
-for k, v in want.items():
-    if k not in got:
-        bad.append("%s: not set at all after sourcing" % k)
-    elif got[k] != v:
-        # NEVER print either value. The difference is the finding; the content is
-        # not, and this runs inside a deploy log.
-        bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
-                   % (k, len(v), len(got[k])))
-if bad:
-    sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
-    for b in bad:
-        sys.stderr.write("!!   %s\n" % b)
-    sys.exit(1)
-
-# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
-# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
-# single `shared` identity, which looks exactly like the registry never having been
-# configured. Refuse to write a file that would do that.
-raw = got.get("CENTRALIS_TOKENS", "")
-if raw:
-    try:
-        reg = json.loads(raw)
-    except ValueError as exc:
-        sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
-        sys.exit(1)
-    if not isinstance(reg, dict) or not reg:
-        sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
-        sys.exit(1)
-    if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
-        sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
-        sys.exit(1)
-    # Subjects are names on audit lines, not credentials. Printing them is the whole
-    # point: it is how an operator sees at a glance that nobody was lost.
-    sys.stderr.write("    CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
-                     % (len(reg), ", ".join(sorted(reg))))
-
-sys.stderr.write("    %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
-VERIFY
-  ) || rc=1
-
-  # Only now, with a file read back through bash and matched value for value, does
-  # anything touch $dest. 0600: four bearer credentials for the estate's control
-  # plane, with no group that needs them.
-  if [ "$rc" -eq 0 ]; then
-    if [ -f "$dest" ] && [ -w "$dest" ]; then
-      cat "$work/out.env" > "$dest" || rc=1
-    elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
-    elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
-    else
-      echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
-      rc=1
-    fi
-  else
-    echo "!! render aborted; ${dest} left untouched" >&2
-  fi
-
-  rm -rf "$work"
-  return "$rc"
-}

infra/deploy/render-secrets.sh

@@ -272,30 +272,6 @@ render_centralis_secrets() {
     echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
     return 1
   fi
-
-  # Same early-return dispatch as render_thermograph_secrets, for the same reason
-  # given there: everything below — the age-key sudo lift, the JSON decrypt, the
-  # quote-and-prove pipeline, the three-way write — stays on exactly the code path it
-  # has always been on, so this cannot regress the backend still authoritative for
-  # Centralis. The OpenBao path needs no age key, so it returns before that check.
-  local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
-  if [ "$backend" = openbao ]; then
-    local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
-    if [ ! -f "$bao_lib" ]; then
-      echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
-      echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
-      return 1
-    fi
-    # shellcheck source=/dev/null
-    . "$bao_lib"
-    thermograph_render_openbao_centralis "$env_name" "$dest"
-    return
-  fi
-  if [ "$backend" != sops ]; then
-    echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
-    return 1
-  fi
-
   if [ ! -f "$key" ]; then
     echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
     return 1

infra/openbao/README.md

@@ -112,22 +112,6 @@ thermograph/data/legacy/age      the age PRIVATE key — see "the age key surviv
 Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
 isolation lives in the policy. Today both live in one shell variable.
 
-**`centralis/<env>` renders differently from everything above it, and must.** The
-app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
-is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
-pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
-sourcing it in a clean shell and comparing every value back before the file is
-written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
-strips its quotes, Centralis fails closed on the malformed result and serves a
-single `shared` identity, which is indistinguishable from the file never having
-been written. That is the 2026-07-24 incident, and the round-trip check exists so a
-render carrying it cannot reach `/etc`.
-
-Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
-metacharacters, no newlines) only to `common` and `env/*`. Applying them to
-`centralis/*` would reject data that renders perfectly well — the validator matches
-the renderer that will consume the path, not a single global rule.
-
 `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
 rule expressible as exactly one path, with no wildcard over `env/*` able to
 accidentally re-include it.
@@ -164,52 +148,22 @@ SOPS and retiring age are two different projects.
 ### Stand it up (once, by the operator, on vps2)
 
 ```sh
-sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, ufw, unit, init
+sudo bash infra/openbao/bootstrap.sh          # binary, TLS, seal key, unit, init
 # custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
-export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
+export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
 export BAO_TOKEN=<root token>
 sudo -E bash infra/openbao/bootstrap-policies.sh   # mount, policies, approles
+bao token revoke -self
 ```
 
-> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
+### Seed and prove (from your own machine — it needs your age key)
-> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
-> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
-> minting a root token requires a token you already have. **The recovery keys are not
-> a way back in**; they rekey, they do not authenticate. Unless
-> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
-> the last root token locks you out of an otherwise healthy, unsealed vault, and the
-> only remedy is re-initialising from scratch. Revoke only once a second admin
-> identity exists.
-
-Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
-`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
-openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
-with `certificate signed by unknown authority`.
-
-### Seed and prove (on vps2 — the age key is already there)
 
 ```sh
-export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
 infra/openbao/seed-from-sops.sh --dry-run     # key names + counts, writes nothing
 infra/openbao/seed-from-sops.sh --all
+infra/openbao/verify-parity.sh --all          # MUST pass before any flip
 ```
 
-Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
-to the host that legitimately renders it, so an environment can only be verified from
-its own host:
-
-```sh
-# on vps2
-infra/openbao/verify-parity.sh --env prod                          # expect 32 keys
-infra/openbao/verify-parity.sh --env beta                          # expect 24 keys
-# on vps1
-cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev  # expect 12 keys
-```
-
-Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per
-environment, so beta authenticates as `tg-beta` rather than falling back to prod's
-credentials and being denied its own path by `tg-host-prod.hcl`.
-
 `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
 the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
 inherits that rule.

infra/openbao/bootstrap-policies.sh

@@ -63,21 +63,15 @@ add_role tg-centralis tg-centralis  10.10.0.1/32
 
 # Install the local (vps2) credentials. prod and beta both render on this box.
 #
-# Both files are owned by `agent`, because that is the single identity both
+# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
-# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
+# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
-# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
+# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
-# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
+# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
-# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
+# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
-# they do not, and the chown simply failed.
+# creates one.
 #
-# So be accurate about what separate credentials buy on this box: NOT deploy-user
+# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
-# isolation — there is one deploy user, and it can read both files. What they buy is
+# the one age key. This defends against MISCONFIGURATION, not against intrusion.
-# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
-# *mistake* crossing between environments: a beta render authenticating with beta's
-# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
-# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
-# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
-# a separate SSH credential for beta in deploy.yml — a larger change than this script.
 install_creds() {
   local role="$1" dest="$2" owner="$3"
   local rid sid
@@ -85,17 +79,7 @@ install_creds() {
   sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
   # umask before creation, so the file is never briefly world-readable.
   ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
-  # Abort the function if chown fails, and remove the half-installed file. The call
+  chown "$owner" "$dest"
-  # site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
-  # so without this check a failed chown falls straight through to chmod and prints a
-  # line asserting an ownership that was never applied. A root-owned credential the
-  # renderer cannot read, reported as installed, is worse than no credential at all:
-  # it fails at deploy time instead of here.
-  if ! chown "$owner" "$dest"; then
-    rm -f "$dest"
-    echo "    !! chown $owner failed for $dest — removed, nothing installed" >&2
-    return 1
-  fi
   chmod 0400 "$dest"
   echo "    installed $dest (0400 $owner)"
 }
@@ -103,7 +87,8 @@ install_creds() {
 if [ "$(id -u)" = 0 ]; then
   install -d -o root -g root -m 0755 /etc/thermograph
   install_creds tg-prod      /etc/thermograph/openbao-approle       "agent:agent"
-  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "agent:agent"
+  install_creds tg-beta      /etc/thermograph/openbao-approle-beta  "deploy:deploy" \
+    || echo "    (beta creds skipped — no 'deploy' user on this box?)"
   install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
 else
   echo "!! not root; skipping credential install. Re-run with sudo -E." >&2

infra/openbao/bootstrap.sh

@@ -52,19 +52,13 @@ if ! command -v bao >/dev/null 2>&1; then
   # The checksums file is signed by the release; verify before installing.
   tmpd="$(mktemp -d)"
   base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
-  # Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
+  curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
-  # The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
+  curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
-  tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
+  ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
-  curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
-  curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
-  # Anchor to end-of-line and save under the real asset name: checksums.txt also lists
-  # <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
-  # so an unanchored match fails on a file that was never fetched.
-  ( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
     echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
     rm -rf "$tmpd"; exit 1
   }
-  tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
+  tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
   install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
   rm -rf "$tmpd"
 fi
@@ -94,11 +88,7 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
 echo "==> 3/7 static seal key"
 if [ ! -f /etc/openbao/unseal.key ]; then
   # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
-  # tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
+  ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
-  # static seal reads this file RAW. That one trailing byte makes the key neither
-  # 32 raw bytes nor clean base64, so bao refuses to start with
-  # `Error configuring seal "static": unknown encoding for AES-256 key`.
-  ( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
   chown openbao:openbao /etc/openbao/unseal.key
   chmod 0400 /etc/openbao/unseal.key
   echo "    generated /etc/openbao/unseal.key (0400 openbao)"
@@ -135,24 +125,6 @@ systemctl is-active --quiet openbao.service || {
   exit 1
 }
 
-# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
-# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
-# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
-# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
-# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
-# desktop, the phone — has any business reaching the secret store.
-if command -v ufw >/dev/null 2>&1; then
-  if ufw status | grep -q '8200.*10\.10\.0\.2'; then
-    echo "    ufw: 8200/tcp from 10.10.0.2 already allowed"
-  else
-    ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
-      comment 'vps1/dev -> openbao' >/dev/null
-    echo "    ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
-  fi
-else
-  echo "    !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
-fi
-
 export BAO_ADDR="https://127.0.0.1:8200"
 export BAO_CACERT=/etc/openbao/tls/bao.crt
 
@@ -168,16 +140,9 @@ else
   echo "    ############################################################"
   echo
   # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
-  # yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
+  # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
-  # is redundancy against loss, not separation of duty — there is one operator.
+  # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
-  #
+  # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
-  # These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
-  # the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
-  # (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
-  # minting a root token requires a token you already have. The deprecated endpoint is
-  # off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
-  # Consequence: revoking the last root token before another admin identity exists is
-  # a one-way door. See the ordering note in the handoff below.
   bao operator init -recovery-shares=3 -recovery-threshold=2
   echo
   echo "    Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
@@ -191,36 +156,22 @@ cat <<EOF
 
 ==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
 
-Paste the initial root token printed above and run:
+Paste the initial root token (or one minted from recovery keys) and run:
 
   export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
   export BAO_TOKEN=<root token>
   bash ${SELF_DIR}/bootstrap-policies.sh
 
-KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
-cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
-\`bao token revoke -self\` locks you out of an otherwise healthy vault.
-
 That script enables the KV mount, writes the policies, creates the AppRoles and
-installs each host's credentials. Then seed — on THIS box, where the age key already
+installs each host's credentials. Then, from YOUR OWN machine (it needs your age
-lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
+key, and per infra/CLAUDE.md a script that reads production secrets is not for an
-secrets is not for an agent to run:
+agent to run):
 
-  export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
+  infra/openbao/seed-from-sops.sh --dry-run     # check first
-  infra/openbao/seed-from-sops.sh --dry-run     # check first: 16/12/8/16 keys
   infra/openbao/seed-from-sops.sh --all
+  infra/openbao/verify-parity.sh --all          # must PASS before any flip
 
-Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
+Then revoke the root token:  bao token revoke -self
-that legitimately renders it, so an environment can only be verified from its own host.
-
-  # here (vps2)
-  infra/openbao/verify-parity.sh --env prod                        # expect 32 keys
-  infra/openbao/verify-parity.sh --env beta                        # expect 24 keys
-  # on vps1
-  cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev   # expect 12 keys
-
-Only once all three PASS, and only after you have a second admin identity that is not
-the root token, revoke it:  bao token revoke -self
 
 NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
 every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative

infra/openbao/config/openbao.hcl

@@ -25,12 +25,8 @@
 
 ui           = false
 cluster_name = "thermograph"
+disable_mlock = true
 log_level    = "info"
-// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
-// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
-// setting the server ignores only trains the operator to skim startup warnings,
-// which is where a real one will eventually hide. The corresponding note about
-// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
 
 // STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
 // DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
@@ -114,21 +110,9 @@ seal "static" {
 //
 // logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
 // unlinked inode and the log silently stops growing.
-// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
-// device NAME, not its type — and device settings go in `options`. Written as
-// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
-// be specified"); with type+path but a bare file_path it starts but silently
-// IGNORES the path and mode, which is the worse failure of the two.
 audit "file" {
-  type = "file"
+  file_path = "/var/log/openbao/audit.log"
-  path = "file/"
+  mode      = "0600"
-  options = {
-    file_path = "/var/log/openbao/audit.log"
-    mode      = "0600"
-  }
 }
 
-audit "syslog" {
+audit "syslog" {}
-  type = "syslog"
-  path = "syslog/"
-}

infra/openbao/seed-from-sops.sh

@@ -119,17 +119,7 @@ for target in "${TARGETS[@]}"; do
   # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
   # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
   # SEED time — before anything depends on it — rather than at deploy time.
-  # Which renderer will consume this path decides which values are legal, so the
+  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
-  # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
-  # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
-  # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
-  # render_centralis_secrets and its OpenBao counterpart, where every byte inside
-  # '...' is literal — so applying the dotenv rules to it rejects data that renders
-  # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
-  quoted=0
-  case "$target" in centralis.*) quoted=1 ;; esac
-
-  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
 import json, os, re, sys
 
 doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
@@ -137,9 +127,6 @@ if not isinstance(doc, dict) or not doc:
     sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
     sys.exit(1)
 
-# 1 => destined for a single-quoted env file; 0 => raw dotenv.
-QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
-
 KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 UNSAFE = set("`$\\\"'")
 
@@ -154,13 +141,12 @@ for key, val in doc.items():
         val = ""
     if not isinstance(val, str):
         val = json.dumps(val, separators=(",", ":"))
-    if not QUOTED:
+    if "\n" in val or "\r" in val:
-        if "\n" in val or "\r" in val:
+        bad.append(f"{key}: contains a newline")
-            bad.append(f"{key}: contains a newline")
+        continue
-            continue
+    if UNSAFE & set(val):
-        if UNSAFE & set(val):
+        bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
-            bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
+        continue
-            continue
     out[key] = val
 
 if bad: