infra/deploy/render-secrets-openbao.sh

@@ -1,9 +1,14 @@
 #!/usr/bin/env bash
 # OpenBao source backend for render-secrets.sh. Sourced, never run directly.
 #
-# Two functions:
-#   thermograph_openbao_source <env>        -> merged dotenv on STDOUT
-#   thermograph_render_openbao  <env> <out> -> source, then write <out>
+# Functions:
+#   thermograph_openbao_source <env>                 -> merged dotenv on STDOUT
+#   thermograph_render_openbao  <env> <out>          -> source, then write <out>
+#   thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
+#
+# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
+# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
+# a different shape for a different consumer, not an inconsistency.
 #
 # render_thermograph_secrets in render-secrets.sh dispatches to the second when
 # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
@@ -113,9 +118,16 @@ thermograph_openbao_source() {
   #     /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
   #     deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
   #     `$(...)` or a backtick in a secret is a live command-execution path on the
-  #     deploy host. The SOPS path has always had this hazard and has never tripped it
-  #     only because every current value happens to be alphanumeric. Refusing here
-  #     turns a latent code-execution bug into a loud render failure.
+  #     deploy host. Refusing here turns a latent code-execution bug into a loud
+  #     render failure.
+  #
+  #     An earlier version of this comment claimed the SOPS path had never tripped
+  #     this "only because every current value happens to be alphanumeric". That is
+  #     false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
+  #     tripped THESE paths because it is not in them — it lives at
+  #     centralis/<env> and renders to /etc/centralis.env, which is single-quoted
+  #     precisely because raw dotenv cannot carry it. See
+  #     thermograph_render_openbao_centralis at the end of this file.
   THERMOGRAPH_BAO_COMMON="$common_json" \
   THERMOGRAPH_BAO_ENV="$env_json" \
   python3 - <<'PY'
@@ -243,3 +255,239 @@ thermograph_render_openbao() {
   rm -f "$tmp"
   return "$rc"
 }
+
+# ---------------------------------------------------------------------------
+# CENTRALIS
+# ---------------------------------------------------------------------------
+# /etc/centralis.env is a different file, with a different consumer, and therefore a
+# different output shape. render-secrets.sh:206-233 carries the full argument; the
+# short version is that it is consumed by exactly one thing and that thing is a shell:
+#
+#     sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
+#
+# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
+# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
+# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
+# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
+# by serving a single `shared` identity — indistinguishable from the file never having
+# been written. That is the 2026-07-24 incident. So this path emits POSIX
+# single-quoted assignments and PROVES them by sourcing the result in a clean shell
+# before anything touches $dest.
+#
+# This mirrors render_centralis_secrets rather than sharing with it, for the same
+# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
+# and all new risk stays in this file. Consolidate the two when the SOPS path is being
+# deleted anyway, not before.
+
+# _thermograph_openbao_login <approle_file> <addr> <ca>  -> token on STDOUT
+#
+# Separate from the login inline in thermograph_openbao_source deliberately: that
+# function is now exercised by prod and beta parity and is left untouched.
+_thermograph_openbao_login() {
+  local approle="$1" addr="$2" ca="$3"
+  local creds role_id secret_id
+  if [ -r "$approle" ]; then
+    creds=$(cat "$approle")
+  else
+    creds=$(sudo cat "$approle" 2>/dev/null || true)
+  fi
+  [ -n "$creds" ] || {
+    echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
+    return 1
+  }
+  role_id=$(printf '%s\n' "$creds" | sed -n '1p')
+  secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
+  [ -n "$role_id" ] && [ -n "$secret_id" ] || {
+    echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
+    return 1
+  }
+  export BAO_ADDR="$addr"
+  [ -f "$ca" ] && export BAO_CACERT="$ca"
+  bao write -field=token auth/approle/login \
+    role_id="$role_id" secret_id="$secret_id" 2>/dev/null
+}
+
+# thermograph_render_openbao_centralis <env_name> [dest]
+thermograph_render_openbao_centralis() {
+  local env_name="${1:?env name required}"
+  local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
+  local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
+  local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
+  local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
+  local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
+
+  command -v bao >/dev/null 2>&1 || {
+    echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
+    return 1
+  }
+  command -v python3 >/dev/null 2>&1 || {
+    echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
+    return 1
+  }
+
+  echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
+
+  local token
+  token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
+  [ -n "$token" ] || {
+    echo "!! OpenBao AppRole login failed for centralis" >&2
+    echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
+    return 1
+  }
+  export BAO_TOKEN="$token"
+
+  # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
+  # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
+  # persists into the caller's shell and re-fires on its next `source`.
+  local work rc=0
+  work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
+  chmod 700 "$work"
+
+  # Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
+  # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
+  # command that is the left operand of `||`, which this subshell is.
+  (
+    umask 077
+
+    bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
+      > "$work/kv.json" 2>/dev/null || {
+      echo "!! cannot read ${mount}/centralis/${env_name}" >&2
+      exit 1
+    }
+
+    python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
+import json, sys
+doc = json.load(open(sys.argv[1], encoding="utf-8"))
+data = doc.get("data", {}).get("data", {}) or {}
+if not data:
+    sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
+    sys.exit(1)
+json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
+UNWRAP
+
+    python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
+import json, re, sys
+
+src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
+NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
+
+
+def shell_quote(v):
+    """POSIX single-quoting. Inside '...' every byte is literal except ' itself,
+    which is closed, backslash-escaped and reopened. There is no escape sequence
+    to get wrong and no character class to keep up to date: it is total, and
+    total is the only property worth having for a file bash will expand."""
+    return "'" + v.replace("'", "'\\''") + "'"
+
+
+def scalar(k, v):
+    if isinstance(v, str):
+        return v
+    if isinstance(v, bool):          # before int: bool is an int in Python
+        return "true" if v else "false"
+    if isinstance(v, (int, float)):
+        return json.dumps(v)
+    if v is None:
+        return ""
+    sys.stderr.write(
+        "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
+    sys.exit(1)
+
+
+data = json.load(open(src, encoding="utf-8"))
+want, lines = {}, []
+for k, v in data.items():
+    if not NAME.match(k):
+        sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
+        sys.exit(1)
+    want[k] = scalar(k, v)
+    lines.append("%s=%s\n" % (k, shell_quote(want[k])))
+
+with open(out_env, "w", encoding="utf-8") as fh:
+    fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
+    fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
+    fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
+    fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
+    fh.writelines(lines)
+with open(want_json, "w", encoding="utf-8") as fh:
+    json.dump(want, fh)
+sys.stderr.write("    %d keys quoted\n" % len(want))
+EMIT
+
+    # PROVE IT. Source the rendered file exactly the way the deploy does — clean
+    # environment, `set -a`, nothing inherited that could mask a dropped key — and
+    # read every value back. This is the check the 2026-07-24 file could not pass.
+    # shellcheck disable=SC2016  # single quotes are the point: "$1" must be expanded
+    # by the inner `bash -c` against the argument after `_`, not by this shell.
+    env -i PATH="$PATH" bash -c \
+      'set -a; . "$1"; set +a; exec python3 -c "
+import json, os, sys
+sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
+      > "$work/got.json" || exit 1
+
+    python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
+import json, sys
+
+want = json.load(open(sys.argv[1], encoding="utf-8"))
+got = json.load(open(sys.argv[2], encoding="utf-8"))
+
+bad = []
+for k, v in want.items():
+    if k not in got:
+        bad.append("%s: not set at all after sourcing" % k)
+    elif got[k] != v:
+        # NEVER print either value. The difference is the finding; the content is
+        # not, and this runs inside a deploy log.
+        bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
+                   % (k, len(v), len(got[k])))
+if bad:
+    sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
+    for b in bad:
+        sys.stderr.write("!!   %s\n" % b)
+    sys.exit(1)
+
+# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
+# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
+# single `shared` identity, which looks exactly like the registry never having been
+# configured. Refuse to write a file that would do that.
+raw = got.get("CENTRALIS_TOKENS", "")
+if raw:
+    try:
+        reg = json.loads(raw)
+    except ValueError as exc:
+        sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
+        sys.exit(1)
+    if not isinstance(reg, dict) or not reg:
+        sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
+        sys.exit(1)
+    if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
+        sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
+        sys.exit(1)
+    # Subjects are names on audit lines, not credentials. Printing them is the whole
+    # point: it is how an operator sees at a glance that nobody was lost.
+    sys.stderr.write("    CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
+                     % (len(reg), ", ".join(sorted(reg))))
+
+sys.stderr.write("    %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
+VERIFY
+  ) || rc=1
+
+  # Only now, with a file read back through bash and matched value for value, does
+  # anything touch $dest. 0600: four bearer credentials for the estate's control
+  # plane, with no group that needs them.
+  if [ "$rc" -eq 0 ]; then
+    if [ -f "$dest" ] && [ -w "$dest" ]; then
+      cat "$work/out.env" > "$dest" || rc=1
+    elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
+    elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
+    else
+      echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
+      rc=1
+    fi
+  else
+    echo "!! render aborted; ${dest} left untouched" >&2
+  fi
+
+  rm -rf "$work"
+  return "$rc"
+}

infra/deploy/render-secrets.sh

@@ -272,6 +272,30 @@ render_centralis_secrets() {
     echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
     return 1
   fi
+
+  # Same early-return dispatch as render_thermograph_secrets, for the same reason
+  # given there: everything below — the age-key sudo lift, the JSON decrypt, the
+  # quote-and-prove pipeline, the three-way write — stays on exactly the code path it
+  # has always been on, so this cannot regress the backend still authoritative for
+  # Centralis. The OpenBao path needs no age key, so it returns before that check.
+  local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
+  if [ "$backend" = openbao ]; then
+    local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
+    if [ ! -f "$bao_lib" ]; then
+      echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
+      echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
+      return 1
+    fi
+    # shellcheck source=/dev/null
+    . "$bao_lib"
+    thermograph_render_openbao_centralis "$env_name" "$dest"
+    return
+  fi
+  if [ "$backend" != sops ]; then
+    echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
+    return 1
+  fi
+
   if [ ! -f "$key" ]; then
     echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
     return 1

infra/openbao/README.md

@@ -112,6 +112,22 @@ thermograph/data/legacy/age      the age PRIVATE key — see "the age key surviv
 Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
 isolation lives in the policy. Today both live in one shell variable.
 
+**`centralis/<env>` renders differently from everything above it, and must.** The
+app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
+is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
+pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
+sourcing it in a clean shell and comparing every value back before the file is
+written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
+strips its quotes, Centralis fails closed on the malformed result and serves a
+single `shared` identity, which is indistinguishable from the file never having
+been written. That is the 2026-07-24 incident, and the round-trip check exists so a
+render carrying it cannot reach `/etc`.
+
+Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
+metacharacters, no newlines) only to `common` and `env/*`. Applying them to
+`centralis/*` would reject data that renders perfectly well — the validator matches
+the renderer that will consume the path, not a single global rule.
+
 `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
 rule expressible as exactly one path, with no wildcard over `env/*` able to
 accidentally re-include it.

infra/openbao/seed-from-sops.sh

@@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do
   # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
   # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
   # SEED time — before anything depends on it — rather than at deploy time.
-  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
+  # Which renderer will consume this path decides which values are legal, so the
+  # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
+  # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
+  # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
+  # render_centralis_secrets and its OpenBao counterpart, where every byte inside
+  # '...' is literal — so applying the dotenv rules to it rejects data that renders
+  # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
+  quoted=0
+  case "$target" in centralis.*) quoted=1 ;; esac
+
+  reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
 import json, os, re, sys
 
 doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
@@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc:
     sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
     sys.exit(1)
 
+# 1 => destined for a single-quoted env file; 0 => raw dotenv.
+QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
+
 KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 UNSAFE = set("`$\\\"'")
 
@@ -141,12 +154,13 @@ for key, val in doc.items():
         val = ""
     if not isinstance(val, str):
         val = json.dumps(val, separators=(",", ":"))
-    if "\n" in val or "\r" in val:
-        bad.append(f"{key}: contains a newline")
-        continue
-    if UNSAFE & set(val):
-        bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
-        continue
+    if not QUOTED:
+        if "\n" in val or "\r" in val:
+            bad.append(f"{key}: contains a newline")
+            continue
+        if UNSAFE & set(val):
+            bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
+            continue
     out[key] = val
 
 if bad: